分布式调度系统设计与实现(二)

  |   0 评论   |   1,926 浏览

任务DAG的作用

  • 查看任务血缘关系

  • 实时查看任务流程的运行状态

  • 回溯历史运行统计信息

任务DAG的实现

任务依赖信息的存储

创建任务时,如果任务有前置依赖任务,则以逗号分隔符的形式,记录下前置依赖

image.png

Job任务表增加一列“DepJobIDs”,如下图

image.png

表结构说明

如Job任务表中有如下记录

image.png

对应的DAG图如下

image.png

生成dagreD3-DAG所需的JSON格式数据

生成edges的JSON数据

[
    {
        "jobId": "705【报表平台】XX-订单成功率日报",
        "dependenceInfo": "group,orderrate",
        "cost": "0毫秒",
        "id": 705,
        "did": 672,
        "dependenceJobId": "672【报表平台】XX集团分类"
    },
    {
        "jobId": "705【报表平台】XX-订单成功率日报",
        "dependenceInfo": "group,orderrate",
        "cost": "1小时55分钟56秒0毫秒",
        "id": 705,
        "did": 704,
        "dependenceJobId": "704【报表平台】XX-订单请求数据ETL"
    },
    {
        "jobId": "704【报表平台】XX-订单请求数据ETL",
        "dependenceInfo": "hw_cm_info",
        "cost": "0毫秒",
        "id": 704,
        "did": 619,
        "dependenceJobId": "619【mysql】接入cm_info"
    },
    {
        "jobId": "780【邮件】cm订单成功率监控-日报",
        "dependenceInfo": "可定,成功",
        "cost": "20小时55分钟27秒919毫秒",
        "id": 780,
        "did": 705,
        "dependenceJobId": "705【报表平台】XX-订单成功率日报"
    }
]

生成vertices的JSON数据

[
    {
        "nodeName": "672【报表平台】XX集团分类",
        "task": {
            "jobId": 672,
            "workerId": 7,
            "beginTime": 1527274800000,
            "endTime": 1527274933000,
            "id": 289783,
            "job": {
                "type": "【报表平台】",
                "name": "【报表平台】XX集团分类",
                "id": 672,
                "status": 2
            },
            "cycle": "2018-05-26",
            "status": 10
        },
        "cost": "2分钟13秒0毫秒",
        "style": "fill: #5CB85C"
    },
    {
        "nodeName": "704【报表平台】XX-订单请求数据ETL",
        "task": {
            "jobId": 704,
            "workerId": 7,
            "beginTime": 1527267931000,
            "endTime": 1527267977000,
            "id": 289591,
            "job": {
                "type": "【报表平台】",
                "name": "【报表平台】XX-订单请求数据ETL",
                "status": 2
            },
            "cycle": "2018-05-26",
            "reRun": false,
            "notify": true,
            "status": 10
        },
        "cost": "46秒0毫秒",
        "style": "fill: #5CB85C"
    },
    {
        "nodeName": "705【报表平台】XX-订单成功率日报",
        "task": {
            "jobId": 705,
            "workerId": 7,
            "beginTime": 1527274933000,
            "endTime": 1527275041000,
            "id": 289595,
            "job": {
                "name": "【报表平台】XX-订单成功率日报",
                "info": "【报表平台】XX-订单成功率日报",
                "status": 2
            },
            "cycle": "2018-05-26",
            "reRun": false,
            "notify": true,
            "status": 10
        },
        "cost": "1分钟48秒0毫秒",
        "style": "fill: #5CB85C"
    },
    {
        "nodeName": "619【mysql】接入cm_info",
        "task": {
            "jobId": 619,
            "workerId": 5,
            "beginTime": 1527267900000,
            "endTime": 1527267931000,
            "id": 289590,
            "job": {
                "name": "【mysql】接入cm_info",
                "id": 619,
                "info": "【mysql】接入cm_info",
                "status": 2
            },
            "cycle": "2018-05-26",
            "reRun": false,
            "notify": true,
            "status": 10
        },
        "cost": "31秒0毫秒",
        "style": "fill: #5CB85C"
    },
    {
        "nodeName": "780【邮件】cm订单成功率监控-日报",
        "task": {
            "workerId": 0,
            "reRun": false,
            "notify": true,
            "status": 0
        }
    }
]

生成DAG图

dagre-d3介绍

Dagre是一个JavaScript库,可以很容易地在客户端上绘制有向图。 dagre-d3库作为dagre的前端,提供使用D3的实际渲染。

dagre-d3运行时需要两个JSON对象,edges和vertices,分别代表DAG的边和顶点

dagre-d3的GitHub地址

dagre-d3基本用法

var edges = []
var vertices = [] 
$(document).ready(function() {
    var g = new dagreD3.graphlib.Graph().setGraph({});
    vertices.forEach(function(vertice) {
        var value = vertice;
        value.label = vertice.nodeName;
        value.rx = value.ry = 5;
        var task = vertice.task;
        if (task.workerId == 0) {
            vertice.style = "fill:#FFFACD"
        }
        g.setNode(vertice.nodeName, value);
    });
    edges.forEach(function(edge) {
        g.setEdge(edge.dependenceJobId, edge.jobId, {
            label: "等待时长:" + edge.cost
        });
    });
    var svg = d3.select("svg"),
    inner = svg.select("g");
    // Set up zoom support
    var zoom = d3.behavior.zoom().on("zoom",
    function() {
        inner.attr("transform", "translate(" + d3.event.translate + ")" + "scale(" + d3.event.scale + ")");
    });
    svg.call(zoom);
    // Create the renderer
    var render = new dagreD3.render();
    // Run the renderer. This is what draws the final graph.
    render(inner, g);
});

生成任务DAG

image.png

读后有收获可以支付宝请作者喝咖啡