这里有一些大文件一天内就要来，不是很频繁，每天2-3个，它们被转换成json格式。
文件内容如下所示：

[
    {
        "spa_ref_data": {
            "approval_action": "New",
            "spa_ref_no": "6500781413",
            "begin_date": null,
            "end_date": "20191009",
            "doc_file_name": "LEN_SPA_6500781413.json",
            "LEN_V": "v1",
            "version_no": null,
            "spa_ref_id": null,
            "spa_ref_notes": "MC00020544",
            "vend_code": "LEN"
        },
        "cust_data": [
            {
                "cust_name": null,
                "cust_no": null,
                "cust_type": "E",
                "state": null,
                "country": null
            },
            {
                "cust_name": null,
                "cust_no": null,
                "cust_type": "C",
                "state": null,
                "country": null
            }
        ],
        "product_data": [
            {
                "mfg_partno": "40AH0135US",
                "std_price": null,
                "rebate_amt": "180",
                "max_spa_qty": null,
                "rebate_type": null,
                "min_spa_qty": null,
                "min_cust_qty": null,
                "max_cust_qty": null,
                "begin_date": "20180608",
                "end_date": null
            },
            {
                "mfg_partno": "40AJ0135US",
                "std_price": null,
                "rebate_amt": "210",
                "max_spa_qty": null,
                "rebate_type": null,
                "min_spa_qty": null,
                "min_cust_qty": null,
                "max_cust_qty": null,
                "begin_date": "20180608",
                "end_date": null
            }
        ]
    },
    {
        "spa_ref_data": {
            "approval_action": "New",
            "spa_ref_no": "5309745006",
            "begin_date": null,
            "end_date": "20190426",
            "doc_file_name": "LEN_SPA_5309745006.json",
            "LEN_V": "v1",
            "version_no": null,
            "spa_ref_id": null,
            "spa_ref_notes": "MC00020101",
            "vend_code": "LEN"
        },
        "cust_data": [
            {
                "cust_name": null,
                "cust_no": null,
                "cust_type": "E",
                "state": null,
                "country": null
            },
            {
                "cust_name": null,
                "cust_no": null,
                "cust_type": "C",
                "state": null,
                "country": null
            }
        ],
        "product_data": [
            {
                "mfg_partno": "10M8S0HU00",
                "std_price": null,
                "rebate_amt": "698",
                "max_spa_qty": null,
                "rebate_type": null,
                "min_spa_qty": null,
                "min_cust_qty": null,
                "max_cust_qty": null,
                "begin_date": "20180405",
                "end_date": null
            },
            {
                "mfg_partno": "20K5S0CM00",
                "std_price": null,
                "rebate_amt": "1083",
                "max_spa_qty": null,
                "rebate_type": null,
                "min_spa_qty": null,
                "min_cust_qty": null,
                "max_cust_qty": null,
                "begin_date": "20180405",
                "end_date": null
            }
        ]
    }
]

这是一个模拟数据文件。实际上，它是一个长度为30000+的数组。
我的目标是比较下一个和最新的。获取更改的数据。
领导说我必须使用大数据技术。而且表演一定很好。
我们使用apachenifi和hadoop大数据工具来实现这一点。
有什么建议吗？