From 8eb6078b2ea4a8bf611f79149f93e059c62b9aec Mon Sep 17 00:00:00 2001 From: anuveyatsu Date: Mon, 23 Mar 2020 00:17:41 +0600 Subject: [PATCH] [checkpoints][xs]: save data so it's easy to develop new flows. --- .gitignore | 3 +++ process.py | 1 + 2 files changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 4d34fa8..3991ac8 100644 --- a/.gitignore +++ b/.gitignore @@ -144,3 +144,6 @@ static # put you virtual env names here env36/ + +# dataflows checkpoints +.checkpoints/ diff --git a/process.py b/process.py index 0177661..08661f6 100644 --- a/process.py +++ b/process.py @@ -23,6 +23,7 @@ Flow( load(f'{BASE_URL}{CONFIRMED}'), load(f'{BASE_URL}{RECOVERED}'), load(f'{BASE_URL}{DEATH}'), + checkpoint('load_data'), unpivot(unpivoting_fields, extra_keys, extra_value), find_replace([{'name': 'Date', 'patterns': [{'find': '/', 'replace': '-'}]}]), to_normal_date,