CoOpenData/controller.py at main · TRACE-LAC/CoOpenData · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
import sys
import os
import os.path

# import config as cf
import model
# assert cf


def download_sivigila_data(years, events):
    '''
    Downloads data from Sivigila website and saves it in a csv file in the data folder.
    '''
    scrapper = model.download_sivigila_data(years, events)


def exit_server(scrapper):
    '''
    TODO Exits the server.
    '''
    model.exit_server(scrapper)


def preprocess_sivigila_data(years, events, years_save, save=True, data_path='data/Multicity/Data', save_path='data/Census_2017-2018/Raw/DEMOGRAFICO'):
    '''
    Preprocesses the data downloaded from Sivigila website.
    '''
    raw_data_path = os.path.join(os.path.abspath(os.getcwd())
                                        , data_path, 'Raw_data')
    data_clean, save_name = model.preprocess_sivigila_data(years, events, raw_data_path)
    if save:
        # Create folder for clean data
        clean_data_path = os.path.join(os.path.abspath(os.getcwd())
                                        , data_path, 'Clean_data')
        this_folder_events = ''
        for event in events:
            if ' ' in event:
                event = event.replace(' ', '-')
            this_folder_events += event + '_'
        save_dir = this_folder_events + '(' + str(years_save) + ')'
        if not os.path.isdir(os.path.join(clean_data_path, save_dir)):
            os.makedirs(os.path.join(clean_data_path, save_dir))
        # Save data as csv
        data_clean.to_csv(os.path.join(clean_data_path, save_dir, save_name), index=False)

    return data_clean, save_name


def preprocess_cesus_household( save=True, data_path='data/Census_2017-2018/Raw', save_path='data/Census_2017-2018/Clean_data/VIVIENDAS', save_data=True):
    '''
    Preprocesses the data downloaded from CENSUS website.
    '''
    raw_data_path = os.path.join(os.path.abspath(os.getcwd())
                                        , data_path)
    save = False
    if save_data is True:
        save = True
    if save is True:
        # Create folder for clean data
        clean_data_path = os.path.join(os.path.abspath(os.getcwd())
                                        , save_path, 'Clean_data')
        if not os.path.isdir(os.path.join(clean_data_path, clean_data_path)):
            os.makedirs(os.path.join(clean_data_path, clean_data_path))

    model.preprocess_cesus_household(raw_data_path, save_path, save=True)

    return save_path


def preprocess_cesus_demographic( save=True, data_path='data/Census_2017-2018/Raw', save_path='data/Census_2017-2018/Clean_data/DEMOGRAFICO', save_data=True):
    '''
    Preprocesses the data downloaded from CENSUS website.
    '''
    raw_data_path = os.path.join(os.path.abspath(os.getcwd())
                                        , data_path)
    save = False
    if save_data is True:
        save = True
    if save is True:
        # Create folder for clean data
        clean_data_path = os.path.join(os.path.abspath(os.getcwd())
                                        , save_path, 'Clean_data')
        if not os.path.isdir(os.path.join(clean_data_path, clean_data_path)):
            os.makedirs(os.path.join(clean_data_path, clean_data_path))

    model.preprocess_cesus_household(raw_data_path, save_path, save=True)

    return save_path