Blame view

app/commands/commands.py 11.9 KB
e0778a0d   hitier   New fake_lesia_na...
1
2
import csv
import os
335db4c0   hitier   Allow randoming c...
3
import sys
a17327bf   hitier   New db feeding co...
4
5
6
import click
import random

21724174   hitier   Fix mysql_lesia_u...
7
from flask import current_app
74494ea1   hitier   cli.create_db hol...
8
from sqlalchemy.exc import OperationalError, IntegrityError
67814e41   hitier   New cli command f...
9
from sqlalchemy.orm.exc import NoResultFound
e817ff5e   hitier   Random Charge Age...
10
from sqlalchemy.sql import func
a17327bf   hitier   New db feeding co...
11
12
13
14
from sqlalchemy.ext.automap import automap_base
from sqlalchemy.orm import Session
from sqlalchemy import create_engine

a5a365e8   hitier   Move User model t...
15
from app.models import db, Agent, Service, Project, Capacity, Period, Charge
149f7875   hitier   Now user_add acce...
16
from app.auth.models import User, _nameToRole
a17327bf   hitier   New db feeding co...
17

a17327bf   hitier   New db feeding co...
18
19
from . import bp

335db4c0   hitier   Allow randoming c...
20

67814e41   hitier   New cli command f...
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
@bp.cli.command("feed_from_irap")
@click.option('--csv-file', '-f', 'csv_file_name', help="the csv file path to feed from")
def feed_from_irap(csv_file_name):
    """
    Use an Irap csv charges files and feed db with

    :param csv_file_name:
    :return:
    """

    rows = []

    with open(csv_file_name, newline='') as csvfile:
        csvreader = csv.DictReader(csvfile, delimiter=',', quotechar='"')
        for row in csvreader:
            # print('\n'.join(row.keys()))
            # break
28a7e0a2   hitier   Strip row values ...
38
39
            # Remove any leading/trailing spaces
            row = {k: v.strip() for k, v in row.items()}
67814e41   hitier   New cli command f...
40
41
            rows.append(row)

28a7e0a2   hitier   Strip row values ...
42
43
44
45
    firstname_key = 'NOM'
    secondname_key = 'prénom'
    project_key = 'PROJETS'
    service_key = 'Groupe métier'
61e60a1f   hitier   Now add default c...
46
47
    # typology_title = 'TYPOLOGIE'
    # thematic_title = 'thématique'
67814e41   hitier   New cli command f...
48

28a7e0a2   hitier   Strip row values ...
49
50
51
    # Get the columns values
    #
    projects = [r[project_key] for r in rows]
67814e41   hitier   New cli command f...
52
    projects = sorted(set(projects))
28a7e0a2   hitier   Strip row values ...
53
    agents = [(r[firstname_key], r[secondname_key].strip()) for r in rows]
67814e41   hitier   New cli command f...
54
    agents = sorted(set(agents))
28a7e0a2   hitier   Strip row values ...
55
    services = [r[service_key] for r in rows]
67814e41   hitier   New cli command f...
56
57
    services = sorted(set(services))

28a7e0a2   hitier   Strip row values ...
58
59
60
61
62
63
64
    # thematics = [r[thematic_title] for r in rows]
    # thematics = sorted(set(thematics))
    # typologies = [r[typology_title] for r in rows]
    # typologies = sorted(set(typologies))

    # Feed agents from column
    #
67814e41   hitier   New cli command f...
65
66
67
68
    for a in agents:
        n_a = Agent(firstname=a[0], secondname=a[1])
        db.session.add(n_a)
    db.session.commit()
67814e41   hitier   New cli command f...
69

28a7e0a2   hitier   Strip row values ...
70
71
    # Feed projects from column
    #
67814e41   hitier   New cli command f...
72
73
74
75
76
    for p in projects:
        n_p = Project(name=p)
        db.session.add(n_p)
    db.session.commit()

28a7e0a2   hitier   Strip row values ...
77
78
    # Feed services from column
    #
67814e41   hitier   New cli command f...
79
80
81
82
83
    for s in services:
        n_s = Service(name=s)
        db.session.add(n_s)
    db.session.commit()

770d2129   hitier   Now add only non-...
84
    # Feed periods names
61e60a1f   hitier   Now add default c...
85
86
    # Todo: are statically built,
    #       should come from year column name.
770d2129   hitier   Now add only non-...
87
    #
67814e41   hitier   New cli command f...
88
89
90
91
92
    for p in range(2011, 2030):
        n_p = Period(name=f"{p}")
        db.session.add(n_p)
    db.session.commit()

61e60a1f   hitier   Now add default c...
93
94
95
96
    # Add one default capacity
    db.session.add(Capacity(name="Travailleur"))
    db.session.commit()

770d2129   hitier   Now add only non-...
97
98
99
100
101
    # Now feed the charges.
    #
    # At least one for each csv row
    # At most one for each year
    #
67814e41   hitier   New cli command f...
102
    for r in rows:
28a7e0a2   hitier   Strip row values ...
103
104
105
        p = Project.query.filter(Project.name == r[project_key]).one()
        a = Agent.query.filter(Agent.firstname == r[firstname_key], Agent.secondname == r[secondname_key]).one()
        s = Service.query.filter(Service.name == r[service_key]).one()
61e60a1f   hitier   Now add default c...
106
        c = Capacity.query.first()
67814e41   hitier   New cli command f...
107
108
109
        for period_name in range(2011, 2030):
            t = Period.query.filter(Period.name == period_name).one()
            charge = r[f"{period_name}"]
67814e41   hitier   New cli command f...
110
            try:
28a7e0a2   hitier   Strip row values ...
111
                charge = int(100 * float(charge))
67814e41   hitier   New cli command f...
112
            except ValueError:
67814e41   hitier   New cli command f...
113
                charge = 0
770d2129   hitier   Now add only non-...
114
115
            if charge == 0:
                continue
67814e41   hitier   New cli command f...
116
117
118
            n_c = Charge(agent_id=a.id,
                         project_id=p.id,
                         service_id=s.id,
61e60a1f   hitier   Now add default c...
119
                         capacity_id=c.id,
67814e41   hitier   New cli command f...
120
121
122
123
124
125
                         period_id=t.id,
                         charge_rate=charge)
            db.session.add(n_c)
    db.session.commit()


e0778a0d   hitier   New fake_lesia_na...
126
127
@bp.cli.command("fake_lesia_names")
def fake_lesia_names():
14f36f55   hitier   Move lesia init c...
128
    """
e0778a0d   hitier   New fake_lesia_na...
129
130
131
132
133
    Extract fake name from resources files to change names in db.
    Mainly after a lesia import, for confidential reasons

    Changes nams in tables:

14f36f55   hitier   Move lesia init c...
134
135
136
137
138
139
        - services
        - capacities
        - projects
    :return:
    """

e0778a0d   hitier   New fake_lesia_na...
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
    current_app.logger.info("Faking names from resources files")
    # get  resources files
    #
    # 1- projects
    #
    fake_projects_file = os.path.join(current_app.config['PDC_RESOURCES_DIR'], 'fake-db-names', 'fake-projects.txt')
    with open(fake_projects_file, newline='') as csvfile:
        spamreader = csv.reader(csvfile, delimiter=';', quotechar='|')
        fake_projects_names = [', '.join(row) for row in spamreader]
    fake_projects_names_iterator = iter(fake_projects_names)

    # 2- functions/capacities
    #
    fake_capacities_file = os.path.join(current_app.config['PDC_RESOURCES_DIR'], 'fake-db-names',
                                        'fake-capacities.txt')
    with open(fake_capacities_file, newline='') as csvfile:
        spamreader = csv.reader(csvfile, delimiter=';', quotechar='|')
        fake_capacities_names = [row for [row] in spamreader]
    fake_capacities_names_iterator = iter(fake_capacities_names)

    # 3- services
    #
    fake_services_file = os.path.join(current_app.config['PDC_RESOURCES_DIR'], 'fake-db-names',
                                      'fake-services.txt')
    with open(fake_services_file, newline='') as csvfile:
        spamreader = csv.reader(csvfile, delimiter=';', quotechar='|')
        fake_services_names = [row for row in spamreader]
    fake_services_names_iterator = iter(fake_services_names)

    # Skip columns names
    #
    next(fake_projects_names_iterator)
    next(fake_capacities_names_iterator)
    next(fake_services_names_iterator)

    for s in Service.query.all():
        next_service = next(fake_services_names_iterator)
        s.name = next_service[0]
        s.abbr = next_service[1]

    for p in Project.query.all():
        p.name = next(fake_projects_names_iterator)

    for c in Capacity.query.all():
        c.name = next(fake_capacities_names_iterator)

    db.session.commit()

14f36f55   hitier   Move lesia init c...
188

a17327bf   hitier   New db feeding co...
189
190
@bp.cli.command("feed_from_lesia")
def feed_from_lesia():
74494ea1   hitier   cli.create_db hol...
191
192
    """
    Feed db with agents from a lesia like mysql database.
d8a6b942   hitier   More install doc
193

e0778a0d   hitier   New fake_lesia_na...
194
    Remember to configure the proper database uri in the db_config.py file.
d8a6b942   hitier   More install doc
195
    """
14f36f55   hitier   Move lesia init c...
196
197
    from .lesia_db import lesia_agent, lesia_session, lesia_service, lesia_project, \
        lesia_fonction, lesia_periods, lesia_affectation
1002c994   hitier   Also feed ids fro...
198

f1026243   hitier   Feed db periods a...
199
    agents = lesia_session.query(lesia_agent).all()
a17327bf   hitier   New db feeding co...
200
    for a in agents:
1002c994   hitier   Also feed ids fro...
201
        n_a = Agent(id=a.IDagent, firstname=a.nom, secondname=a.prenom)
a17327bf   hitier   New db feeding co...
202
203
204
        db.session.add(n_a)
    db.session.commit()

f1026243   hitier   Feed db periods a...
205
    services = lesia_session.query(lesia_service).all()
a17327bf   hitier   New db feeding co...
206
    for s in services:
1002c994   hitier   Also feed ids fro...
207
        n_s = Service(id=s.id, name=s.nom, abbr=s.abbreviation)
a17327bf   hitier   New db feeding co...
208
209
210
        db.session.add(n_s)
    db.session.commit()

f1026243   hitier   Feed db periods a...
211
    projects = lesia_session.query(lesia_project).all()
a17327bf   hitier   New db feeding co...
212
    for p in projects:
1002c994   hitier   Also feed ids fro...
213
        n_p = Project(id=p.id, name=p.nom)
a17327bf   hitier   New db feeding co...
214
215
216
        db.session.add(n_p)
    db.session.commit()

f1026243   hitier   Feed db periods a...
217
    fonctions = lesia_session.query(lesia_fonction).all()
a17327bf   hitier   New db feeding co...
218
    for f in fonctions:
1002c994   hitier   Also feed ids fro...
219
        n_c = Capacity(id=f.id, name=f.nom)
a17327bf   hitier   New db feeding co...
220
221
222
        db.session.add(n_c)
    db.session.commit()

f1026243   hitier   Feed db periods a...
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
    periods = lesia_session.query(lesia_periods)
    for p in periods:
        n_p = Period(name=p.id_semestre)
        db.session.add(n_p)
    db.session.commit()

    affectations = lesia_session.query(lesia_affectation)
    for f in affectations:
        p = Period.query.filter(Period.name == f.semestre_id).one()
        n_c = Charge(agent_id=f.agent_id,
                     project_id=f.projet_id,
                     service_id=f.service_id,
                     capacity_id=f.fonction_id,
                     period_id=p.id,
                     charge_rate=f.charge)
        db.session.add(n_c)
    db.session.commit()

335db4c0   hitier   Allow randoming c...
241

a17327bf   hitier   New db feeding co...
242
@bp.cli.command("feed_periods")
fa86be39   hitier   Enhance error cat...
243
244
245
246
247
@click.option('--begin-year', '-b', 'begin_year', default=2005, help="the start year to begin periods with")
@click.option('--end-year', '-e', 'end_year', default=2025, help="the last year to end periods with")
def feed_periods(begin_year, end_year):
    """ Fill in the periods names in the database. """
    for y in range(begin_year, end_year):
a17327bf   hitier   New db feeding co...
248
249
250
251
        for s in ['S1', 'S2']:
            period_name = "{}_{}".format(y, s)
            p = Period(name=period_name)
            db.session.add(p)
fa86be39   hitier   Enhance error cat...
252
253
254
255
    try:
        db.session.commit()
    except IntegrityError:
        current_app.logger.error("Periods already exist")
a17327bf   hitier   New db feeding co...
256
257


d8a6b942   hitier   More install doc
258
259
260
261
@bp.cli.command("feed_random_charges")
@click.option('--agent', '-a', 'agent', default=None, help="the agent id you want to charge")
def feed_random_charges(agent):
    """ Randomly fill in the agents charges. """
a17327bf   hitier   New db feeding co...
262
    for i in range(0, 100):
335db4c0   hitier   Allow randoming c...
263
264
265
266
        if agent is None:
            agent_id = random.choice([i for (i,) in db.session.query(Agent.id).all()])
        else:
            agent_id = int(agent)
a17327bf   hitier   New db feeding co...
267
268
269
270
271
        project_id = random.choice([i for (i,) in db.session.query(Project.id).all()])
        service_id = random.choice([i for (i,) in db.session.query(Service.id).all()])
        capacity_id = random.choice([i for (i,) in db.session.query(Capacity.id).all()])
        period_id = random.choice([i for (i,) in db.session.query(Period.id).all()])
        percent = random.choice(range(10, 110, 10))
e817ff5e   hitier   Random Charge Age...
272
273
274
275
276
        # check max agent charge for the period
        total_charge = db.session.query(func.sum(Charge.charge_rate).label("total_charge")) \
            .filter(Charge.agent_id == agent_id,
                    Charge.period_id == period_id
                    ).scalar()
a5a365e8   hitier   Move User model t...
277
        if total_charge is not None and (total_charge + percent) >= 100:
e817ff5e   hitier   Random Charge Age...
278
279
            print("Skipping agent {} for period {}".format(agent_id, period_id))
            continue
335db4c0   hitier   Allow randoming c...
280
        charge = Charge(agent_id=agent_id,
a17327bf   hitier   New db feeding co...
281
282
283
284
285
286
287
288
289
290
                        project_id=project_id,
                        service_id=service_id,
                        capacity_id=capacity_id,
                        period_id=period_id,
                        charge_rate=percent)
        print("adding {}_{}_{}_{}_{}_{}".format(agent_id, project_id, service_id, capacity_id, period_id, percent))
        db.session.add(charge)
        db.session.commit()


d8a6b942   hitier   More install doc
291
@bp.cli.command('user_delete')
a17327bf   hitier   New db feeding co...
292
@click.argument('user_id')
d8a6b942   hitier   More install doc
293
294
def user_delete(user_id):
    """Delete the user by given id (see user_show_all")."""
a17327bf   hitier   New db feeding co...
295
296
297
298
299
300
301
    user = User.query.get(user_id)
    db.session.delete(user)
    db.session.commit()


@bp.cli.command('create_db')
def create_db():
74494ea1   hitier   cli.create_db hol...
302
303
304
305
306
    """
    Create the database structure. Database should be empty.

    configure the proper database uri in the db_config.py file.
    """
a17327bf   hitier   New db feeding co...
307
    db.create_all()
82642adb   hitier   Update routes and...
308
309
    admin = User(email='admin@nowhere.org', name='admin', login='admin', role='admin')
    admin.set_password('admin')
74494ea1   hitier   cli.create_db hol...
310
311
312
313
314
315
316
    sqlite_uri = db.engine.url.__str__() if 'sqlite' in db.engine.url.__str__() else None
    try:
        db.session.add(admin)
        db.session.commit()
    except IntegrityError:
        current_app.logger.error("User admin already exists, database should be empty at create")
        if sqlite_uri:
14f36f55   hitier   Move lesia init c...
317
            current_app.logger.error("see " + sqlite_uri)
74494ea1   hitier   cli.create_db hol...
318
319
320
        sys.exit(-1)

    if sqlite_uri:
14f36f55   hitier   Move lesia init c...
321
        current_app.logger.info("Created sqlite db: " + sqlite_uri)
a17327bf   hitier   New db feeding co...
322
323


d8a6b942   hitier   More install doc
324
@bp.cli.command('user_add')
a17327bf   hitier   New db feeding co...
325
326
@click.argument('email')
@click.argument('name')
3ed62121   hitier   Add login to user
327
@click.argument('login')
a17327bf   hitier   New db feeding co...
328
@click.argument('password')
149f7875   hitier   Now user_add acce...
329
330
@click.argument('role')
def user_add(email, name, login, password, role):
d8a6b942   hitier   More install doc
331
    """ Add a new user in db."""
149f7875   hitier   Now user_add acce...
332
333
    user = User.query.filter(User.name == name).one_or_none()
    if (user):
82642adb   hitier   Update routes and...
334
335
        current_app.logger.error(f"user already exists {name}")
        return
149f7875   hitier   Now user_add acce...
336
    user = User(email=email, name=name, login=login, password=password, role=role)
a17327bf   hitier   New db feeding co...
337
338
    db.session.add(user)
    db.session.commit()
82642adb   hitier   Update routes and...
339
    current_app.logger.info(f"added {name}")
a17327bf   hitier   New db feeding co...
340
341


149f7875   hitier   Now user_add acce...
342
343
344
345
346
347
@bp.cli.command('show_roles')
def show_roles():
    """ List all available roles for a user"""
    print("\n".join(list(_nameToRole)))


d8a6b942   hitier   More install doc
348
349
350
@bp.cli.command('user_show_all')
def user_show_all():
    """ Show all users in db."""
4113c4f3   hitier   Fix user_show_all
351
352
    print("{:<5} {:<15} {:<15} {:<15}".format('id', 'name', 'login', 'email'))
    print("{:<5} {:<15} {:<15} {:<15}".format('-' * 5, '-' * 15, '-' * 15, '-' * 15))
a17327bf   hitier   New db feeding co...
353
354
    for user in User.query.all():
        print(user.login)
4113c4f3   hitier   Fix user_show_all
355
        print("{:<5} {:<15} {:<15} {:<15}".format(
a17327bf   hitier   New db feeding co...
356
357
358
            user.id,
            user.name,
            user.login,
a17327bf   hitier   New db feeding co...
359
360
            user.email
        ))