Blame view

app/commands/commands.py 13.3 KB
e0778a0d   hitier   New fake_lesia_na...
1
2
import csv
import os
335db4c0   hitier   Allow randoming c...
3
import sys
a17327bf   hitier   New db feeding co...
4
5
6
import click
import random

21724174   hitier   Fix mysql_lesia_u...
7
from flask import current_app
74494ea1   hitier   cli.create_db hol...
8
from sqlalchemy.exc import OperationalError, IntegrityError
67814e41   hitier   New cli command f...
9
from sqlalchemy.orm.exc import NoResultFound
e817ff5e   hitier   Random Charge Age...
10
from sqlalchemy.sql import func
a17327bf   hitier   New db feeding co...
11
12
13
14
from sqlalchemy.ext.automap import automap_base
from sqlalchemy.orm import Session
from sqlalchemy import create_engine

a5a365e8   hitier   Move User model t...
15
from app.models import db, Agent, Service, Project, Capacity, Period, Charge
946dda30   hitier   New cli command u...
16
from app.auth.models import User, _nameToRole, _roleToName
a17327bf   hitier   New db feeding co...
17

a17327bf   hitier   New db feeding co...
18
19
from . import bp

335db4c0   hitier   Allow randoming c...
20

946dda30   hitier   New cli command u...
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
@bp.cli.command('create_db')
def create_db():
    """
    Create the database structure. Database should be empty.

    configure the proper database uri in the db_config.py file.
    """
    db.create_all()
    admin = User(email='admin@nowhere.org', name='admin', login='admin', role='admin')
    admin.set_password('admin')
    sqlite_uri = db.engine.url.__str__() if 'sqlite' in db.engine.url.__str__() else None
    try:
        db.session.add(admin)
        db.session.commit()
    except IntegrityError:
        current_app.logger.error("User admin already exists, database should be empty at create")
        if sqlite_uri:
            current_app.logger.error("see " + sqlite_uri)
        sys.exit(-1)

    if sqlite_uri:
        current_app.logger.info("Created sqlite db: " + sqlite_uri)


67814e41   hitier   New cli command f...
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
@bp.cli.command("feed_from_irap")
@click.option('--csv-file', '-f', 'csv_file_name', help="the csv file path to feed from")
def feed_from_irap(csv_file_name):
    """
    Use an Irap csv charges files and feed db with

    :param csv_file_name:
    :return:
    """

    rows = []

    with open(csv_file_name, newline='') as csvfile:
        csvreader = csv.DictReader(csvfile, delimiter=',', quotechar='"')
        for row in csvreader:
            # print('\n'.join(row.keys()))
            # break
28a7e0a2   hitier   Strip row values ...
62
63
            # Remove any leading/trailing spaces
            row = {k: v.strip() for k, v in row.items()}
67814e41   hitier   New cli command f...
64
65
            rows.append(row)

28a7e0a2   hitier   Strip row values ...
66
67
68
69
    firstname_key = 'NOM'
    secondname_key = 'prénom'
    project_key = 'PROJETS'
    service_key = 'Groupe métier'
61e60a1f   hitier   Now add default c...
70
71
    # typology_title = 'TYPOLOGIE'
    # thematic_title = 'thématique'
67814e41   hitier   New cli command f...
72

28a7e0a2   hitier   Strip row values ...
73
74
75
    # Get the columns values
    #
    projects = [r[project_key] for r in rows]
67814e41   hitier   New cli command f...
76
    projects = sorted(set(projects))
28a7e0a2   hitier   Strip row values ...
77
    agents = [(r[firstname_key], r[secondname_key].strip()) for r in rows]
67814e41   hitier   New cli command f...
78
    agents = sorted(set(agents))
28a7e0a2   hitier   Strip row values ...
79
    services = [r[service_key] for r in rows]
67814e41   hitier   New cli command f...
80
81
    services = sorted(set(services))

28a7e0a2   hitier   Strip row values ...
82
83
84
85
86
87
88
    # thematics = [r[thematic_title] for r in rows]
    # thematics = sorted(set(thematics))
    # typologies = [r[typology_title] for r in rows]
    # typologies = sorted(set(typologies))

    # Feed agents from column
    #
67814e41   hitier   New cli command f...
89
90
91
92
    for a in agents:
        n_a = Agent(firstname=a[0], secondname=a[1])
        db.session.add(n_a)
    db.session.commit()
67814e41   hitier   New cli command f...
93

28a7e0a2   hitier   Strip row values ...
94
95
    # Feed projects from column
    #
67814e41   hitier   New cli command f...
96
97
98
99
100
    for p in projects:
        n_p = Project(name=p)
        db.session.add(n_p)
    db.session.commit()

28a7e0a2   hitier   Strip row values ...
101
102
    # Feed services from column
    #
67814e41   hitier   New cli command f...
103
104
105
106
107
    for s in services:
        n_s = Service(name=s)
        db.session.add(n_s)
    db.session.commit()

770d2129   hitier   Now add only non-...
108
    # Feed periods names
61e60a1f   hitier   Now add default c...
109
110
    # Todo: are statically built,
    #       should come from year column name.
770d2129   hitier   Now add only non-...
111
    #
67814e41   hitier   New cli command f...
112
113
114
115
116
    for p in range(2011, 2030):
        n_p = Period(name=f"{p}")
        db.session.add(n_p)
    db.session.commit()

61e60a1f   hitier   Now add default c...
117
118
119
120
    # Add one default capacity
    db.session.add(Capacity(name="Travailleur"))
    db.session.commit()

770d2129   hitier   Now add only non-...
121
122
123
124
125
    # Now feed the charges.
    #
    # At least one for each csv row
    # At most one for each year
    #
67814e41   hitier   New cli command f...
126
    for r in rows:
28a7e0a2   hitier   Strip row values ...
127
128
129
        p = Project.query.filter(Project.name == r[project_key]).one()
        a = Agent.query.filter(Agent.firstname == r[firstname_key], Agent.secondname == r[secondname_key]).one()
        s = Service.query.filter(Service.name == r[service_key]).one()
61e60a1f   hitier   Now add default c...
130
        c = Capacity.query.first()
67814e41   hitier   New cli command f...
131
132
133
        for period_name in range(2011, 2030):
            t = Period.query.filter(Period.name == period_name).one()
            charge = r[f"{period_name}"]
796a5688   hitier   Add % sign in cha...
134
135
            # Charge are stored as percent in db, but as fraction of ETP in irap csv
            # we make the conversion here.
67814e41   hitier   New cli command f...
136
            try:
28a7e0a2   hitier   Strip row values ...
137
                charge = int(100 * float(charge))
67814e41   hitier   New cli command f...
138
            except ValueError:
67814e41   hitier   New cli command f...
139
                charge = 0
770d2129   hitier   Now add only non-...
140
141
            if charge == 0:
                continue
67814e41   hitier   New cli command f...
142
143
144
            n_c = Charge(agent_id=a.id,
                         project_id=p.id,
                         service_id=s.id,
61e60a1f   hitier   Now add default c...
145
                         capacity_id=c.id,
67814e41   hitier   New cli command f...
146
147
148
149
150
151
                         period_id=t.id,
                         charge_rate=charge)
            db.session.add(n_c)
    db.session.commit()


946dda30   hitier   New cli command u...
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
@bp.cli.command("feed_from_lesia")
def feed_from_lesia():
    """
    Feed db with agents from a lesia like mysql database.

    Remember to configure the proper database uri in the db_config.py file.
    """
    from .lesia_db import lesia_agent, lesia_session, lesia_service, lesia_project, \
        lesia_fonction, lesia_periods, lesia_affectation

    agents = lesia_session.query(lesia_agent).all()
    for a in agents:
        n_a = Agent(id=a.IDagent, firstname=a.nom, secondname=a.prenom)
        db.session.add(n_a)
    db.session.commit()

    services = lesia_session.query(lesia_service).all()
    for s in services:
        n_s = Service(id=s.id, name=s.nom, abbr=s.abbreviation)
        db.session.add(n_s)
    db.session.commit()

    projects = lesia_session.query(lesia_project).all()
    for p in projects:
        n_p = Project(id=p.id, name=p.nom)
        db.session.add(n_p)
    db.session.commit()

    fonctions = lesia_session.query(lesia_fonction).all()
    for f in fonctions:
        n_c = Capacity(id=f.id, name=f.nom)
        db.session.add(n_c)
    db.session.commit()

    periods = lesia_session.query(lesia_periods)
    for p in periods:
        n_p = Period(name=p.id_semestre)
        db.session.add(n_p)
    db.session.commit()

    affectations = lesia_session.query(lesia_affectation)
    for f in affectations:
        p = Period.query.filter(Period.name == f.semestre_id).one()
        n_c = Charge(agent_id=f.agent_id,
                     project_id=f.projet_id,
                     service_id=f.service_id,
                     capacity_id=f.fonction_id,
                     period_id=p.id,
                     charge_rate=f.charge)
        db.session.add(n_c)
    db.session.commit()


e0778a0d   hitier   New fake_lesia_na...
205
206
@bp.cli.command("fake_lesia_names")
def fake_lesia_names():
14f36f55   hitier   Move lesia init c...
207
    """
e0778a0d   hitier   New fake_lesia_na...
208
209
210
211
212
    Extract fake name from resources files to change names in db.
    Mainly after a lesia import, for confidential reasons

    Changes nams in tables:

14f36f55   hitier   Move lesia init c...
213
214
215
216
217
218
        - services
        - capacities
        - projects
    :return:
    """

e0778a0d   hitier   New fake_lesia_na...
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
    current_app.logger.info("Faking names from resources files")
    # get  resources files
    #
    # 1- projects
    #
    fake_projects_file = os.path.join(current_app.config['PDC_RESOURCES_DIR'], 'fake-db-names', 'fake-projects.txt')
    with open(fake_projects_file, newline='') as csvfile:
        spamreader = csv.reader(csvfile, delimiter=';', quotechar='|')
        fake_projects_names = [', '.join(row) for row in spamreader]
    fake_projects_names_iterator = iter(fake_projects_names)

    # 2- functions/capacities
    #
    fake_capacities_file = os.path.join(current_app.config['PDC_RESOURCES_DIR'], 'fake-db-names',
                                        'fake-capacities.txt')
    with open(fake_capacities_file, newline='') as csvfile:
        spamreader = csv.reader(csvfile, delimiter=';', quotechar='|')
        fake_capacities_names = [row for [row] in spamreader]
    fake_capacities_names_iterator = iter(fake_capacities_names)

    # 3- services
    #
    fake_services_file = os.path.join(current_app.config['PDC_RESOURCES_DIR'], 'fake-db-names',
                                      'fake-services.txt')
    with open(fake_services_file, newline='') as csvfile:
        spamreader = csv.reader(csvfile, delimiter=';', quotechar='|')
        fake_services_names = [row for row in spamreader]
    fake_services_names_iterator = iter(fake_services_names)

    # Skip columns names
    #
    next(fake_projects_names_iterator)
    next(fake_capacities_names_iterator)
    next(fake_services_names_iterator)

    for s in Service.query.all():
        next_service = next(fake_services_names_iterator)
        s.name = next_service[0]
        s.abbr = next_service[1]

    for p in Project.query.all():
        p.name = next(fake_projects_names_iterator)

    for c in Capacity.query.all():
        c.name = next(fake_capacities_names_iterator)

    db.session.commit()

14f36f55   hitier   Move lesia init c...
267

a17327bf   hitier   New db feeding co...
268
@bp.cli.command("feed_periods")
fa86be39   hitier   Enhance error cat...
269
270
271
272
273
@click.option('--begin-year', '-b', 'begin_year', default=2005, help="the start year to begin periods with")
@click.option('--end-year', '-e', 'end_year', default=2025, help="the last year to end periods with")
def feed_periods(begin_year, end_year):
    """ Fill in the periods names in the database. """
    for y in range(begin_year, end_year):
a17327bf   hitier   New db feeding co...
274
275
276
277
        for s in ['S1', 'S2']:
            period_name = "{}_{}".format(y, s)
            p = Period(name=period_name)
            db.session.add(p)
fa86be39   hitier   Enhance error cat...
278
279
280
281
    try:
        db.session.commit()
    except IntegrityError:
        current_app.logger.error("Periods already exist")
a17327bf   hitier   New db feeding co...
282
283


d8a6b942   hitier   More install doc
284
285
286
287
@bp.cli.command("feed_random_charges")
@click.option('--agent', '-a', 'agent', default=None, help="the agent id you want to charge")
def feed_random_charges(agent):
    """ Randomly fill in the agents charges. """
a17327bf   hitier   New db feeding co...
288
    for i in range(0, 100):
335db4c0   hitier   Allow randoming c...
289
290
291
292
        if agent is None:
            agent_id = random.choice([i for (i,) in db.session.query(Agent.id).all()])
        else:
            agent_id = int(agent)
a17327bf   hitier   New db feeding co...
293
294
295
296
297
        project_id = random.choice([i for (i,) in db.session.query(Project.id).all()])
        service_id = random.choice([i for (i,) in db.session.query(Service.id).all()])
        capacity_id = random.choice([i for (i,) in db.session.query(Capacity.id).all()])
        period_id = random.choice([i for (i,) in db.session.query(Period.id).all()])
        percent = random.choice(range(10, 110, 10))
e817ff5e   hitier   Random Charge Age...
298
299
300
301
302
        # check max agent charge for the period
        total_charge = db.session.query(func.sum(Charge.charge_rate).label("total_charge")) \
            .filter(Charge.agent_id == agent_id,
                    Charge.period_id == period_id
                    ).scalar()
a5a365e8   hitier   Move User model t...
303
        if total_charge is not None and (total_charge + percent) >= 100:
e817ff5e   hitier   Random Charge Age...
304
305
            print("Skipping agent {} for period {}".format(agent_id, period_id))
            continue
335db4c0   hitier   Allow randoming c...
306
        charge = Charge(agent_id=agent_id,
a17327bf   hitier   New db feeding co...
307
308
309
310
311
312
313
314
315
316
                        project_id=project_id,
                        service_id=service_id,
                        capacity_id=capacity_id,
                        period_id=period_id,
                        charge_rate=percent)
        print("adding {}_{}_{}_{}_{}_{}".format(agent_id, project_id, service_id, capacity_id, period_id, percent))
        db.session.add(charge)
        db.session.commit()


d8a6b942   hitier   More install doc
317
@bp.cli.command('user_add')
a17327bf   hitier   New db feeding co...
318
319
@click.argument('email')
@click.argument('name')
3ed62121   hitier   Add login to user
320
@click.argument('login')
a17327bf   hitier   New db feeding co...
321
@click.argument('password')
149f7875   hitier   Now user_add acce...
322
323
@click.argument('role')
def user_add(email, name, login, password, role):
d8a6b942   hitier   More install doc
324
    """ Add a new user in db."""
149f7875   hitier   Now user_add acce...
325
326
    user = User.query.filter(User.name == name).one_or_none()
    if (user):
82642adb   hitier   Update routes and...
327
328
        current_app.logger.error(f"user already exists {name}")
        return
149f7875   hitier   Now user_add acce...
329
    user = User(email=email, name=name, login=login, password=password, role=role)
a17327bf   hitier   New db feeding co...
330
331
    db.session.add(user)
    db.session.commit()
82642adb   hitier   Update routes and...
332
    current_app.logger.info(f"added {name}")
a17327bf   hitier   New db feeding co...
333
334


946dda30   hitier   New cli command u...
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
@bp.cli.command('user_update')
@click.option('--name', '-n', 'name', default=None, help="the name to set for that user")
@click.option('--role', '-r', 'role', default=None, help="the role to set for that user")
@click.option('--email', '-e', 'email', default=None, help="the email to set for that user")
@click.option('--password', '-p', 'password', default=None, help="the password to set for that user")
@click.argument('user_id')
def user_update(user_id, name, role, email, password):
    """Update the user by given id and given parameters."""
    user = User.query.get(user_id)
    if not user:
        current_app.logger.error(f"such user_id doesnt exists {user_id}")
        return
    if name:
        user.name = name
        print(f"User --{user.name}-- name updated to {user.name}")
    if role:
        user.set_role(role)
        print(f"User --{user.name}-- role updated to {_roleToName[user.role]}")
    if email:
        user.email=email
        print(f"User --{user.name}-- email updated to {user.email}")
    if password:
        print(f"User --{user.name}-- password updated")
        user.set_password(password)
    if not ( name or role or email or password):
        print(f"No update for user --{user.name}--")
    db.session.commit()


@bp.cli.command('user_delete')
@click.argument('user_id')
def user_delete(user_id):
    """Delete the user by given id (see user_show_all")."""
    user = User.query.get(user_id)
    db.session.delete(user)
    db.session.commit()


149f7875   hitier   Now user_add acce...
373
374
375
376
377
378
@bp.cli.command('show_roles')
def show_roles():
    """ List all available roles for a user"""
    print("\n".join(list(_nameToRole)))


d8a6b942   hitier   More install doc
379
380
381
@bp.cli.command('user_show_all')
def user_show_all():
    """ Show all users in db."""
4113c4f3   hitier   Fix user_show_all
382
383
    print("{:<5} {:<15} {:<15} {:<15}".format('id', 'name', 'login', 'email'))
    print("{:<5} {:<15} {:<15} {:<15}".format('-' * 5, '-' * 15, '-' * 15, '-' * 15))
a17327bf   hitier   New db feeding co...
384
385
    for user in User.query.all():
        print(user.login)
4113c4f3   hitier   Fix user_show_all
386
        print("{:<5} {:<15} {:<15} {:<15}".format(
a17327bf   hitier   New db feeding co...
387
388
389
            user.id,
            user.name,
            user.login,
a17327bf   hitier   New db feeding co...
390
391
            user.email
        ))