Update category/label data model

hitier
2 parents e20cd5d4 74c7dad5
Showing 11 changed files with 270 additions and 112 deletions Show diff stats
app/commands/commands.py
app/db_mgr.py
app/main/routes.py
app/main/templates/project.html
app/main/templates/projects.html
app/models.py
resources/lesia-btp.sqlite
tests/backend_tests.py
tests/common_db_feed.py
tests/db_tests.py
tests/frontend_tests.py
@@ -10,7 +10,7 @@ from sqlalchemy.exc import IntegrityError
 from sqlalchemy.sql import func
  
 from app.models import db, Agent, Service, Project, Capacity, Period, Charge, AgentStatus, Company, AgentBap, \
-    AgentGrade, Category, Label, ProjectLabel
+    AgentGrade, Category, Label, ProjectLabel, CategoryLabel
 # TODO: rename to methods and add get_roles()
 from app.auth.models import User, _nameToRole, _roleToName
  
@@ -79,15 +79,22 @@ def feed_from_irap(csv_file_name):
     grades = []
     companies = []
     statuses = []
+    labels = []
  
-    # build a category dict of lists
+    # Build a category dict of lists
     #   key being the category name,
     #   list being filled with corresponding labels
-    categories = {k: [] for k in categorie_keys}
+    categorie_labels = {k: [] for k in categorie_keys}
  
+    # Projects' labels is a dict of lists
+    #   indexed by project name
+    #   containing labels for that project
     #
-    categorized_projects = {}
+    project_labels = {}
  
+    #
+    # Parse the rows and fill in various lists
+    #
     for r in rows:
         services.append(r[service_key])
         baps.append(r[bap_key])
@@ -95,16 +102,20 @@ def feed_from_irap(csv_file_name):
         companies.append(r[company_key])
         statuses.append(r[status_key])
  
-        # categorized_projects
-        project = r[project_key]
-        categorized_projects[project] = []
+        # the projet and its labels
+        project_name = r[project_key]
+        project_labels[project_name] = []
  
-        # fill in the category-labels dict
+        # now fill in both
+        #   the labels list,
+        #   the category-labels dict,
+        #   and the project-labels dict
         for k in categorie_keys:
-            categories[k].append(r[k])
-            categorized_projects[project].append(r[k])
+            labels.append(r[k])
+            categorie_labels[k].append(r[k])
+            project_labels[project_name].append(r[k])
  
-        # build agents list of dicts
+        # create the agents list of dicts
         agents.append({
             'firstname': r[firstname_key],
             'secondname': r[secondname_key],
@@ -124,25 +135,25 @@ def feed_from_irap(csv_file_name):
     #
     #  1- first remove empty string with filter()
     #  2- then keep only uniq item with set()
-    #  3- at last alpha sort  with sorted()
+    #  3- at last alpha sort with sorted()
     #
     services = sorted(set(filter(None, services)))
     baps = sorted(set(filter(None, baps)))
     grades = sorted(set(filter(None, grades)))
     companies = sorted(set(filter(None, companies)))
     statuses = sorted(set(filter(None, statuses)))
+    labels = sorted(set(filter(None, labels)))
  
-    # Do the same for the projects, that are keys of categorized_projects
+    # Do the same for the projects, that are keys of project_labels
     #
-    projects = sorted(set(categorized_projects.keys()))
+    projects = sorted(set(project_labels.keys()))
  
-    # Do the same for the category labels stored as a dict
+    # Do the same for the labels inside each category
+    #  c is the category name containing the labels list
     #
-    #   k is category name
-    #   v is labels list for that category
-    for k in categorie_keys:
-        labels = sorted(set(filter(None, categories[k])))
-        categories[k] = labels
+    for c in categorie_keys:
+        c_labels = sorted(set(filter(None, categorie_labels[c])))
+        categorie_labels[c] = c_labels
  
     # At least, as agents is a list of dicts, sorting is a bit tricky
     #
@@ -153,6 +164,12 @@ def feed_from_irap(csv_file_name):
     agents = list({(a['firstname'], a['secondname']): a for a in agents}.values())
     agents = sorted(agents, key=lambda a: (a['firstname'], a['secondname']))
  
+    #
+    # We are done with collecting data
+    #
+    # Now we write to database
+    #
+
     # Feed baps from column
     #
     for b in baps:
@@ -188,6 +205,20 @@ def feed_from_irap(csv_file_name):
         db.session.add(n_p)
     db.session.commit()
  
+    # Feed labels from column
+    #
+    for _l in labels:
+        n_l = Label(name=_l)
+        db.session.add(n_l)
+    db.session.commit()
+
+    # Feed categories from initial list
+    #
+    for _c in categorie_keys:
+        n_c = Category(name=_c)
+        db.session.add(n_c)
+    db.session.commit()
+
     # Feed services from column
     #
     for s in services:
@@ -212,25 +243,25 @@ def feed_from_irap(csv_file_name):
  
     # Feed categories and labels
     #
-    for c, l in categories.items():
-        current_app.logger.debug("Adding category " + c)
-        n_c = Category(name=c.capitalize())
-        db.session.add(n_c)
-        # db.session.commit()
-        for label in l:
-            current_app.logger.debug("Adding label {} for id {}".format(label, n_c.id))
-            n_l = Label(name=label, category=n_c)
-            db.session.add(n_l)
+    for category, labels in categorie_labels.items():
+        print(category)
+        n_c = Category.query.filter_by(name=category).one()
+        for label in labels:
+            print(label)
+            n_l = Label.query.filter(Label.name == label).one()
+            current_app.logger.debug(f"Adding label {label} to category {category}")
+            n_cl = CategoryLabel(label=n_l, category=n_c)
+            db.session.add(n_cl)
     db.session.commit()
  
     # Feed project's labels
     #
-    for p, labels in categorized_projects.items():
-        n_p = Project.query.filter(Project.name == p).one()
-        for l in labels:
-            n_l = Label.query.filter(Label.name == l).one()
-            n_c = n_l.category
-            n_pl = ProjectLabel(project=n_p, category=n_c, label=n_l)
+    for project, labels in project_labels.items():
+        print(f"Project {project}")
+        n_p = Project.query.filter(Project.name == project).one()
+        for label in labels:
+            n_l = Label.query.filter(Label.name == label).one()
+            n_pl = ProjectLabel(project=n_p, label=n_l)
             db.session.add(n_pl)
     db.session.commit()
  
@@ -313,8 +344,9 @@ def feed_from_lesia():
     domain_category = Category(name="Domaine")
     domains = lesia_session.query(lesia_domains)
     for d in domains:
-        n_l = Label(name=d.nom, category=domain_category)
-        db.session.add(n_l)
+        n_l = Label(name=d.nom)
+        n_cl = CategoryLabel(category=domain_category, label=n_l)
+        db.session.add(n_cl)
     db.session.commit()
  
     # Feed all lesia 'pôle' names and link to new category  "Pôle"
@@ -322,8 +354,9 @@ def feed_from_lesia():
     pole_category = Category(name="Pôle")
     poles = lesia_session.query(lesia_poles)
     for p in poles:
-        n_l = Label(name=p.nom, category=pole_category)
-        db.session.add(n_l)
+        n_l = Label(name=p.nom)
+        n_cl = CategoryLabel(category=pole_category, label=n_l)
+        db.session.add(n_cl)
     db.session.commit()
  
     # Feed lesia project with proper "pôle"
@@ -338,7 +371,7 @@ def feed_from_lesia():
         pole_name = lesia_session.query(lesia_poles).filter(lesia_poles.id == p.pole_id).one().nom
         # search corresponding Label and store in ProjectLabel table
         n_l = Label.query.filter(Label.name == pole_name).one()
-        n_pl = ProjectLabel(project=n_p, category=n_l.category, label=n_l)
+        n_pl = ProjectLabel(project=n_p, label=n_l)
         db.session.add(n_pl)
         db.session.commit()
  
@@ -350,7 +383,7 @@ def feed_from_lesia():
         domain_name = lesia_session.query(lesia_domains).filter(lesia_domains.id == dp.domaine_id).one().nom
         n_p = Project.query.filter(Project.name == project_name).one()
         n_l = Label.query.filter(Label.name == domain_name).one()
-        n_pl = ProjectLabel(project=n_p, category=n_l.category, label=n_l)
+        n_pl = ProjectLabel(project=n_p, label=n_l)
         db.session.add(n_pl)
         # Some projects have 2 domain labels in lesia db
         # That is not allowed any more in the new model.
@@ -10,7 +10,13 @@ charge_unit = 100
  
 def projects():
     """
-    Build the list of all agents, with their charges for the current period
+    Build the list of all projects, with their charges for the current period
+
+    Returns a table with headers in the first line:
+    Id, Project name, Category_1, ... , Category_n, Total_Charge_for_the_period
+
+    The category cells embed the list of the labels of the project for that category.
+
     :return:
     """
     current_period_id = get_current_period()
@@ -26,21 +32,33 @@ def projects():
     projects_charges = db.session.execute(sql_txt).fetchall()
     # First row is table titles
     all_projects = [["Id", "Projet"]]
-    # Add all categories as last table titles
-    for c in Category.query.all():
+    # Add all categories as next table titles
+    # headers will then look like [["Id", "Projet", "Domaine", "Pôle"]]
+    categories = Category.query.all()
+    for c in categories:
         all_projects[0].append(c.name)
+    # then add charge title
+    # headers become [["Id", "Projet", "Domaine", "Pôle", "Charge"]]
     all_projects[0].append("Charge")
     # Build the table row by row
-    for pc in projects_charges:
-        p_id = pc[0]
-        p = Project.query.get(int(p_id))
-        p_labels = p.labels
-        # adding 3 first columns: id, name and total charge
-        project_row = [p.id, p.name]
-        # then labels, one for each category
-        for pl in p_labels:
-            project_row.append(pl.label.name)
-        project_row.append(pc[1])
+    for _pc in projects_charges:
+        p_id = _pc[0]
+        project = Project.query.get(int(p_id))
+        # adding 2 first columns: id, name
+        project_row = [project.id, project.name]
+        # then labels, many for each category
+        for category in categories:
+            # we build the labels.name list of the intersection of current category's labels with project's labels
+            # Comprehensive shortcut is:
+            # labels = [_cl.label.name for _cl in category.labels if _cl.label in [_pl.label for _pl in project.labels]]
+            #
+            category_labels = [_cl.label for _cl in category.labels]
+            project_labels = [_pl.label for _pl in project.labels]
+            intersection_labels = list(set.intersection(set(project_labels), set(category_labels)))
+            labels = [label.name for label in intersection_labels]
+            project_row.append(labels)
+        # then add total charge
+        project_row.append(_pc[1])
         all_projects.append(project_row)
     return all_projects
  
@@ -91,7 +91,7 @@ def project(project_id):
     this_project = Project.query.get(int(project_id))
     charges_table = db_mgr.charges_by_project(project_id)
     return render_template('project.html',
-                           project=this_project,
+                           project=this_project.to_struct(),
                            charges=charges_table,
                            subtitle="{}".format(this_project.name))
  
@@ -15,10 +15,10 @@
   <div class="card-body">
     <dl class="row">
       <dt class="col-sm-2 text-right">ID :</dt>
-      <dd class="col-sm-10 text-left">{{project.id}}</dd>
-      {% for pl in project.labels %}
-        <dt class="col-sm-2 text-right">{{pl.label.category.name}} :</dt>
-        <dd class="col-sm-10 text-left">{{pl.label.name}}</dd>
+      <dd class="col-sm-10 text-left">{{project['id']}}</dd>
+      {% for category, labels in project['category_labels'].items() %}
+        <dt class="col-sm-2 text-right">{{category}} :</dt>
+        <dd class="col-sm-10 text-left">{{labels|join(",")}}</dd>
       {% endfor %}
       <dt class="col-sm-2 text-right">Etat :</dt>
       <dd class="col-sm-10 text-left"></dd>
@@ -18,8 +18,9 @@
     <td><a href="{{url_for('main.project', project_id=project[0])}}">
       {{ project[1] }}</a>
     </td>
+    {#the category cells#}
     {% for c in project[2:-1] %}
-    <td>{{c}}</td>
+    <td>{{c|join(',')}}</td>
     {% endfor %}
     <td>{{ project[-1] /100}}</td>
   </tr>
@@ -41,36 +41,82 @@ class Formable:
 #
 # Categorized projects
 #
+# There is one label list,
+# each label belongs to one or more categories.
+#
+# The projects are labelled by one or more label.
+#
+# Thus this is modeled with classes
+#    Project, Label and Category
+# And many_to_many association are done through
+#    ProjectLabel and CategoryLabel
+#
  
-class Project(db.Model):
+class Project(db.Model, Formable):
     id = db.Column(db.Integer, primary_key=True)
     name = db.Column(db.String)
     labels = relationship("ProjectLabel", back_populates="project")
  
+    export_keys = ['name']
  
-class Category(db.Model):
+    def to_struct(self):
+        """
+        overide parent method to include one key: agent.fullname
+
+        Mainly we add the 'labels' element containing a dict
+        where category key contains labels for that project.
+        :return:
+        """
+        _struct = super(Project, self).to_struct()
+        _struct['category_labels'] = {}
+        for _c in Category.query.all():
+            category_labels = []
+            for _l in self.labels:
+                if _l.label in [_cl.label for _cl in _c.labels]:
+                    category_labels.append(_l.label.name)
+            _struct['category_labels'][_c.name] = category_labels
+        return _struct
+
+
+class ProjectLabel(db.Model):
+    """
+    Labelling projects.
+    On project can have many labels.
+    And one label will be set to many projects
+    """
     id = db.Column(db.Integer, primary_key=True)
-    name = db.Column(db.String)
-    labels = relationship("Label", back_populates="category")
-    projects = relationship("ProjectLabel", back_populates="category")
+    project_id = db.Column(db.Integer, db.ForeignKey('project.id'))
+    label_id = db.Column(db.Integer, db.ForeignKey('label.id'))
+    project = relationship("Project", back_populates="labels")
+    label = relationship("Label", back_populates="projects")
  
  
 class Label(db.Model):
     id = db.Column(db.Integer, primary_key=True)
     name = db.Column(db.String, unique=True)
     category_id = db.Column(db.Integer, db.ForeignKey('category.id'))
-    category = relationship("Category", back_populates="labels")
     projects = relationship("ProjectLabel", back_populates="label")
+    categories = relationship("CategoryLabel", back_populates="label")
  
  
-class ProjectLabel(db.Model):
-    project_id = db.Column(db.Integer, db.ForeignKey('project.id'), primary_key=True)
-    category_id = db.Column(db.Integer, db.ForeignKey('category.id'), primary_key=True)
+class CategoryLabel(db.Model):
+    """
+    Categorizing labels:
+    one label can be added to many categories
+    one category hosts many labels
+    """
+    id = db.Column(db.Integer, primary_key=True)
+    category_id = db.Column(db.Integer, db.ForeignKey('category.id'))
     label_id = db.Column(db.Integer, db.ForeignKey('label.id'))
+    category = relationship("Category", back_populates="labels")
+    label = relationship("Label", back_populates="categories")
  
-    project = relationship("Project", back_populates="labels")
-    category = relationship("Category", back_populates="projects")
-    label = relationship("Label", back_populates="projects")
+
+class Category(db.Model):
+    id = db.Column(db.Integer, primary_key=True)
+    name = db.Column(db.String)
+    labels = relationship("Label", back_populates="category")
+    labels = relationship("CategoryLabel", back_populates="category")
  
  
 #
 import os
 import unittest
+from pprint import pprint
  
 from pdc_config import TestConfig
 from app import create_app, db_mgr
@@ -40,6 +41,22 @@ class DbMgrTestCase(BaseTestCase):
         all_projects = db_mgr.projects()
         self.assertEqual(5, len(all_projects[0]))
  
+    def test_projects_all_columns(self):
+        all_projects = db_mgr.projects()
+        for p in all_projects:
+            with self.subTest(p):
+                self.assertEqual(5, len(p), f"Failed on {p[1]}")
+
+    def test_projects_labels(self):
+        self.assertEqual(0, 0)
+        all_projects = db_mgr.projects()
+        # test all projects but skip headers first row
+        for p in all_projects[1:]:
+            with self.subTest(p):
+                # the number of labels for each category and any project should be less than 2
+                self.assertTrue(len(p[2]) <= 2, f"Failed on {p[1]}: " + ", ".join(p[2]))
+                self.assertTrue(len(p[3]) <= 2, f"Failed on {p[1]}: " + ", ".join(p[3]))
+
     def test_agents(self):
         all_agents = db_mgr.agents()
         self.assertEqual(548, len(all_agents))
@@ -14,41 +14,42 @@ def resources_to_instancedb(app):
     copyfile(sqlite_file_name, db_path)
     return db_path
  
-
-categorized_labels = {'pole': ['Spatial', 'Sol'], 'domaine': ['soleil-terre', 'atmosphere', 'r&t', 'géologie']}
-projects_categories = {'ChemCam': {'pole': 'Spatial', 'domaine': 'géologie'},
-                       'Pilot': {'pole': 'Spatial', 'domaine': 'atmosphere'},
-                       'Ambre': {'pole': 'Spatial', 'domaine': 'soleil-terre'}}
-
-
-def feed_projects():
-    for c_name, labels in categorized_labels.items():
-        n_c = Category(name=c_name)
-        db.session.add(n_c)
-        for l_name in labels:
-            n_l = Label(name=l_name, category=n_c)
-            db.session.add(n_l)
-
-    db.session.commit()
-
-    project_names = projects_categories.keys()
-
-    for p_name in set(project_names):
-        n_p = Project(name=p_name)
-        db.session.add(n_p)
-    db.session.commit()
-
-    for p, categories in projects_categories.items():
-        n_p = db.session.query(Project).filter(Project.name == p).one()
-        for c_name, l_name in categories.items():
-            n_c = db.session.query(Category).filter(Category.name == c_name).one()
-            n_l = db.session.query(Label).filter(Label.name == l_name).one()
-            n_pc = ProjectLabel(project=n_p, category=n_c, label=n_l)
-            db.session.add(n_pc)
-
-    db.session.commit()
-
-
-def show_categories():
-    for c in db.session.query(Category).all():
-        print(c.name + ": ", ",".join([l.name for l in c.categorized_labels]))
+# TODO: rewrite for new category/label model
+#
+# categorized_labels = {'pole': ['Spatial', 'Sol'], 'domaine': ['soleil-terre', 'atmosphere', 'r&t', 'géologie']}
+# projects_categories = {'ChemCam': {'pole': 'Spatial', 'domaine': 'géologie'},
+#                        'Pilot': {'pole': 'Spatial', 'domaine': 'atmosphere'},
+#                        'Ambre': {'pole': 'Spatial', 'domaine': 'soleil-terre'}}
+#
+#
+# def feed_projects():
+#     for c_name, labels in categorized_labels.items():
+#         n_c = Category(name=c_name)
+#         db.session.add(n_c)
+#         for l_name in labels:
+#             n_l = Label(name=l_name, category=n_c)
+#             db.session.add(n_l)
+#
+#     db.session.commit()
+#
+#     project_names = projects_categories.keys()
+#
+#     for p_name in set(project_names):
+#         n_p = Project(name=p_name)
+#         db.session.add(n_p)
+#     db.session.commit()
+#
+#     for p, categories in projects_categories.items():
+#         n_p = db.session.query(Project).filter(Project.name == p).one()
+#         for c_name, l_name in categories.items():
+#             n_c = db.session.query(Category).filter(Category.name == c_name).one()
+#             n_l = db.session.query(Label).filter(Label.name == l_name).one()
+#             n_pc = ProjectLabel(project=n_p, category=n_c, label=n_l)
+#             db.session.add(n_pc)
+#
+#     db.session.commit()
+#
+#
+# def show_categories():
+#     for c in db.session.query(Category).all():
+#         print(c.name + ": ", ",".join([l.name for l in c.categorized_labels]))
 import os
 import unittest
+from pprint import pprint
  
 from app import create_app, db, User
 from app.models import Project
 from pdc_config import TestConfig
-from tests.common_db_feed import feed_projects, resources_to_instancedb
+from tests.common_db_feed import resources_to_instancedb
  
  
 class DbBaseTestCase(unittest.TestCase):
@@ -36,7 +37,8 @@ class DbBaseTestCase(unittest.TestCase):
         #
         if self.mode == 'memory':
             db.create_all()
-            feed_projects()
+            # TODO: to be rewriten for new category/label model
+            # feed_projects()
  
     def tearDown(self):
         if self.mode == 'memory':
@@ -48,6 +50,21 @@ class DbBaseTestCase(unittest.TestCase):
         self.app_context.pop()
  
  
+class AnyModelTestCase(DbBaseTestCase):
+    def setUp(self):
+        DbBaseTestCase.setUp(self, 'btp')
+
+    def tearDown(self):
+        DbBaseTestCase.tearDown(self)
+
+    def test_projects_struct(self):
+        for project in Project.query.all():
+            with self.subTest(project):
+                categories = list(project.to_struct()['category_labels'].keys())
+                self.assertEqual(['Domaine', 'Pôle'], categories,
+                                 f"Failed on {project.name} categories: {categories}")
+
+
 class ChargeModelTestCase(DbBaseTestCase):
     def setUp(self):
         DbBaseTestCase.setUp(self, 'btp')
@@ -112,6 +112,20 @@ class AccessTestCase(BaseFrontTestCase):
         self.driver.get(target_url)
         self.assertEqual(self.driver.current_url, 'http://localhost:8943/projects')
  
+    def test_projects_trs(self):
+        target_url = self.get_server_url() + url_for('main.projects')
+        self.driver.get(target_url)
+        td_elmts = self.driver.find_elements_by_xpath("//table/tbody/tr")
+        self.assertEqual(101, len(td_elmts))
+
+    def test_projects_tds(self):
+        target_url = self.get_server_url() + url_for('main.projects')
+        self.driver.get(target_url)
+        for tr in self.driver.find_elements_by_xpath("//table/tbody/tr"):
+            with self.subTest(tr):
+                tds = tr.find_elements_by_tag_name('td')
+                self.assertEqual(4, len(tds), tds[0].text + " has wrong number of columns")
+
     def test_project_page(self):
         project_id = 8
         target_url = self.get_server_url() + url_for('main.project', project_id=project_id)
@@ -120,6 +134,17 @@ class AccessTestCase(BaseFrontTestCase):
         td_elmts = self.driver.find_elements_by_xpath("//table[@id='charge_table']/tbody/tr/td")
         self.assertEqual(1085, len(td_elmts))
  
+    def test_project_page_categories(self):
+        project_id = 84
+        target_url = self.get_server_url() + url_for('main.project', project_id=project_id)
+        self.driver.get(target_url)
+        self.assertEqual(self.driver.current_url, f'http://localhost:8943/project/{project_id}')
+        # pole_dt_db = self.driver.find_element_by_xpath("//dt[text()='Pôle']/following-sibling::dd")
+        pole_dt_db = self.driver.find_element_by_xpath("//dt[contains(text(),'Pôle')]/following-sibling::dd")
+        self.assertEqual('Solaire', pole_dt_db.text)
+        domaine_dt_db = self.driver.find_element_by_xpath("//dt[contains(text(),'Domaine')]/following-sibling::dd")
+        self.assertTrue('LESIA' in domaine_dt_db.text)
+
  
 class FormsTestCase(BaseFrontTestCase):