feat(contents): add full presences and new tribunes

README.md

@@ -18,9 +18,15 @@ Ensure you installed `inkscape` and `chromium`.
 
 Then, run the data ingestion:
 ```sh
+## Generate tribunes from raw sources
 npm run tribunes
+## Or only the one you want
+npm run tribunes -- 2023-07-douai-notre-ville.md
+## Generate profiles banners
 npm run profiles
+## Compute writting stats
 npm run stats
+## Compute presence stats
 npm run presences
 ```
 

bin/presences.ts

@@ -8,12 +8,12 @@ import {
   shrinkStats,
   sortByDate,
 } from "../src/utils/stats";
-import type { Author } from "../src/utils/tribunes";
+import type { BaseAuthor } from "../src/utils/tribunes";
 import type {
   PresenceItem,
   PresenceStatItem,
   PresenceStatsSummary,
-  StatsSummary,
+  GlobalStatsSummary,
 } from "../src/utils/writters";
 
 run();
@@ -33,8 +33,8 @@ async function run() {
       .filter((id) => id)
       .map((line) => line.split(",").map((cell) => cell.trim()));
     const persons: Record<
-      Author["id"],
-      Pick<Author, "id" | "name"> & {
+      BaseAuthor["id"],
+      Pick<BaseAuthor, "id" | "name"> & {
         presences: (PartialPresence & { date: string })[];
       }
     > = {};
@@ -69,7 +69,7 @@ async function run() {
     const instanceCode = file.replace(/\.csv$/, "");
     const globalStats = JSON.parse(
       (await readFile(pathJoin("contents", `globalStats.json`))).toString()
-    ) as StatsSummary;
+    ) as GlobalStatsSummary;
     const globalOccurences: Record<
       keyof PresenceStatsSummary,
       OccurenceItem[]

bin/stats.ts

@@ -15,20 +15,24 @@ import { toASCIIString } from "../src/utils/ascii";
 import type {
   SentimentOccurenceItem,
   StatsSummary,
+  WritterStats,
+  GlobalStatsSummary,
 } from "../src/utils/writters";
-import type {
-  BaseGroup,
-  TribuneFrontmatterMetadata,
-} from "../src/utils/tribunes";
+import type { TribuneFrontmatterMetadata } from "../src/utils/tribunes";
+import type { GroupStats } from "../src/utils/groups";
+
+type TemporaryStats<T> = Omit<T, "summary" | "words"> & {
+  words: Record<string, number>;
+};
 
 run();
 
 async function run() {
-  const globalStats = await createBaseStatsObject();
+  const globalStats: GlobalStatsSummary = await createBaseStatsObject();
   const nl = await nodeLefff.load();
   const files = await readDir(pathJoin("contents", "tribunes"));
-  const writtersAggregations = {};
-  const groupsAggregations = {};
+  const writtersAggregations: Record<string, TemporaryStats<WritterStats>> = {};
+  const groupsAggregations: Record<string, TemporaryStats<GroupStats>> = {};
 
   for (const file of files) {
     console.warn(`➕ - Processing ${file}.`);
@@ -67,25 +71,28 @@ async function run() {
     );
 
     const { id, date, group, authors } = metadata;
-    const groupAggregation = groupsAggregations[group.id] || {
+    const groupAggregation: TemporaryStats<GroupStats> = groupsAggregations[
+      group.id
+    ] || {
       id: group.id,
       name: group.name,
       party: group.party,
-      partyAbbr: group.abbr,
+      partyAbbr: group.partyAbbr,
       type: group.type,
       logo: group.logo,
-      totalWords: 0,
+      words: {},
       writtings: [],
       sentences: [],
       sentiments: [],
       exclamations: [],
       questions: [],
       bolds: [],
       caps: [],
-      words: [],
       authors,
       locality: "Douai",
       country: "France",
+      totalWords: 0,
+      totalSignificantWords: 0,
     };
 
     groupsAggregations[group.id] = groupAggregation;
@@ -102,23 +109,26 @@ async function run() {
     const aggregationsList = [
       groupAggregation,
       ...authors.map(({ name, mandates, portrait }) => {
-        const writerAggregation = writtersAggregations[toASCIIString(name)] || {
-          totalWords: 0,
-          writtings: [],
-          sentences: [],
-          sentiments: [],
-          exclamations: [],
-          questions: [],
-          bolds: [],
-          caps: [],
-          words: [],
-          portrait,
-          name,
-          mandates,
-          groups: [group],
-          locality: "Douai",
-          country: "France",
-        };
+        const writerAggregation: TemporaryStats<WritterStats> =
+          writtersAggregations[toASCIIString(name)] || {
+            id: toASCIIString(name),
+            writtings: [],
+            sentences: [],
+            sentiments: [],
+            exclamations: [],
+            questions: [],
+            bolds: [],
+            caps: [],
+            words: {},
+            portrait,
+            name,
+            mandates,
+            groups: [group],
+            locality: "Douai",
+            country: "France",
+            totalWords: 0,
+            totalSignificantWords: 0,
+          };
 
         writtersAggregations[toASCIIString(name)] = writerAggregation;
         writerAggregation.groups = [
@@ -241,10 +251,20 @@ async function run() {
       (word) => word.length > 3
     );
 
+    writtersAggregations[key].totalSignificantWords = allWords.length;
+
+    globalStats.authors[key] = globalStats.authors[key] || {
+      id: key,
+      name: writtersAggregations[key].name,
+      mandates: writtersAggregations[key].mandates,
+      portrait: writtersAggregations[key].portrait,
+      totalWords: 0,
+      totalSignificantWords: 0,
+    };
     globalStats.authors[key].totalWords = writtersAggregations[key].totalWords;
     globalStats.authors[key].totalSignificantWords = allWords.length;
 
-    const finalAggregation = {
+    const finalAggregation: Omit<WritterStats, "authors"> = {
       ...writtersAggregations[key],
       summary: shrinkSummary(summary),
       words: allWords
@@ -279,6 +299,26 @@ async function run() {
       caps: computeStats(groupsAggregations[key].caps),
       sentiments: computeSentimentStats(groupsAggregations[key].sentiments),
     };
+    const allWords = Object.keys(groupsAggregations[key].words).filter(
+      (word) => word.length > 3
+    );
+
+    groupsAggregations[key].totalSignificantWords = allWords.length;
+
+
+    globalStats.groups[key] = globalStats.groups[key] || {
+      id: key,
+      name: groupsAggregations[key].name,
+      type: groupsAggregations[key].type,
+      party: groupsAggregations[key].party,
+      partyAbbr: groupsAggregations[key].partyAbbr,
+      logo: groupsAggregations[key].logo,
+      totalWords: 0,
+      totalSignificantWords: 0,
+    };
+    globalStats.groups[key].totalWords = groupsAggregations[key].totalWords;
+    globalStats.groups[key].totalSignificantWords = allWords.length;
+
 
     aggregatesStats(summary.sentences, globalStats.sentences);
     aggregatesStats(summary.exclamations, globalStats.exclamations);
@@ -325,7 +365,7 @@ async function run() {
   );
 }
 
-async function createBaseStatsObject(): Promise<StatsSummary> {
+async function createBaseStatsObject(): Promise<GlobalStatsSummary> {
   let globalStats;
 
   try {
@@ -338,6 +378,7 @@ async function createBaseStatsObject(): Promise<StatsSummary> {
 
   return {
     authors: {},
+    groups: {},
     ...globalStats,
     sentences: createBaseStatsItem(),
     exclamations: createBaseStatsItem(),
@@ -369,9 +410,7 @@ function computeSentimentStats(
   }, {} as Partial<StatsSummary["sentiments"]>) as StatsSummary["sentiments"];
 }
 
-function shrinkSummary(
-  summary: Omit<StatsSummary, "authors">
-): Omit<StatsSummary, "authors"> {
+function shrinkSummary(summary: StatsSummary): StatsSummary {
   return {
     ...summary,
     sentences: shrinkStats(summary.sentences),

bin/tribunes.ts

@@ -1,17 +1,20 @@
 import { readFile, writeFile, readDir, access } from "../src/utils/files";
-import { join as pathJoin } from "path";
+import { join as pathJoin } from "node:path";
 import { tmpdir } from "node:os";
+import { argv } from "node:process";
 import { exec as _exec } from "node:child_process";
 import { promisify } from "node:util";
 import { toASCIIString } from "../src/utils/ascii";
-import type { Author, BaseGroup } from "../src/utils/tribunes";
+import type { BaseAuthor, BaseGroup } from "../src/utils/tribunes";
 
 const exec = promisify(_exec);
 
 run();
 
 async function run() {
-  const files = await readDir(pathJoin("sources", "tribunes"));
+  const files = (await readDir(pathJoin("sources", "tribunes"))).filter(
+    (file) => (argv.slice(2).length ? argv.includes(file) : true)
+  );
 
   for (const file of files) {
     const content = await readFile(pathJoin("sources", "tribunes", file));
@@ -68,7 +71,7 @@ async function run() {
         console.error(`🤔 - Author parts for ${file} looks strange!`);
       }
 
-      const authors: Author[] = [];
+      const authors: BaseAuthor[] = [];
 
       do {
         const name = authorParts.shift() as string;
@@ -82,7 +85,7 @@ async function run() {
           portrait = "default.svg";
         }
 
-        const author: Author = {
+        const author: BaseAuthor = {
           id,
           name,
           mandates,
@@ -131,7 +134,7 @@ group:
   name: "${group.name}"
   type: "${group.type}"
   party: "${group.party}"
-  abbr: "${group.abbr}"
+  partyAbbr: "${group.partyAbbr}"
   logo: "${group.logo}"
 date: "${date}"
 publication: "${publication}"
@@ -201,7 +204,7 @@ function buildGroupDetails(fullName): BaseGroup {
     name,
     type,
     party,
-    abbr,
+    partyAbbr: abbr,
     logo,
   };
 }