From 381f4dd02c0eca2f4982ab92eb943265f7340572 Mon Sep 17 00:00:00 2001 From: Nicolas Froidure Date: Fri, 25 Aug 2023 10:45:38 +0200 Subject: [PATCH] feat(scripts): split script into granular ones --- README.md | 7 +- bin/parse.ts | 563 ------------------ bin/{presence.ts => presences.ts} | 1 - bin/{profile.ts => profiles.ts} | 0 bin/stats.ts | 388 ++++++++++++ bin/tribunes.ts | 207 +++++++ contents/globalStats.json | 42 +- contents/groups/douai-au-coeur-eelv.json | 24 +- contents/groups/douai-au-coeur-pcf.json | 16 +- contents/groups/douai-au-coeur-ps.json | 60 +- contents/groups/douai-au-coeur-se.json | 64 +- .../douai-dynamique-et-durable-dvd.json | 32 +- ...i-plus-belle-plus-propre-plus-sure-rn.json | 12 +- .../groups/douaisiens-passionnement-ump.json | 32 +- .../groups/ensemble-faisons-douai-se.json | 4 +- contents/groups/jacques-vernier-ump.json | 4 +- .../rassemblement-douai-bleu-marine-rn.json | 8 +- contents/groups/vivons-douai-eelv.json | 4 +- contents/groups/vivons-douai-pcf.json | 8 +- contents/groups/vivons-douai-ps.json | 28 +- package-lock.json | 14 + package.json | 8 +- src/utils/tribunes.ts | 4 +- 23 files changed, 727 insertions(+), 803 deletions(-) delete mode 100644 bin/parse.ts rename bin/{presence.ts => presences.ts} (99%) rename bin/{profile.ts => profiles.ts} (100%) create mode 100644 bin/stats.ts create mode 100644 bin/tribunes.ts diff --git a/README.md b/README.md index c9a8db7b..5aa68d2e 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,10 @@ Ensure you installed `inkscape` and `chromium`. Then, run the data ingestion: ```sh -npm run parse +npm run tribunes +npm run profiles +npm run stats +npm run presences ``` And finally run the app: @@ -30,6 +33,6 @@ npm run dev Lemmatization is not working properly. May be useful to [see this](http://www.erwanlenagard.com/general/tutoriel-implementer-stanford-corenlp-avec-talend-1354). -## Windows users +## Windows & MAC users Lol. diff --git a/bin/parse.ts b/bin/parse.ts deleted file mode 100644 index 167b9472..00000000 --- a/bin/parse.ts +++ /dev/null @@ -1,563 +0,0 @@ -import { readFile, writeFile, readDir, access } from "../src/utils/files"; -import { join as pathJoin } from "path"; -import axios from "axios"; -import { tmpdir } from "node:os"; -import { exec as _exec } from "node:child_process"; -import { promisify } from "node:util"; -import nodeLefff from "node-lefff"; -import { toASCIIString } from "../src/utils/ascii"; -import type { - SentimentOccurenceItem, - StatsSummary, -} from "../src/utils/writters"; -import type { Author, BaseGroup } from "../src/utils/tribunes"; -import { - createBaseStatsItem, - shrinkStats, - sortByName, - aggregatesStats, - computeStats, -} from "../src/utils/stats"; - -const exec = promisify(_exec); - -run(); - -async function run() { - const nl = await nodeLefff.load(); - - const files = await readDir(pathJoin("sources", "tribunes")); - const writtersAggregations = {}; - const groupsAggregations = {}; - const globalStats = createBaseStatsObject(); - - for (const file of files) { - const content = await readFile(pathJoin("sources", "tribunes", file)); - - console.warn(`➕ - Processing ${file}.`); - - const publication = file.split("-").slice(2).join("-").replace(/\.md$/, ""); - const occurence = file.split("-").slice(0, 2).join("-"); - const parts = content.split(/[\-]{3,}/gm); - const sources = parts[0] - .split(/(\r?\n)+/gm) - .filter((line) => line.startsWith("https://")); - const tempDir = tmpdir(); - const sourcesCaptures = await sources.reduce( - async (sourcesCaptures, source) => { - const captures = await sourcesCaptures; - const filename = `${publication}-${occurence}-p${source.replace( - /^.*p([0-9]+).svgz/, - "$1" - )}.png`; - const captureDestination = pathJoin( - "public", - "images", - "sources", - filename - ); - const tempFile = pathJoin(tempDir, filename); - - // await new Promise((resolve) => setTimeout(resolve, 500)); - // await exec( - // `google-chrome --headless --screenshot=${tempFile} ${source}` - // ); - // await new Promise((resolve) => setTimeout(resolve, 500)); - // await exec(`convert ${tempFile} -trim ${captureDestination}`); - return [...captures, captureDestination]; - }, - Promise.resolve([] as string[]) - ); - - if (!sources.length) { - console.error(`🤔 - No sources for ${file} !`); - continue; - } - - const tribunes = parts.slice(1); - - for (const tribune of tribunes) { - const parts = tribune.trim().split(/\n[\n]+/gm); - const authorPart = (parts.pop() as string).trim(); - const mayorCase = !!authorPart.match(/^\s*votre maire/i); - const authorParts = authorPart.split(/\n/).slice(mayorCase ? 1 : 0); - - if (authorParts.length % 2 !== 0) { - console.error(`🤔 - Author parts for ${file} looks strange!`); - } - - const authors: Author[] = []; - - do { - const name = authorParts.shift() as string; - const id = toASCIIString(name); - let portrait = id + ".jpg"; - const mandates = (authorParts.shift() as string).split(/\s*,\s+/); - - try { - await access(pathJoin("public", "images", "portraits", portrait)); - } catch (err) { - portrait = "default.svg"; - } - - const author = { - id, - name, - mandates, - portrait, - totalSignificantWords: 0, - totalWords: 0, - }; - - authors.push(author); - globalStats.authors[author.id] = author; - } while (authorParts.length); - - const content = parts - .slice(mayorCase ? 0 : 1) - .join("\n\n") - .trim(); - const group: BaseGroup = buildGroupDetails( - mayorCase - ? "Majorité municipale : Douai au Cœur (Parti Socialiste)" - : parts.slice(0, 1).join("") - ); - const source = mayorCase - ? sourcesCaptures[1] || sourcesCaptures[0] - : sourcesCaptures[0]; - - const date = `${occurence}-01T00:00:00Z`; - const id = `${occurence}-${publication}-${authors - .map(({ id }) => id) - .join("-")}`; - const response = await axios({ - method: "post", - url: `http://localhost:9000`, - params: { - properties: - '{"annotators":"tokenize,ssplit,mwt,pos,lemma,ner,parse,sentiment","outputFormat":"json"}', - }, - data: content, - }); - - const sentiments = response.data.sentences.map( - ({ sentiment }) => sentiment - ); - const words = response.data.sentences.reduce( - (words, sentence) => - words.concat( - sentence.tokens - .filter(({ pos }) => ["NOUN", "ADJ", "VERB"].includes(pos)) - .map(({ lemma, characterOffsetBegin, characterOffsetEnd }) => ({ - word: lemma, - offset: [characterOffsetBegin, characterOffsetEnd], - })) - ), - [] - ); - - const groupAggregation = groupsAggregations[group.id] || { - id: group.id, - name: group.name, - party: group.party, - partyAbbr: group.abbr, - type: group.type, - logo: group.logo, - totalWords: 0, - writtings: [], - sentences: [], - sentiments: [], - exclamations: [], - questions: [], - bolds: [], - caps: [], - words: [], - authors, - locality: "Douai", - country: "France", - }; - - groupsAggregations[group.id] = groupAggregation; - groupAggregation.authors = [ - ...groupAggregation.authors, - ...authors.filter( - ({ name }) => - !groupAggregation.authors.some( - ({ name: otherName }) => name === otherName - ) - ), - ].sort(sortByName); - - const aggregationsList = [ - groupAggregation, - ...authors.map(({ name, mandates, portrait }) => { - const writerAggregation = writtersAggregations[ - toASCIIString(name) - ] || { - totalWords: 0, - writtings: [], - sentences: [], - sentiments: [], - exclamations: [], - questions: [], - bolds: [], - caps: [], - words: [], - portrait, - name, - mandates, - groups: [group], - locality: "Douai", - country: "France", - }; - - writtersAggregations[toASCIIString(name)] = writerAggregation; - writerAggregation.groups = [ - ...writerAggregation.groups, - ...(writerAggregation.groups.some( - ({ name: otherName }) => name === otherName - ) - ? [] - : []), - ].sort(sortByName); - - writerAggregation.mandates = [ - ...Array.from(new Set(writerAggregation.mandates.concat(mandates))), - ]; - - return writerAggregation; - }), - ]; - - aggregationsList.forEach((aggregation) => { - aggregation.writtings.push({ date, id }); - }); - - const exclamation = { - id, - date, - count: response.data.sentences.reduce((count, sentence) => { - return sentence.tokens.reduce((count, { lemma }) => { - return count + (lemma === "!" ? 1 : 0); - }, count); - }, 0), - }; - aggregationsList.forEach((aggregation) => { - aggregation.exclamations.push(exclamation); - }); - - const question = { - id, - date, - count: response.data.sentences.reduce((count, sentence) => { - return sentence.tokens.reduce((count, { lemma }) => { - return count + (lemma === "?" ? 1 : 0); - }, count); - }, 0), - }; - aggregationsList.forEach((aggregation) => { - aggregation.questions.push(question); - }); - - const bold = { - id, - date, - // Very approximative count, could parse MD contents and - // use the AST - count: - Math.floor([...Array.from(content.matchAll(/\*\*/gm))]?.length / 2) || - 0, - }; - aggregationsList.forEach((aggregation) => { - aggregation.bolds.push(bold); - }); - - const cap = { - id, - date, - count: response.data.sentences.reduce((count, sentence) => { - return sentence.tokens.reduce((count, { originalText }) => { - return ( - count + - (originalText.match(/[a-zA-Z]{5,}/) && - originalText.toUpperCase() === originalText - ? 1 - : 0) - ); - }, count); - }, 0), - }; - aggregationsList.forEach((aggregation) => { - aggregation.caps.push(cap); - }); - - const sentence = { id, date, count: response.data.sentences.length }; - aggregationsList.forEach((aggregation) => { - aggregation.sentences.push(sentence); - }); - - const sentiment = { - id, - date, - positive: sentiments.filter((sentiment) => sentiment === "Positive") - .length, - neutral: sentiments.filter((sentiment) => sentiment === "Neutral") - .length, - negative: sentiments.filter((sentiment) => sentiment === "Negative") - .length, - }; - aggregationsList.forEach((aggregation) => { - aggregation.sentiments.push(sentiment); - }); - - words.forEach((word) => { - const lemma = nl.lem(word.word); - - aggregationsList.forEach((aggregation) => { - aggregation.words[lemma] = (aggregation.words[lemma] || 0) + 1; - aggregation.totalWords += word.word.length > 3 ? 1 : 0; - }); - }); - - const markdown = `--- -id: "${id}" -authors:${authors - .map( - ({ id, name, mandates, portrait }) => ` -- id: "${id}" - name: "${name}" - mandates: ${mandates - .map( - (mandate) => ` - - "${mandate}"` - ) - .join("")} - portrait: "${portrait}"` - ) - .join("")} -group: - id: "${group.id}" - name: "${group.name}" - type: "${group.type}" - party: "${group.party}" - abbr: "${group.abbr}" - logo: "${group.logo}" -date: "${date}" -publication: "${publication}" -source: "${source}" -language: "fr" -locality: "Douai" -country: "France" ---- - -${content} -`; - await writeFile(pathJoin("contents", "tribunes", `${id}.md`), markdown); - } - } - - for (const key of Object.keys(writtersAggregations)) { - const summary = { - sentences: computeStats(writtersAggregations[key].sentences), - exclamations: computeStats(writtersAggregations[key].exclamations), - questions: computeStats(writtersAggregations[key].questions), - bolds: computeStats(writtersAggregations[key].bolds), - caps: computeStats(writtersAggregations[key].caps), - sentiments: computeSentimentStats(writtersAggregations[key].sentiments), - }; - const allWords = Object.keys(writtersAggregations[key].words).filter( - (word) => word.length > 3 - ); - - globalStats.authors[key].totalWords = writtersAggregations[key].totalWords; - globalStats.authors[key].totalSignificantWords = allWords.length; - - const finalAggregation = { - ...writtersAggregations[key], - summary: shrinkSummary(summary), - words: allWords - .sort((wordA, wordB) => - writtersAggregations[key].words[wordA] < - writtersAggregations[key].words[wordB] - ? 1 - : writtersAggregations[key].words[wordA] > - writtersAggregations[key].words[wordB] - ? -1 - : 0 - ) - .slice(0, 25) - .map((word) => ({ - word, - count: writtersAggregations[key].words[word], - })), - }; - - await writeFile( - pathJoin("contents", "writters", `${key}.json`), - JSON.stringify(finalAggregation, null, 2) - ); - } - - for (const key of Object.keys(groupsAggregations)) { - const summary = { - sentences: computeStats(groupsAggregations[key].sentences), - exclamations: computeStats(groupsAggregations[key].exclamations), - questions: computeStats(groupsAggregations[key].questions), - bolds: computeStats(groupsAggregations[key].bolds), - caps: computeStats(groupsAggregations[key].caps), - sentiments: computeSentimentStats(groupsAggregations[key].sentiments), - }; - - aggregatesStats(summary.sentences, globalStats.sentences); - aggregatesStats(summary.exclamations, globalStats.exclamations); - aggregatesStats(summary.questions, globalStats.questions); - aggregatesStats(summary.bolds, globalStats.bolds); - aggregatesStats(summary.caps, globalStats.caps); - aggregatesStats( - summary.sentiments.positive, - globalStats.sentiments.positive - ); - aggregatesStats( - summary.sentiments.negative, - globalStats.sentiments.negative - ); - aggregatesStats(summary.sentiments.neutral, globalStats.sentiments.neutral); - - const finalAggregation = { - ...groupsAggregations[key], - summary: shrinkSummary(summary), - words: Object.keys(groupsAggregations[key].words) - .filter((word) => word.length > 3) - .sort((wordA, wordB) => - groupsAggregations[key].words[wordA] < - groupsAggregations[key].words[wordB] - ? 1 - : groupsAggregations[key].words[wordA] > - groupsAggregations[key].words[wordB] - ? -1 - : 0 - ) - .slice(0, 25) - .map((word) => ({ word, count: groupsAggregations[key].words[word] })), - }; - - await writeFile( - pathJoin("contents", "groups", `${key}.json`), - JSON.stringify(finalAggregation, null, 2) - ); - } - - await writeFile( - pathJoin("contents", `globalStats.json`), - JSON.stringify(shrinkSummary(globalStats), null, 2) - ); -} - -function buildGroupDetails(fullName): BaseGroup { - const matches = fullName.match(/^(.*) :([^\(]*)(\(.*\)|)$/); - const name = matches[2].trim() || "Non-Affilié·es"; - const type = matches[1].trim(); - let party = matches[3].trim() || "Sans-Étiquette"; - let abbr = "SE"; - let logo = "default.svg"; - - if (party.includes("Europe Écologie les Verts")) { - party = "Europe Écologie-Les Verts"; - abbr = "EELV"; - logo = "eelv-douaisis.svg"; - } - if (party.includes("Vivre Douai")) { - party = "Citoyen·nes de Vivre Douai"; - abbr = "SE"; - logo = "douai-au-coeur.svg"; - } - if (party.includes("Parti Socialiste")) { - party = "Parti Socialiste"; - abbr = "PS"; - logo = "ps.png"; - } - if ( - party.includes("L’humain d’abord pour Douai") || - party.includes("Parti Communiste") - ) { - party = "Parti Communiste Français"; - abbr = "PCF"; - logo = "pcf.svg"; - } - if (party.includes("Rassemblement National")) { - party = "Rassemblement National"; - abbr = "RN"; - } - if (party.includes("UMP")) { - party = "Union pour un Mouvement Populaire"; - abbr = "UMP"; - } - if (fullName.includes("Douai dynamique et durable")) { - party = "Alliance LReM-Modem"; - abbr = "DVD"; - } - if (fullName.includes("Ensemble faisons Douai")) { - party = "Sans-Étiquette"; - abbr = "SE"; - } - - return { - id: toASCIIString(`${name} ${abbr}`), - name, - type, - party, - abbr, - logo, - }; -} - -function createBaseStatsObject() { - return { - authors: {}, - sentences: createBaseStatsItem(), - exclamations: createBaseStatsItem(), - questions: createBaseStatsItem(), - bolds: createBaseStatsItem(), - caps: createBaseStatsItem(), - sentiments: { - positive: createBaseStatsItem(), - neutral: createBaseStatsItem(), - negative: createBaseStatsItem(), - }, - }; -} - -function computeSentimentStats( - occurences: SentimentOccurenceItem[] -): StatsSummary["sentiments"] { - return ["positive", "neutral", "negative"].reduce((stats, sentiment) => { - const reshapedOccurences = occurences.map(({ id, date, ...occurence }) => ({ - id, - date, - count: occurence[sentiment], - })); - - return { - ...stats, - [sentiment]: computeStats(reshapedOccurences), - }; - }, {} as Partial) as StatsSummary["sentiments"]; -} - -function shrinkSummary( - summary: Omit -): Omit { - return { - ...summary, - sentences: shrinkStats(summary.sentences), - exclamations: shrinkStats(summary.exclamations), - questions: shrinkStats(summary.questions), - bolds: shrinkStats(summary.bolds), - caps: shrinkStats(summary.caps), - sentiments: { - positive: shrinkStats(summary.sentiments.positive), - neutral: shrinkStats(summary.sentiments.neutral), - negative: shrinkStats(summary.sentiments.negative), - }, - }; -} diff --git a/bin/presence.ts b/bin/presences.ts similarity index 99% rename from bin/presence.ts rename to bin/presences.ts index f5686779..4ddf1e66 100644 --- a/bin/presence.ts +++ b/bin/presences.ts @@ -4,7 +4,6 @@ import { toASCIIString } from "../src/utils/ascii"; import { OccurenceItem, aggregatesStats, - computeStats, createBaseStatsItem, shrinkStats, sortByDate, diff --git a/bin/profile.ts b/bin/profiles.ts similarity index 100% rename from bin/profile.ts rename to bin/profiles.ts diff --git a/bin/stats.ts b/bin/stats.ts new file mode 100644 index 00000000..f25c0e55 --- /dev/null +++ b/bin/stats.ts @@ -0,0 +1,388 @@ +import { + createBaseStatsItem, + shrinkStats, + sortByName, + aggregatesStats, + computeStats, +} from "../src/utils/stats"; +import { parse } from "yaml"; +import { readFile, writeFile, readDir } from "../src/utils/files"; +import { join as pathJoin } from "path"; +import axios from "axios"; +import { exec as _exec } from "node:child_process"; +import nodeLefff from "node-lefff"; +import { toASCIIString } from "../src/utils/ascii"; +import type { + SentimentOccurenceItem, + StatsSummary, +} from "../src/utils/writters"; +import type { + BaseGroup, + TribuneFrontmatterMetadata, +} from "../src/utils/tribunes"; + +run(); + +async function run() { + const globalStats = await createBaseStatsObject(); + const nl = await nodeLefff.load(); + const files = await readDir(pathJoin("contents", "tribunes")); + const writtersAggregations = {}; + const groupsAggregations = {}; + + for (const file of files) { + console.warn(`➕ - Processing ${file}.`); + + const content = await readFile(pathJoin("contents", "tribunes", file)); + const parts = content + .split(/\-\-\-/) + .map((s) => s.trim()) + .filter((id) => id); + const metadata = parse(parts[0]) as TribuneFrontmatterMetadata; + const markdown = parts[1]; + const response = await axios({ + method: "post", + url: `http://localhost:9000`, + params: { + properties: + '{"annotators":"tokenize,ssplit,mwt,pos,lemma,ner,parse,sentiment","outputFormat":"json"}', + }, + data: markdown, + }); + + const sentiments = response.data.sentences.map( + ({ sentiment }) => sentiment + ); + const words = response.data.sentences.reduce( + (words, sentence) => + words.concat( + sentence.tokens + .filter(({ pos }) => ["NOUN", "ADJ", "VERB"].includes(pos)) + .map(({ lemma, characterOffsetBegin, characterOffsetEnd }) => ({ + word: lemma, + offset: [characterOffsetBegin, characterOffsetEnd], + })) + ), + [] + ); + + const { id, date, group, authors } = metadata; + const groupAggregation = groupsAggregations[group.id] || { + id: group.id, + name: group.name, + party: group.party, + partyAbbr: group.abbr, + type: group.type, + logo: group.logo, + totalWords: 0, + writtings: [], + sentences: [], + sentiments: [], + exclamations: [], + questions: [], + bolds: [], + caps: [], + words: [], + authors, + locality: "Douai", + country: "France", + }; + + groupsAggregations[group.id] = groupAggregation; + groupAggregation.authors = [ + ...groupAggregation.authors, + ...authors.filter( + ({ name }) => + !groupAggregation.authors.some( + ({ name: otherName }) => name === otherName + ) + ), + ].sort(sortByName); + + const aggregationsList = [ + groupAggregation, + ...authors.map(({ name, mandates, portrait }) => { + const writerAggregation = writtersAggregations[toASCIIString(name)] || { + totalWords: 0, + writtings: [], + sentences: [], + sentiments: [], + exclamations: [], + questions: [], + bolds: [], + caps: [], + words: [], + portrait, + name, + mandates, + groups: [group], + locality: "Douai", + country: "France", + }; + + writtersAggregations[toASCIIString(name)] = writerAggregation; + writerAggregation.groups = [ + ...writerAggregation.groups, + ...(writerAggregation.groups.some( + ({ name: otherName }) => name === otherName + ) + ? [] + : []), + ].sort(sortByName); + + writerAggregation.mandates = [ + ...Array.from(new Set(writerAggregation.mandates.concat(mandates))), + ]; + + return writerAggregation; + }), + ]; + + aggregationsList.forEach((aggregation) => { + aggregation.writtings.push({ date, id }); + }); + + const exclamation = { + id, + date, + count: response.data.sentences.reduce((count, sentence) => { + return sentence.tokens.reduce((count, { lemma }) => { + return count + (lemma === "!" ? 1 : 0); + }, count); + }, 0), + }; + aggregationsList.forEach((aggregation) => { + aggregation.exclamations.push(exclamation); + }); + + const question = { + id, + date, + count: response.data.sentences.reduce((count, sentence) => { + return sentence.tokens.reduce((count, { lemma }) => { + return count + (lemma === "?" ? 1 : 0); + }, count); + }, 0), + }; + aggregationsList.forEach((aggregation) => { + aggregation.questions.push(question); + }); + + const bold = { + id, + date, + // Very approximative count, could parse MD contents and + // use the AST + count: + Math.floor([...Array.from(content.matchAll(/\*\*/gm))]?.length / 2) || + 0, + }; + aggregationsList.forEach((aggregation) => { + aggregation.bolds.push(bold); + }); + + const cap = { + id, + date, + count: response.data.sentences.reduce((count, sentence) => { + return sentence.tokens.reduce((count, { originalText }) => { + return ( + count + + (originalText.match(/[a-zA-Z]{5,}/) && + originalText.toUpperCase() === originalText + ? 1 + : 0) + ); + }, count); + }, 0), + }; + aggregationsList.forEach((aggregation) => { + aggregation.caps.push(cap); + }); + + const sentence = { id, date, count: response.data.sentences.length }; + aggregationsList.forEach((aggregation) => { + aggregation.sentences.push(sentence); + }); + + const sentiment = { + id, + date, + positive: sentiments.filter((sentiment) => sentiment === "Positive") + .length, + neutral: sentiments.filter((sentiment) => sentiment === "Neutral").length, + negative: sentiments.filter((sentiment) => sentiment === "Negative") + .length, + }; + aggregationsList.forEach((aggregation) => { + aggregation.sentiments.push(sentiment); + }); + + words.forEach((word) => { + const lemma = nl.lem(word.word); + + aggregationsList.forEach((aggregation) => { + aggregation.words[lemma] = (aggregation.words[lemma] || 0) + 1; + aggregation.totalWords += word.word.length > 3 ? 1 : 0; + }); + }); + } + + for (const key of Object.keys(writtersAggregations)) { + const summary = { + sentences: computeStats(writtersAggregations[key].sentences), + exclamations: computeStats(writtersAggregations[key].exclamations), + questions: computeStats(writtersAggregations[key].questions), + bolds: computeStats(writtersAggregations[key].bolds), + caps: computeStats(writtersAggregations[key].caps), + sentiments: computeSentimentStats(writtersAggregations[key].sentiments), + }; + const allWords = Object.keys(writtersAggregations[key].words).filter( + (word) => word.length > 3 + ); + + globalStats.authors[key].totalWords = writtersAggregations[key].totalWords; + globalStats.authors[key].totalSignificantWords = allWords.length; + + const finalAggregation = { + ...writtersAggregations[key], + summary: shrinkSummary(summary), + words: allWords + .sort((wordA, wordB) => + writtersAggregations[key].words[wordA] < + writtersAggregations[key].words[wordB] + ? 1 + : writtersAggregations[key].words[wordA] > + writtersAggregations[key].words[wordB] + ? -1 + : 0 + ) + .slice(0, 25) + .map((word) => ({ + word, + count: writtersAggregations[key].words[word], + })), + }; + + await writeFile( + pathJoin("contents", "writters", `${key}.json`), + JSON.stringify(finalAggregation, null, 2) + ); + } + + for (const key of Object.keys(groupsAggregations)) { + const summary = { + sentences: computeStats(groupsAggregations[key].sentences), + exclamations: computeStats(groupsAggregations[key].exclamations), + questions: computeStats(groupsAggregations[key].questions), + bolds: computeStats(groupsAggregations[key].bolds), + caps: computeStats(groupsAggregations[key].caps), + sentiments: computeSentimentStats(groupsAggregations[key].sentiments), + }; + + aggregatesStats(summary.sentences, globalStats.sentences); + aggregatesStats(summary.exclamations, globalStats.exclamations); + aggregatesStats(summary.questions, globalStats.questions); + aggregatesStats(summary.bolds, globalStats.bolds); + aggregatesStats(summary.caps, globalStats.caps); + aggregatesStats( + summary.sentiments.positive, + globalStats.sentiments.positive + ); + aggregatesStats( + summary.sentiments.negative, + globalStats.sentiments.negative + ); + aggregatesStats(summary.sentiments.neutral, globalStats.sentiments.neutral); + + const finalAggregation = { + ...groupsAggregations[key], + summary: shrinkSummary(summary), + words: Object.keys(groupsAggregations[key].words) + .filter((word) => word.length > 3) + .sort((wordA, wordB) => + groupsAggregations[key].words[wordA] < + groupsAggregations[key].words[wordB] + ? 1 + : groupsAggregations[key].words[wordA] > + groupsAggregations[key].words[wordB] + ? -1 + : 0 + ) + .slice(0, 25) + .map((word) => ({ word, count: groupsAggregations[key].words[word] })), + }; + + await writeFile( + pathJoin("contents", "groups", `${key}.json`), + JSON.stringify(finalAggregation, null, 2) + ); + } + + await writeFile( + pathJoin("contents", `globalStats.json`), + JSON.stringify(shrinkSummary(globalStats), null, 2) + ); +} + +async function createBaseStatsObject(): Promise { + let globalStats; + + try { + globalStats = JSON.parse( + (await readFile(pathJoin("contents", `globalStats.json`))).toString() + ) as StatsSummary; + } catch (err) { + console.error(`💥 - Cannot read the global stats object.`); + } + + return { + authors: {}, + ...globalStats, + sentences: createBaseStatsItem(), + exclamations: createBaseStatsItem(), + questions: createBaseStatsItem(), + bolds: createBaseStatsItem(), + caps: createBaseStatsItem(), + sentiments: { + positive: createBaseStatsItem(), + neutral: createBaseStatsItem(), + negative: createBaseStatsItem(), + }, + }; +} + +function computeSentimentStats( + occurences: SentimentOccurenceItem[] +): StatsSummary["sentiments"] { + return ["positive", "neutral", "negative"].reduce((stats, sentiment) => { + const reshapedOccurences = occurences.map(({ id, date, ...occurence }) => ({ + id, + date, + count: occurence[sentiment], + })); + + return { + ...stats, + [sentiment]: computeStats(reshapedOccurences), + }; + }, {} as Partial) as StatsSummary["sentiments"]; +} + +function shrinkSummary( + summary: Omit +): Omit { + return { + ...summary, + sentences: shrinkStats(summary.sentences), + exclamations: shrinkStats(summary.exclamations), + questions: shrinkStats(summary.questions), + bolds: shrinkStats(summary.bolds), + caps: shrinkStats(summary.caps), + sentiments: { + positive: shrinkStats(summary.sentiments.positive), + neutral: shrinkStats(summary.sentiments.neutral), + negative: shrinkStats(summary.sentiments.negative), + }, + }; +} diff --git a/bin/tribunes.ts b/bin/tribunes.ts new file mode 100644 index 00000000..5ad0b7de --- /dev/null +++ b/bin/tribunes.ts @@ -0,0 +1,207 @@ +import { readFile, writeFile, readDir, access } from "../src/utils/files"; +import { join as pathJoin } from "path"; +import { tmpdir } from "node:os"; +import { exec as _exec } from "node:child_process"; +import { promisify } from "node:util"; +import { toASCIIString } from "../src/utils/ascii"; +import type { Author, BaseGroup } from "../src/utils/tribunes"; + +const exec = promisify(_exec); + +run(); + +async function run() { + const files = await readDir(pathJoin("sources", "tribunes")); + + for (const file of files) { + const content = await readFile(pathJoin("sources", "tribunes", file)); + + console.warn(`➕ - Processing ${file}.`); + + const publication = file.split("-").slice(2).join("-").replace(/\.md$/, ""); + const occurence = file.split("-").slice(0, 2).join("-"); + const parts = content.split(/[\-]{3,}/gm); + const sources = parts[0] + .split(/(\r?\n)+/gm) + .filter((line) => line.startsWith("https://")); + const tempDir = tmpdir(); + const sourcesCaptures = await sources.reduce( + async (sourcesCaptures, source) => { + const captures = await sourcesCaptures; + const filename = `${publication}-${occurence}-p${source.replace( + /^.*p([0-9]+).svgz/, + "$1" + )}.png`; + const captureDestination = pathJoin( + "public", + "images", + "sources", + filename + ); + const tempFile = pathJoin(tempDir, filename); + + await new Promise((resolve) => setTimeout(resolve, 500)); + await exec( + `google-chrome --headless --screenshot=${tempFile} ${source}` + ); + await new Promise((resolve) => setTimeout(resolve, 500)); + await exec(`convert ${tempFile} -trim ${captureDestination}`); + return [...captures, captureDestination]; + }, + Promise.resolve([] as string[]) + ); + + if (!sources.length) { + console.error(`🤔 - No sources for ${file} !`); + continue; + } + + const tribunes = parts.slice(1); + + for (const tribune of tribunes) { + const parts = tribune.trim().split(/\n[\n]+/gm); + const authorPart = (parts.pop() as string).trim(); + const mayorCase = !!authorPart.match(/^\s*votre maire/i); + const authorParts = authorPart.split(/\n/).slice(mayorCase ? 1 : 0); + + if (authorParts.length % 2 !== 0) { + console.error(`🤔 - Author parts for ${file} looks strange!`); + } + + const authors: Author[] = []; + + do { + const name = authorParts.shift() as string; + const id = toASCIIString(name); + let portrait = id + ".jpg"; + const mandates = (authorParts.shift() as string).split(/\s*,\s+/); + + try { + await access(pathJoin("public", "images", "portraits", portrait)); + } catch (err) { + portrait = "default.svg"; + } + + const author: Author = { + id, + name, + mandates, + portrait, + }; + + authors.push(author); + } while (authorParts.length); + + const content = parts + .slice(mayorCase ? 0 : 1) + .join("\n\n") + .trim(); + const group: BaseGroup = buildGroupDetails( + mayorCase + ? "Majorité municipale : Douai au Cœur (Parti Socialiste)" + : parts.slice(0, 1).join("") + ); + const source = mayorCase + ? sourcesCaptures[1] || sourcesCaptures[0] + : sourcesCaptures[0]; + + const date = `${occurence}-01T00:00:00Z`; + const id = `${occurence}-${publication}-${authors + .map(({ id }) => id) + .join("-")}`; + + const markdown = `--- +id: "${id}" +authors:${authors + .map( + ({ id, name, mandates, portrait }) => ` +- id: "${id}" + name: "${name}" + mandates: ${mandates + .map( + (mandate) => ` + - "${mandate}"` + ) + .join("")} + portrait: "${portrait}"` + ) + .join("")} +group: + id: "${group.id}" + name: "${group.name}" + type: "${group.type}" + party: "${group.party}" + abbr: "${group.abbr}" + logo: "${group.logo}" +date: "${date}" +publication: "${publication}" +source: "${source}" +language: "fr" +locality: "Douai" +country: "France" +--- + +${content} +`; + await writeFile(pathJoin("contents", "tribunes", `${id}.md`), markdown); + } + } +} + +function buildGroupDetails(fullName): BaseGroup { + const matches = fullName.match(/^(.*) :([^\(]*)(\(.*\)|)$/); + const name = matches[2].trim() || "Non-Affilié·es"; + const type = matches[1].trim(); + let party = matches[3].trim() || "Sans-Étiquette"; + let abbr = "SE"; + let logo = "default.svg"; + + if (party.includes("Europe Écologie les Verts")) { + party = "Europe Écologie-Les Verts"; + abbr = "EELV"; + logo = "eelv-douaisis.svg"; + } + if (party.includes("Vivre Douai")) { + party = "Citoyen·nes de Vivre Douai"; + abbr = "SE"; + logo = "douai-au-coeur.svg"; + } + if (party.includes("Parti Socialiste")) { + party = "Parti Socialiste"; + abbr = "PS"; + logo = "ps.png"; + } + if ( + party.includes("L’humain d’abord pour Douai") || + party.includes("Parti Communiste") + ) { + party = "Parti Communiste Français"; + abbr = "PCF"; + logo = "pcf.svg"; + } + if (party.includes("Rassemblement National")) { + party = "Rassemblement National"; + abbr = "RN"; + } + if (party.includes("UMP")) { + party = "Union pour un Mouvement Populaire"; + abbr = "UMP"; + } + if (fullName.includes("Douai dynamique et durable")) { + party = "Alliance LReM-Modem"; + abbr = "DVD"; + } + if (fullName.includes("Ensemble faisons Douai")) { + party = "Sans-Étiquette"; + abbr = "SE"; + } + + return { + id: toASCIIString(`${name} ${abbr}`), + name, + type, + party, + abbr, + logo, + }; +} diff --git a/contents/globalStats.json b/contents/globalStats.json index a01fe379..666f0725 100644 --- a/contents/globalStats.json +++ b/contents/globalStats.json @@ -602,7 +602,7 @@ "2012-04-douai-notre-ville-jean-pierre-divrechy", "2013-03-douai-notre-ville-jean-pierre-divrechy", "2012-09-douai-notre-ville-odile-hage", - "2014-05-douai-notre-ville-guy-cannie" + "2014-05-douai-notre-ville-francoise-prouvost" ], "restLength": 181 }, @@ -623,11 +623,11 @@ "min": { "value": 0, "ids": [ - "2012-02-douai-notre-ville-cyril-carbonnel", - "2012-06-douai-notre-ville-karine-doyen-carbonnel", - "2012-07-douai-notre-ville-annick-louvion", - "2012-12-douai-notre-ville-monique-amghar", - "2013-01-douai-notre-ville-frederic-chereau" + "2012-01-douai-notre-ville-jacques-vernier", + "2012-04-douai-notre-ville-jacques-vernier", + "2012-06-douai-notre-ville-jacques-vernier", + "2012-07-douai-notre-ville-jacques-vernier", + "2012-09-douai-notre-ville-jacques-vernier" ], "restLength": 362 }, @@ -648,10 +648,10 @@ "value": 0, "ids": [ "2012-09-douai-notre-ville-odile-hage", - "2014-05-douai-notre-ville-guy-cannie", - "2014-07-douai-notre-ville-guy-cannie", - "2015-02-douai-notre-ville-guy-cannie", - "2015-05-douai-notre-ville-guy-cannie" + "2014-05-douai-notre-ville-francoise-prouvost", + "2014-10-douai-notre-ville-marie-helene-quatreboeufs-niklikowski", + "2014-11-douai-notre-ville-marie-delecambre", + "2014-12-douai-notre-ville-chantal-rybak" ], "restLength": 298 }, @@ -671,11 +671,11 @@ "min": { "value": 0, "ids": [ - "2012-01-douai-notre-ville-monique-amghar", - "2012-02-douai-notre-ville-cyril-carbonnel", - "2012-03-douai-notre-ville-rene-lavarde", - "2012-06-douai-notre-ville-karine-doyen-carbonnel", - "2012-07-douai-notre-ville-annick-louvion" + "2012-01-douai-notre-ville-jacques-vernier", + "2012-04-douai-notre-ville-jacques-vernier", + "2012-05-douai-notre-ville-jacques-vernier", + "2012-06-douai-notre-ville-jacques-vernier", + "2012-07-douai-notre-ville-jacques-vernier" ], "restLength": 397 }, @@ -696,11 +696,11 @@ "min": { "value": 0, "ids": [ - "2012-01-douai-notre-ville-monique-amghar", - "2012-07-douai-notre-ville-annick-louvion", - "2013-01-douai-notre-ville-frederic-chereau", "2012-01-douai-notre-ville-jacques-vernier", - "2012-02-douai-notre-ville-jacques-vernier" + "2012-02-douai-notre-ville-jacques-vernier", + "2012-04-douai-notre-ville-jacques-vernier", + "2012-05-douai-notre-ville-jacques-vernier", + "2012-06-douai-notre-ville-jacques-vernier" ], "restLength": 235 }, @@ -744,11 +744,11 @@ "min": { "value": 0, "ids": [ - "2012-01-douai-notre-ville-monique-amghar", "2012-02-douai-notre-ville-jacques-vernier", "2012-03-douai-notre-ville-jacques-vernier", "2012-05-douai-notre-ville-jacques-vernier", - "2012-10-douai-notre-ville-jacques-vernier" + "2012-10-douai-notre-ville-jacques-vernier", + "2012-11-douai-notre-ville-jacques-vernier" ], "restLength": 195 }, diff --git a/contents/groups/douai-au-coeur-eelv.json b/contents/groups/douai-au-coeur-eelv.json index e64b97d4..4c2099cb 100644 --- a/contents/groups/douai-au-coeur-eelv.json +++ b/contents/groups/douai-au-coeur-eelv.json @@ -1317,9 +1317,7 @@ "mandates": [ "Conseiller municipal délégué" ], - "portrait": "guy-caruyer.jpg", - "totalSignificantWords": 0, - "totalWords": 0 + "portrait": "guy-caruyer.jpg" }, { "id": "jean-christophe-leclercq", @@ -1327,9 +1325,7 @@ "mandates": [ "Adjoint au maire" ], - "portrait": "jean-christophe-leclercq.jpg", - "totalSignificantWords": 0, - "totalWords": 0 + "portrait": "jean-christophe-leclercq.jpg" }, { "id": "katia-bittner", @@ -1337,9 +1333,7 @@ "mandates": [ "Conseillère municipale" ], - "portrait": "katia-bittner.jpg", - "totalSignificantWords": 0, - "totalWords": 0 + "portrait": "katia-bittner.jpg" }, { "id": "marie-delattre", @@ -1347,9 +1341,7 @@ "mandates": [ "Adjointe au maire" ], - "portrait": "marie-delattre.jpg", - "totalSignificantWords": 0, - "totalWords": 0 + "portrait": "marie-delattre.jpg" }, { "id": "stephanie-stiernon", @@ -1357,9 +1349,7 @@ "mandates": [ "Adjointe au maire" ], - "portrait": "stephanie-stiernon.jpg", - "totalSignificantWords": 0, - "totalWords": 0 + "portrait": "stephanie-stiernon.jpg" }, { "id": "yves-piquot", @@ -1367,9 +1357,7 @@ "mandates": [ "Conseiller municipal" ], - "portrait": "yves-piquot.jpg", - "totalSignificantWords": 0, - "totalWords": 0 + "portrait": "yves-piquot.jpg" } ], "locality": "Douai", diff --git a/contents/groups/douai-au-coeur-pcf.json b/contents/groups/douai-au-coeur-pcf.json index b8c2eae6..a947f3af 100644 --- a/contents/groups/douai-au-coeur-pcf.json +++ b/contents/groups/douai-au-coeur-pcf.json @@ -1317,9 +1317,7 @@ "mandates": [ "Adjointe au maire" ], - "portrait": "auriane-ait-lasri.jpg", - "totalSignificantWords": 0, - "totalWords": 0 + "portrait": "auriane-ait-lasri.jpg" }, { "id": "jessy-kaboul", @@ -1327,9 +1325,7 @@ "mandates": [ "Adjoint au maire" ], - "portrait": "default.svg", - "totalSignificantWords": 0, - "totalWords": 0 + "portrait": "default.svg" }, { "id": "nora-cherki", @@ -1337,9 +1333,7 @@ "mandates": [ "Conseillère municipale" ], - "portrait": "nora-cherki.jpg", - "totalSignificantWords": 0, - "totalWords": 0 + "portrait": "nora-cherki.jpg" }, { "id": "eric-lemaitre", @@ -1347,9 +1341,7 @@ "mandates": [ "Conseiller municipal" ], - "portrait": "eric-lemaitre.jpg", - "totalSignificantWords": 45, - "totalWords": 47 + "portrait": "eric-lemaitre.jpg" } ], "locality": "Douai", diff --git a/contents/groups/douai-au-coeur-ps.json b/contents/groups/douai-au-coeur-ps.json index 98cb74ae..70724705 100644 --- a/contents/groups/douai-au-coeur-ps.json +++ b/contents/groups/douai-au-coeur-ps.json @@ -333,11 +333,11 @@ }, { "date": "2021-09-01T00:00:00Z", - "id": "2021-09-douai-notre-ville-jean-marie-dupire" + "id": "2021-09-douai-notre-ville-frederic-chereau" }, { "date": "2021-09-01T00:00:00Z", - "id": "2021-09-douai-notre-ville-frederic-chereau" + "id": "2021-09-douai-notre-ville-jean-marie-dupire" }, { "date": "2021-10-01T00:00:00Z", @@ -835,14 +835,14 @@ "count": 21 }, { - "id": "2021-09-douai-notre-ville-jean-marie-dupire", + "id": "2021-09-douai-notre-ville-frederic-chereau", "date": "2021-09-01T00:00:00Z", - "count": 5 + "count": 12 }, { - "id": "2021-09-douai-notre-ville-frederic-chereau", + "id": "2021-09-douai-notre-ville-jean-marie-dupire", "date": "2021-09-01T00:00:00Z", - "count": 12 + "count": 5 }, { "id": "2021-10-douai-notre-ville-frederic-chereau", @@ -1524,18 +1524,18 @@ "negative": 1 }, { - "id": "2021-09-douai-notre-ville-jean-marie-dupire", + "id": "2021-09-douai-notre-ville-frederic-chereau", "date": "2021-09-01T00:00:00Z", "positive": 1, - "neutral": 3, - "negative": 1 + "neutral": 8, + "negative": 2 }, { - "id": "2021-09-douai-notre-ville-frederic-chereau", + "id": "2021-09-douai-notre-ville-jean-marie-dupire", "date": "2021-09-01T00:00:00Z", "positive": 1, - "neutral": 8, - "negative": 2 + "neutral": 3, + "negative": 1 }, { "id": "2021-10-douai-notre-ville-frederic-chereau", @@ -2099,14 +2099,14 @@ "count": 5 }, { - "id": "2021-09-douai-notre-ville-jean-marie-dupire", + "id": "2021-09-douai-notre-ville-frederic-chereau", "date": "2021-09-01T00:00:00Z", - "count": 0 + "count": 2 }, { - "id": "2021-09-douai-notre-ville-frederic-chereau", + "id": "2021-09-douai-notre-ville-jean-marie-dupire", "date": "2021-09-01T00:00:00Z", - "count": 2 + "count": 0 }, { "id": "2021-10-douai-notre-ville-frederic-chereau", @@ -2626,12 +2626,12 @@ "count": 0 }, { - "id": "2021-09-douai-notre-ville-jean-marie-dupire", + "id": "2021-09-douai-notre-ville-frederic-chereau", "date": "2021-09-01T00:00:00Z", "count": 0 }, { - "id": "2021-09-douai-notre-ville-frederic-chereau", + "id": "2021-09-douai-notre-ville-jean-marie-dupire", "date": "2021-09-01T00:00:00Z", "count": 0 }, @@ -3153,12 +3153,12 @@ "count": 0 }, { - "id": "2021-09-douai-notre-ville-jean-marie-dupire", + "id": "2021-09-douai-notre-ville-frederic-chereau", "date": "2021-09-01T00:00:00Z", "count": 0 }, { - "id": "2021-09-douai-notre-ville-frederic-chereau", + "id": "2021-09-douai-notre-ville-jean-marie-dupire", "date": "2021-09-01T00:00:00Z", "count": 0 }, @@ -3680,12 +3680,12 @@ "count": 0 }, { - "id": "2021-09-douai-notre-ville-jean-marie-dupire", + "id": "2021-09-douai-notre-ville-frederic-chereau", "date": "2021-09-01T00:00:00Z", "count": 0 }, { - "id": "2021-09-douai-notre-ville-frederic-chereau", + "id": "2021-09-douai-notre-ville-jean-marie-dupire", "date": "2021-09-01T00:00:00Z", "count": 0 }, @@ -3909,9 +3909,7 @@ "mandates": [ "Adjointe au maire" ], - "portrait": "agnes-dupuis.jpg", - "totalSignificantWords": 0, - "totalWords": 0 + "portrait": "agnes-dupuis.jpg" }, { "id": "carolle-divrechy", @@ -3919,9 +3917,7 @@ "mandates": [ "Conseillère municipale" ], - "portrait": "carolle-divrechy.jpg", - "totalSignificantWords": 0, - "totalWords": 0 + "portrait": "carolle-divrechy.jpg" }, { "id": "frederic-chereau", @@ -3929,9 +3925,7 @@ "mandates": [ "Maire de Douai" ], - "portrait": "frederic-chereau.jpg", - "totalSignificantWords": 0, - "totalWords": 0 + "portrait": "frederic-chereau.jpg" }, { "id": "jean-marie-dupire", @@ -3939,9 +3933,7 @@ "mandates": [ "Conseiller municipal" ], - "portrait": "jean-marie-dupire.jpg", - "totalSignificantWords": 45, - "totalWords": 50 + "portrait": "jean-marie-dupire.jpg" } ], "locality": "Douai", diff --git a/contents/groups/douai-au-coeur-se.json b/contents/groups/douai-au-coeur-se.json index 4556efb5..3ebe6582 100644 --- a/contents/groups/douai-au-coeur-se.json +++ b/contents/groups/douai-au-coeur-se.json @@ -1137,9 +1137,7 @@ "mandates": [ "Conseillère municipale" ], - "portrait": "anne-sophie-audegond.jpg", - "totalSignificantWords": 34, - "totalWords": 41 + "portrait": "anne-sophie-audegond.jpg" }, { "id": "avida-oulahcene", @@ -1147,9 +1145,7 @@ "mandates": [ "Conseillère municipale" ], - "portrait": "avida-oulahcene.jpg", - "totalSignificantWords": 41, - "totalWords": 45 + "portrait": "avida-oulahcene.jpg" }, { "id": "guy-lagache", @@ -1157,9 +1153,7 @@ "mandates": [ "Conseiller municipal" ], - "portrait": "guy-lagache.jpg", - "totalSignificantWords": 0, - "totalWords": 0 + "portrait": "guy-lagache.jpg" }, { "id": "hocine-mazy", @@ -1167,9 +1161,7 @@ "mandates": [ "Adjoint au maire" ], - "portrait": "hocine-mazy.jpg", - "totalSignificantWords": 0, - "totalWords": 0 + "portrait": "hocine-mazy.jpg" }, { "id": "jamila-mekki", @@ -1177,9 +1169,7 @@ "mandates": [ "Conseillère municipale" ], - "portrait": "jamila-mekki.jpg", - "totalSignificantWords": 0, - "totalWords": 0 + "portrait": "jamila-mekki.jpg" }, { "id": "jean-michel-leroy", @@ -1187,9 +1177,7 @@ "mandates": [ "Adjoint au maire" ], - "portrait": "jean-michel-leroy.jpg", - "totalSignificantWords": 0, - "totalWords": 0 + "portrait": "jean-michel-leroy.jpg" }, { "id": "khadija-ahantat", @@ -1197,9 +1185,7 @@ "mandates": [ "Adjointe au maire" ], - "portrait": "khadija-ahantat.jpg", - "totalSignificantWords": 0, - "totalWords": 0 + "portrait": "khadija-ahantat.jpg" }, { "id": "maxime-decupper-laud", @@ -1207,9 +1193,7 @@ "mandates": [ "Conseiller municipal" ], - "portrait": "maxime-decupper-laud.jpg", - "totalSignificantWords": 0, - "totalWords": 0 + "portrait": "maxime-decupper-laud.jpg" }, { "id": "michael-doziere", @@ -1217,9 +1201,7 @@ "mandates": [ "Adjoint au maire" ], - "portrait": "michael-doziere.jpg", - "totalSignificantWords": 0, - "totalWords": 0 + "portrait": "michael-doziere.jpg" }, { "id": "mohamed-kheraki", @@ -1227,9 +1209,7 @@ "mandates": [ "Adjoint au Maire" ], - "portrait": "mohamed-kheraki.jpg", - "totalSignificantWords": 0, - "totalWords": 0 + "portrait": "mohamed-kheraki.jpg" }, { "id": "nadia-bony", @@ -1237,9 +1217,7 @@ "mandates": [ "Conseillère municipale" ], - "portrait": "nadia-bony.jpg", - "totalSignificantWords": 47, - "totalWords": 56 + "portrait": "nadia-bony.jpg" }, { "id": "nathalie-apers", @@ -1247,9 +1225,7 @@ "mandates": [ "Adjointe au maire" ], - "portrait": "nathalie-apers.jpg", - "totalSignificantWords": 0, - "totalWords": 0 + "portrait": "nathalie-apers.jpg" }, { "id": "salima-boukentar", @@ -1257,9 +1233,7 @@ "mandates": [ "Conseillère municipale" ], - "portrait": "salima-boukentar.jpg", - "totalSignificantWords": 0, - "totalWords": 0 + "portrait": "salima-boukentar.jpg" }, { "id": "sebastien-lanclu", @@ -1267,9 +1241,7 @@ "mandates": [ "Conseiller municipal" ], - "portrait": "sebastien-lanclu.jpg", - "totalSignificantWords": 0, - "totalWords": 0 + "portrait": "sebastien-lanclu.jpg" }, { "id": "virginie-malolepszy", @@ -1277,9 +1249,7 @@ "mandates": [ "Adjointe au maire" ], - "portrait": "default.svg", - "totalSignificantWords": 47, - "totalWords": 48 + "portrait": "default.svg" }, { "id": "yvon-sipieter", @@ -1287,9 +1257,7 @@ "mandates": [ "Adjoint au maire" ], - "portrait": "yvon-sipieter.jpg", - "totalSignificantWords": 0, - "totalWords": 0 + "portrait": "yvon-sipieter.jpg" } ], "locality": "Douai", diff --git a/contents/groups/douai-dynamique-et-durable-dvd.json b/contents/groups/douai-dynamique-et-durable-dvd.json index bc655ea5..62d32df0 100644 --- a/contents/groups/douai-dynamique-et-durable-dvd.json +++ b/contents/groups/douai-dynamique-et-durable-dvd.json @@ -1317,9 +1317,7 @@ "mandates": [ "Conseillère municipale d’opposition" ], - "portrait": "anissa-bouchaboun.jpg", - "totalSignificantWords": 40, - "totalWords": 50 + "portrait": "anissa-bouchaboun.jpg" }, { "id": "anne-colin", @@ -1327,9 +1325,7 @@ "mandates": [ "Conseillère municipale d’opposition" ], - "portrait": "anne-colin.jpg", - "totalSignificantWords": 0, - "totalWords": 0 + "portrait": "anne-colin.jpg" }, { "id": "chantal-rybak", @@ -1337,9 +1333,7 @@ "mandates": [ "Conseillère municipale d’opposition" ], - "portrait": "chantal-rybak.jpg", - "totalSignificantWords": 552, - "totalWords": 876 + "portrait": "chantal-rybak.jpg" }, { "id": "coline-craeye", @@ -1347,9 +1341,7 @@ "mandates": [ "Conseillère municipale d’opposition" ], - "portrait": "coline-craeye.jpg", - "totalSignificantWords": 0, - "totalWords": 0 + "portrait": "coline-craeye.jpg" }, { "id": "coline-craeye", @@ -1357,9 +1349,7 @@ "mandates": [ "Conseillère municipale d’opposition" ], - "portrait": "coline-craeye.jpg", - "totalSignificantWords": 617, - "totalWords": 1087 + "portrait": "coline-craeye.jpg" }, { "id": "franz-quatreboeufs", @@ -1367,9 +1357,7 @@ "mandates": [ "Conseiller municipal d’opposition" ], - "portrait": "franz-quatreboeufs.jpg", - "totalSignificantWords": 0, - "totalWords": 0 + "portrait": "franz-quatreboeufs.jpg" }, { "id": "mohamed-felouki", @@ -1377,9 +1365,7 @@ "mandates": [ "Conseiller municipal d’opposition" ], - "portrait": "mohamed-felouki.jpg", - "totalSignificantWords": 43, - "totalWords": 50 + "portrait": "mohamed-felouki.jpg" }, { "id": "xavier-thierry", @@ -1387,9 +1373,7 @@ "mandates": [ "Conseiller municipal d’opposition" ], - "portrait": "xavier-thierry.jpg", - "totalSignificantWords": 0, - "totalWords": 0 + "portrait": "xavier-thierry.jpg" } ], "locality": "Douai", diff --git a/contents/groups/douai-plus-belle-plus-propre-plus-sure-rn.json b/contents/groups/douai-plus-belle-plus-propre-plus-sure-rn.json index 66e0c455..632f85f9 100644 --- a/contents/groups/douai-plus-belle-plus-propre-plus-sure-rn.json +++ b/contents/groups/douai-plus-belle-plus-propre-plus-sure-rn.json @@ -1317,9 +1317,7 @@ "mandates": [ "Conseiller municipal d’opposition" ], - "portrait": "guy-cannie.jpg", - "totalSignificantWords": 0, - "totalWords": 0 + "portrait": "guy-cannie.jpg" }, { "id": "thibaut-francois", @@ -1327,9 +1325,7 @@ "mandates": [ "Conseiller municipal d’opposition" ], - "portrait": "thibaut-francois.jpg", - "totalSignificantWords": 647, - "totalWords": 1274 + "portrait": "thibaut-francois.jpg" }, { "id": "thibaut-francois", @@ -1337,9 +1333,7 @@ "mandates": [ "Conseiller municipal d’opposition" ], - "portrait": "thibaut-francois.jpg", - "totalSignificantWords": 0, - "totalWords": 0 + "portrait": "thibaut-francois.jpg" } ], "locality": "Douai", diff --git a/contents/groups/douaisiens-passionnement-ump.json b/contents/groups/douaisiens-passionnement-ump.json index c7a6034b..c377d6df 100644 --- a/contents/groups/douaisiens-passionnement-ump.json +++ b/contents/groups/douaisiens-passionnement-ump.json @@ -2541,9 +2541,7 @@ "mandates": [ "Conseiller municipal d’opposition" ], - "portrait": "default.svg", - "totalSignificantWords": 0, - "totalWords": 0 + "portrait": "default.svg" }, { "id": "chantal-rybak", @@ -2551,9 +2549,7 @@ "mandates": [ "Conseillère municipale d’opposition" ], - "portrait": "chantal-rybak.jpg", - "totalSignificantWords": 0, - "totalWords": 0 + "portrait": "chantal-rybak.jpg" }, { "id": "cyril-carbonnel", @@ -2561,9 +2557,7 @@ "mandates": [ "Conseiller municipal d’opposition" ], - "portrait": "default.svg", - "totalSignificantWords": 0, - "totalWords": 0 + "portrait": "default.svg" }, { "id": "franz-quatreboeufs", @@ -2571,9 +2565,7 @@ "mandates": [ "Conseiller municipal d’opposition" ], - "portrait": "franz-quatreboeufs.jpg", - "totalSignificantWords": 0, - "totalWords": 0 + "portrait": "franz-quatreboeufs.jpg" }, { "id": "francoise-prouvost", @@ -2581,9 +2573,7 @@ "mandates": [ "Conseillère municipale d’opposition" ], - "portrait": "default.svg", - "totalSignificantWords": 0, - "totalWords": 0 + "portrait": "default.svg" }, { "id": "isabelle-chatelain", @@ -2591,9 +2581,7 @@ "mandates": [ "Conseillère municipale d’opposition" ], - "portrait": "default.svg", - "totalSignificantWords": 0, - "totalWords": 0 + "portrait": "default.svg" }, { "id": "marie-delecambre", @@ -2601,9 +2589,7 @@ "mandates": [ "Conseillère municipale d’opposition" ], - "portrait": "default.svg", - "totalSignificantWords": 0, - "totalWords": 0 + "portrait": "default.svg" }, { "id": "marie-helene-quatreboeufs-niklikowski", @@ -2611,9 +2597,7 @@ "mandates": [ "Conseillère municipale d’opposition" ], - "portrait": "default.svg", - "totalSignificantWords": 0, - "totalWords": 0 + "portrait": "default.svg" } ], "locality": "Douai", diff --git a/contents/groups/ensemble-faisons-douai-se.json b/contents/groups/ensemble-faisons-douai-se.json index 9654b237..9ece6b2f 100644 --- a/contents/groups/ensemble-faisons-douai-se.json +++ b/contents/groups/ensemble-faisons-douai-se.json @@ -1317,9 +1317,7 @@ "mandates": [ "Conseiller municipal d’opposition" ], - "portrait": "francois-guiffard.jpg", - "totalSignificantWords": 0, - "totalWords": 0 + "portrait": "francois-guiffard.jpg" } ], "locality": "Douai", diff --git a/contents/groups/jacques-vernier-ump.json b/contents/groups/jacques-vernier-ump.json index 05ac2a30..a5abfe68 100644 --- a/contents/groups/jacques-vernier-ump.json +++ b/contents/groups/jacques-vernier-ump.json @@ -634,9 +634,7 @@ "Maire de Douai", "Conseiller Régional" ], - "portrait": "default.svg", - "totalSignificantWords": 0, - "totalWords": 0 + "portrait": "default.svg" } ], "locality": "Douai", diff --git a/contents/groups/rassemblement-douai-bleu-marine-rn.json b/contents/groups/rassemblement-douai-bleu-marine-rn.json index f5d3e323..c4a7721f 100644 --- a/contents/groups/rassemblement-douai-bleu-marine-rn.json +++ b/contents/groups/rassemblement-douai-bleu-marine-rn.json @@ -2541,9 +2541,7 @@ "mandates": [ "Conseiller municipal d’opposition" ], - "portrait": "guy-cannie.jpg", - "totalSignificantWords": 0, - "totalWords": 0 + "portrait": "guy-cannie.jpg" }, { "id": "gerard-bailliet", @@ -2551,9 +2549,7 @@ "mandates": [ "Conseiller municipal d’opposition" ], - "portrait": "default.svg", - "totalSignificantWords": 0, - "totalWords": 0 + "portrait": "default.svg" } ], "locality": "Douai", diff --git a/contents/groups/vivons-douai-eelv.json b/contents/groups/vivons-douai-eelv.json index 8c76f429..cfc15d8b 100644 --- a/contents/groups/vivons-douai-eelv.json +++ b/contents/groups/vivons-douai-eelv.json @@ -165,9 +165,7 @@ "mandates": [ "Conseiller municipal d’opposition" ], - "portrait": "default.svg", - "totalSignificantWords": 78, - "totalWords": 93 + "portrait": "default.svg" } ], "locality": "Douai", diff --git a/contents/groups/vivons-douai-pcf.json b/contents/groups/vivons-douai-pcf.json index 446f6038..d817cc46 100644 --- a/contents/groups/vivons-douai-pcf.json +++ b/contents/groups/vivons-douai-pcf.json @@ -201,9 +201,7 @@ "mandates": [ "Conseillère municipale d’opposition" ], - "portrait": "default.svg", - "totalSignificantWords": 52, - "totalWords": 52 + "portrait": "default.svg" }, { "id": "odile-hage", @@ -211,9 +209,7 @@ "mandates": [ "Conseillère municipale d’opposition" ], - "portrait": "default.svg", - "totalSignificantWords": 60, - "totalWords": 62 + "portrait": "default.svg" } ], "locality": "Douai", diff --git a/contents/groups/vivons-douai-ps.json b/contents/groups/vivons-douai-ps.json index cf1fa1ad..6b7dc853 100644 --- a/contents/groups/vivons-douai-ps.json +++ b/contents/groups/vivons-douai-ps.json @@ -525,9 +525,7 @@ "mandates": [ "Conseillère municipale d’opposition" ], - "portrait": "default.svg", - "totalSignificantWords": 72, - "totalWords": 77 + "portrait": "default.svg" }, { "id": "cyril-carbonnel", @@ -535,9 +533,7 @@ "mandates": [ "Conseiller municipal d’opposition" ], - "portrait": "default.svg", - "totalSignificantWords": 0, - "totalWords": 0 + "portrait": "default.svg" }, { "id": "frederic-chereau", @@ -545,9 +541,7 @@ "mandates": [ "Conseiller municipal d’opposition" ], - "portrait": "frederic-chereau.jpg", - "totalSignificantWords": 0, - "totalWords": 0 + "portrait": "frederic-chereau.jpg" }, { "id": "jean-pierre-divrechy", @@ -555,9 +549,7 @@ "mandates": [ "Conseiller municipal d’opposition" ], - "portrait": "default.svg", - "totalSignificantWords": 0, - "totalWords": 0 + "portrait": "default.svg" }, { "id": "karine-doyen-carbonnel", @@ -565,9 +557,7 @@ "mandates": [ "Conseillère municipale d’opposition" ], - "portrait": "default.svg", - "totalSignificantWords": 75, - "totalWords": 85 + "portrait": "default.svg" }, { "id": "monique-amghar", @@ -575,9 +565,7 @@ "mandates": [ "Conseillère municipale d’opposition" ], - "portrait": "default.svg", - "totalSignificantWords": 0, - "totalWords": 0 + "portrait": "default.svg" }, { "id": "rene-lavarde", @@ -585,9 +573,7 @@ "mandates": [ "Conseiller municipal d’opposition" ], - "portrait": "default.svg", - "totalSignificantWords": 64, - "totalWords": 76 + "portrait": "default.svg" } ], "locality": "Douai", diff --git a/package-lock.json b/package-lock.json index 83708e49..1bd6140b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,6 +19,7 @@ "remark-parse": "^10.0.1", "ts-node": "^10.9.1", "unified": "^10.1.2", + "yaml": "^2.3.1", "yerror": "^6.2.1" }, "devDependencies": { @@ -3626,6 +3627,14 @@ "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", "dev": true }, + "node_modules/yaml": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.3.1.tgz", + "integrity": "sha512-2eHWfjaoXgTBC2jNM1LRef62VQa0umtvRiDSk6HSzW7RvS5YtkabJrwYLLEKWBc8a5U2PTSCs+dJjUTJdlHsWQ==", + "engines": { + "node": ">= 14" + } + }, "node_modules/yerror": { "version": "6.2.1", "resolved": "https://registry.npmjs.org/yerror/-/yerror-6.2.1.tgz", @@ -5982,6 +5991,11 @@ "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", "dev": true }, + "yaml": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.3.1.tgz", + "integrity": "sha512-2eHWfjaoXgTBC2jNM1LRef62VQa0umtvRiDSk6HSzW7RvS5YtkabJrwYLLEKWBc8a5U2PTSCs+dJjUTJdlHsWQ==" + }, "yerror": { "version": "6.2.1", "resolved": "https://registry.npmjs.org/yerror/-/yerror-6.2.1.tgz", diff --git a/package.json b/package.json index 8d6060f3..af0a5ba2 100644 --- a/package.json +++ b/package.json @@ -10,9 +10,10 @@ "build": "next build && touch out/.nojekyll && echo 'keskidiz.nicolasfroidure.fr' > out/CNAME", "start": "next start", "types": "tsc", - "presence": "ts-node --esm -O '{ \"target\": \"es2022\", \"module\": \"node16\" }' bin/presence.ts", - "parse": "ts-node --esm -O '{ \"target\": \"es2022\", \"module\": \"node16\" }' bin/parse.ts", - "profile": "ts-node --esm -O '{ \"target\": \"es2022\", \"module\": \"node16\" }' bin/profile.ts" + "presences": "ts-node --esm -O '{ \"target\": \"es2022\", \"module\": \"node16\" }' bin/presences.ts", + "tribunes": "ts-node --esm -O '{ \"target\": \"es2022\", \"module\": \"node16\" }' bin/tribunes.ts", + "stats": "ts-node --esm -O '{ \"target\": \"es2022\", \"module\": \"node16\" }' bin/stats.ts", + "profiles": "ts-node --esm -O '{ \"target\": \"es2022\", \"module\": \"node16\" }' bin/profiles.ts" }, "dependencies": { "@types/inline-style-prefixer": "^5.0.0", @@ -25,6 +26,7 @@ "remark-parse": "^10.0.1", "ts-node": "^10.9.1", "unified": "^10.1.2", + "yaml": "^2.3.1", "yerror": "^6.2.1" }, "devDependencies": { diff --git a/src/utils/tribunes.ts b/src/utils/tribunes.ts index 54c0a8cb..925ace7e 100644 --- a/src/utils/tribunes.ts +++ b/src/utils/tribunes.ts @@ -6,8 +6,8 @@ export type Author = { name: string; mandates: string[]; portrait: string; - totalSignificantWords: number; - totalWords: number; + totalSignificantWords?: number; + totalWords?: number; }; export type BaseGroup = { id: string;