[es-index] plan the structure for index data

[vchrombie]

Need to plan for the structure of the index data which is consumed by Prosoul.

[vchrombie]

I checked the existing index format which I used during the microtask-9.

[vchrombie]

https://localhost:9200/scava-metrics/ (details about mappings and other info)

{
  "scava-metrics": {
    "aliases": {},
    "mappings": {
      "items": {
        "dynamic_templates": [
          {
            "notanalyzed": {
              "match": "*",
              "match_mapping_type": "string",
              "mapping": {
                "type": "keyword"
              }
            }
          },
          {
            "formatdate": {
              "match": "*",
              "match_mapping_type": "date",
              "mapping": {
                "format": "strict_date_optional_time||epoch_millis",
                "type": "date"
              }
            }
          }
        ],
        "properties": {
          "datetime": {
            "type": "date"
          },
          "meta": {
            "properties": {
              "top_projects": {
                "type": "keyword"
              }
            }
          },
          "metric_class": {
            "type": "keyword"
          },
          "metric_desc": {
            "type": "keyword"
          },
          "metric_es_compute": {
            "type": "keyword"
          },
          "metric_es_value": {
            "type": "float"
          },
          "metric_es_value_weighted": {
            "type": "long"
          },
          "metric_id": {
            "type": "keyword"
          },
          "metric_name": {
            "type": "keyword"
          },
          "metric_type": {
            "type": "keyword"
          },
          "project": {
            "type": "keyword"
          },
          "scava": {
            "type": "object",
            "dynamic": "false"
          },
          "uuid": {
            "type": "keyword"
          }
        }
      }
    },
    "settings": {
      "index": {
        "creation_date": "1591118684260",
        "number_of_shards": "5",
        "number_of_replicas": "1",
        "uuid": "aacvzk1lQdSxJrbQtn-jXA",
        "version": {
          "created": "6010099"
        },
        "provided_name": "scava-metrics"
      }
    }
  }
}

[vchrombie]

https://localhost:9200/scava-metrics/_search?pretty (data in the index)

{
  "took" : 22,
  "timed_out" : false,
  "_shards" : {
    "total" : 5,
    "successful" : 5,
    "skipped" : 0,
    "failed" : 0
  },
  "hits" : {
    "total" : 45395,
    "max_score" : 1.0,
    "hits" : [
      {
        "_index" : "scava-metrics",
        "_type" : "items",
        "_id" : "e320092d235a387be6191984d677e265da105e65",
        "_score" : 1.0,
        "_source" : {
          "project" : "gnomethreads",
          "metric_class" : "newsgroups",
          "metric_type" : "LineChart",
          "metric_id" : "newsgroups.cumulativeNewThreads",
          "metric_desc" : "The number of new threads submitted up to and including the current date.",
          "metric_name" : "New Threads (Cumulative)",
          "metric_es_compute" : "sample",
          "datetime" : "2017-10-06T00:00:00+00:00",
          "scava" : {
            "id" : "newsgroups.cumulativeNewThreads",
            "projectId" : "gnomethreads",
            "metricId" : "org.eclipse.scava.metricprovider.historic.newsgroups.newthreads.NewThreadsHistoricMetricProvider",
            "name" : "New Threads (Cumulative)",
            "description" : "The number of new threads submitted up to and including the current date.",
            "type" : "LineChart",
            "datatable" : [
              {
                "Date" : "19980101",
                "Threads" : 0
              },
              {
                "Date" : "19980102",
                "Threads" : 0
              },
.
.
.
              },
              {
                "Date" : "20181228",
                "Threads" : 15453
              },
              {
                "Date" : "20181229",
                "Threads" : 15453
              },
              {
                "Date" : "20181230",
                "Threads" : 15453
              },
              {
                "Date" : "20181231",
                "Threads" : 15453
              }
            ],
            "timeSeries" : true,
            "ordinal" : false,
            "x" : "Date",
            "y" : "Threads",
            "updated" : "20010118",
            "project" : "gnomethreads"
          },
          "metric_es_value" : 15453,
          "uuid" : "d3f8aabce37bf6a275cdfd6b8ea8a9f2ddcac84b",
          "metric_es_value_weighted" : 15453,
          "meta" : {
            "top_projects" : [
              "main"
            ]
          }
        }
      }
    ]
  }
}

[vchrombie]

sample format

{
  "project": "augur" ,
  "metric_class": "bugs" ,
  "metric_type": "LineChart" ,
  "metric_id": "bugs.severity.wontFixBugs_normal" ,
  "metric_desc": [
    "The number of won't-fix bugs per severity level(normal)"
  ] ,
  "metric_name": "Won't-Fix Bug Severity(normal)" ,
  "metric_es_value": 2 ,
  "metric_es_compute": "sample" ,
  "datetime": "2020-01-01T00:00:00+00:00" ,
  "scava_metric_id": "org.eclipse.scava.metricprovider.historic.bugs.severitybugstatus.SeverityBugStatusHistoricMetricProvider" ,
  "uuid": "2fb8da7299fa849c8cda9dc5f9c571493fa67234" ,
  "metric_es_value_weighted": 2 ,
  "meta": {
    "top_projects": [
      "main"
    ]
  }
}

[vchrombie]

sample format

{
  "project": "gnomethreads" ,
  "metric_class": "newsgroups" ,
  "metric_type": "LineChart" ,
  "metric_id": "newsgroups.cumulativeNewThreads" ,
  "metric_desc": "The number of new threads submitted up to and including the current date." ,
  "metric_name": "New Threads (Cumulative)" ,
  "metric_es_compute": "sample" ,
  "datetime": "2017-10-06T00:00:00+00:00" ,
  "metric_es_value": 15453 ,
  "uuid": "d3f8aabce37bf6a275cdfd6b8ea8a9f2ddcac84b" ,
  "metric_es_value_weighted": 15453 ,
  "meta": {
    "top_projects": [
      "main"
    ]
  }
}

[vchrombie]

I worked on the implementation of the gitlab enricher keeping in the target of the above format of the data. I performed the assessment too and it worked like charm. #5 (comment)

{
    "metric_es_compute": "sample",
    "metric_type": "LineChart",
    "metric_class": "issues",
    "metric_id": "issues.numberCreatedIssues",
    "metric_desc": "The number of issues created on a current date.",
    "metric_name": "Number of Created Issues",
    "project": "cms-mobile",
    "datetime": "2020-05-27T00:00:00+00:00",
    "metric_es_value": 1,
    "metric_es_value_weighted": 1,
    "uuid": "730af28a8bbc0fb61448b463f9406aad486640f6",
    "id": "created_issue_730af28a8bbc0fb61448b463f9406aad486640f6",
    "grimoire_creation_date": "2020-05-27T00:00:00+00:00",
    "is_gitlabqm_date": 1
}

I will be mostly following the same format for the rest of the enricher too. 🎉