main.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Basic settings\n",
    "- Make sure to change these variables based on your needs\n",
    "- Some variables may indirectly affect the others (e.g. `vs_package_extract` will error if `vs_package_download` is false)\n",
    "- When creating issue (on GitHub), please also include these variables on your post so I can easily investigate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 15 = VS 2017, 16 = VS 2019, 17 = VS 2022, etc\n",
    "vs_ver_major = 17\n",
    "# release = current channel, pre = preview channel\n",
    "vs_channel = 'release'\n",
    "# Build tools only, no guarantee it will work for other products!\n",
    "vs_product_id = 'Microsoft.VisualStudio.Product.BuildTools'\n",
    "# If neutral language is not found, it will try to find the preferred language(s)\n",
    "# Priority is based on the index and only one language will be chosen at the end\n",
    "# If all of them are not found but the \"language\" key exists, the package will be skipped\n",
    "vs_lang = ['en-US']\n",
    "# Prefer specific chip/arch for packages (works the same way as \"vs_lang\")\n",
    "# Chip value is either X86 or X64, but host arch can be X86, X64, or arm64\n",
    "# The first one should be your system spec, it will be used to check dependency requirement\n",
    "# The second one and the rest are for fallback if the required package is not found\n",
    "vs_chip = ['X64', 'X86']\n",
    "vs_host = ['X64', 'X86']\n",
    "# Target arch should be the same as host to avoid possible file conflict\n",
    "# Currently, there is no reliable (json) key that can be used to check target arch\n",
    "# I don't think the VS installer provides a way to choose target arch either\n",
    "# Because of that, the host and target arch will always be the same\n",
    "# vs_target = ['X64', 'X86']\n",
    "\n",
    "# Automatically accept VS license\n",
    "vs_license_accept = True\n",
    "# Compare the local manifest hash with online hash to avoid redownloading\n",
    "# Sadly, it's unreliable because the online hash is not always be up-to-date\n",
    "# Setting to false will assume the local manifest is always valid and up-to-date\n",
    "vs_verify_manifest = True\n",
    "# Automatically download and/or verify packages\n",
    "# Packages will not be redownloaded if already up-to-date\n",
    "vs_package_download = True\n",
    "# Delete packages that are not listed in payloads list (variable)\n",
    "# That means modifying workloads/components may also delete some packages\n",
    "vs_package_del_outdated = True\n",
    "# Extract downloaded package files for supported package type(s)\n",
    "# Currently, the only supported package type is vsix\n",
    "# If set to false, outputs may still appear but no file will be extracted\n",
    "vs_package_extract = False\n",
    "# Temporarily install Windows SDKs if exist in packages\n",
    "# Files will be copied and Windows SDKs will be uninstalled again after that\n",
    "# Make sure you don't have any conflicting SDKs installed to prevent accidental uninstall\n",
    "vs_package_win_sdk = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# You can use relative path, but absolute path is recommended\n",
    "# Backslashes in path should be escaped and trailing slash must be removed\n",
    "\n",
    "# Also avoid using long path and path with spaces because it will break things\n",
    "# It may be possible once we migrate all scripts to PowerShell (PRs welcome)\n",
    "\n",
    "# For storing json data files (channel info, manifest, etc)\n",
    "data_path = 'data'\n",
    "# For storing downloaded packages\n",
    "cache_path = 'packages'\n",
    "# Root directory for (portable) build tools installation\n",
    "# Unsupported packages will still need to be extracted manually\n",
    "install_path = 'C:\\\\BuildTools'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "# Convert relative path to absolute path\n",
    "# Relative path is based on current working directory\n",
    "data_path = os.path.abspath(data_path)\n",
    "cache_path = os.path.abspath(cache_path)\n",
    "install_path = os.path.abspath(install_path)\n",
    "\n",
    "# Make the data folder if not exist yet\n",
    "if not os.path.exists(data_path):\n",
    "    os.makedirs(data_path)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save channel info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://aka.ms/vs/17/release/channel\n"
     ]
    }
   ],
   "source": [
    "import requests\n",
    "import json\n",
    "\n",
    "channel_info_path = f'{data_path}\\\\channel_info.json'\n",
    "print(f'https://aka.ms/vs/{vs_ver_major}/{vs_channel}/channel')\n",
    "\n",
    "if not vs_verify_manifest and os.path.isfile(channel_info_path):\n",
    "    # Load local channel info file\n",
    "    with open(channel_info_path, encoding = \"utf8\") as file:\n",
    "        load = json.load(file)\n",
    "else:\n",
    "    channel_info = requests.get(url = f'https://aka.ms/vs/{vs_ver_major}/{vs_channel}/channel')\n",
    "    # Get channel info (200 = HTTP OK)\n",
    "    if channel_info.status_code == 200:\n",
    "        # Load the channel info as json and make it pretty\n",
    "        # Dumping it directly may cause encoding error\n",
    "        load = json.loads(channel_info.content)\n",
    "        # Ensure unicode is not converted to ascii\n",
    "        dump = json.dumps(load, indent = 4, ensure_ascii = False)\n",
    "        # Save the channel info\n",
    "        with open(channel_info_path, 'wb') as file:\n",
    "            file.write(dump.encode())\n",
    "    else:\n",
    "        raise ConnectionError('Failed to download channel info!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Case insensitive dict key (value is still case sensitive)\n",
    "# For dealing with case inconsistencies\n",
    "from requests.structures import CaseInsensitiveDict as LowerDict\n",
    "\n",
    "# I noticed some case inconsistencies while observing the manifest\n",
    "# Most of them are related to package/dependency id, chip/arch, and language \n",
    "# This function should be used when comparing/checking string in above scenarios \n",
    "# However, you should keep the original case when saving the string/variable\n",
    "def lower(string):\n",
    "    \"\"\" Convert everything that contains string (`str`, `list`, etc) to lower case \"\"\"\n",
    "    if isinstance(string, str):\n",
    "        return None if string == None else string.lower()\n",
    "    elif isinstance(string, (list, tuple, set, dict, LowerDict)):\n",
    "        return [ None if i == None else i.lower() for i in string ]\n",
    "\n",
    "def lower_remove(list: list, string: str):\n",
    "    \"\"\" Remove all matched string from list (case insensitive) \"\"\"\n",
    "    for i in list:\n",
    "        if i == string or i == lower(string):\n",
    "            list.remove(i)\n",
    "        elif lower(i) == string:\n",
    "            list.remove(lower(i))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Visual Studio Build Tools 2022 (17.7.34031.279)\n",
      "The Visual Studio Build Tools allows you to build native and managed MSBuild-based applications without requiring the Visual Studio IDE. There are options to install the Visual C++ compilers and libraries, MFC, ATL, and C++/CLI support.\n",
      "License: https://go.microsoft.com/fwlink/?LinkId=2179911\n",
      "\n",
      "By continuing, I already accepted the license listed above\n"
     ]
    }
   ],
   "source": [
    "# Regex\n",
    "import re\n",
    "\n",
    "# Read the channel info again\n",
    "channel_info = LowerDict(load)\n",
    "\n",
    "for i in channel_info['channelItems']:\n",
    "    # Search for VS product id (build tools)\n",
    "    if lower(i['id']) == lower(vs_product_id):\n",
    "        # Save latest build version\n",
    "        if i.get('version'): vs_ver = i['version']\n",
    "\n",
    "        if i.get('localizedResources'):\n",
    "            for j in i['localizedResources']:\n",
    "                # \"localizedResources\" doesn't have \"neutral\" language\n",
    "                # Prioritize preferred language(s) based on the index\n",
    "                for k in vs_lang:\n",
    "                    # Print license link and other info\n",
    "                    if lower(j['language']) == lower(k):\n",
    "                        print(f'{j[\"title\"]} ({vs_ver})')\n",
    "                        print(j['description'])\n",
    "                        print(f'License: {j[\"license\"]}\\n')\n",
    "                        break\n",
    "            break\n",
    "\n",
    "# Save the minor version for later\n",
    "vs_ver_minor = re.search(r'^\\d+\\.(\\d+)\\.', vs_ver).group(1)\n",
    "\n",
    "if vs_license_accept:\n",
    "    print('By continuing, I already accepted the license listed above')\n",
    "else:\n",
    "    while True:\n",
    "        vs_license_accept = input('Do you accept the license listed above (Y/N)?')\n",
    "        if lower(vs_license_accept) == 'y': break\n",
    "        else: print('You must accept the license before continuing!')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save manifest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Microsoft.VisualStudio.Manifests.VisualStudio\n",
      "https://download.visualstudio.microsoft.com/download/pr/e514a25b-a89d-4051-a63c-05ccd9be41e9/5f11f64927ac42b2e9f4c4c943b309572ca7b2702770731e3a3180a9c7033daf/VisualStudio.vsman\n"
     ]
    }
   ],
   "source": [
    "from warnings import warn\n",
    "import hashlib\n",
    "\n",
    "def check_sha256sum(file_path: str, correct_sha256: str) -> bool:\n",
    "    \"\"\" Get sha256sum of a file and compare it with the correct sha256sum \"\"\"\n",
    "    sha = hashlib.sha256()\n",
    "    with open(file_path, 'rb') as f:\n",
    "        while True:\n",
    "            # Set buffer as 64 kB\n",
    "            # Prevent out-of-memory for large files\n",
    "            data = f.read(65536)\n",
    "            if not data: break\n",
    "            sha.update(data)\n",
    "    return lower(correct_sha256) == lower(sha.hexdigest())\n",
    "\n",
    "# Get manifest from channel info\n",
    "for i in channel_info['channelItems']:\n",
    "    # Get product id and manifest URL\n",
    "    if lower(i['type']) == lower('Manifest'):\n",
    "        print(i['id'])\n",
    "        # Normally only one payload exists, therefore index 0\n",
    "        print(i['payloads'][0]['url'])\n",
    "        if len(i['payloads']) > 1:\n",
    "            warn('Manifest has more than 1 payload!')\n",
    "        # Save the manifest URL\n",
    "        manifest = i['payloads'][0]['url']\n",
    "        manifest_sha256 = i['payloads'][0]['sha256']\n",
    "        break\n",
    "\n",
    "manifest_path = f'{data_path}\\\\manifest.json'\n",
    "manifest_valid = False\n",
    "\n",
    "if os.path.isfile(manifest_path):\n",
    "    if vs_verify_manifest:\n",
    "        manifest_valid = check_sha256sum(manifest_path, manifest_sha256)\n",
    "    else:\n",
    "        manifest_valid = True\n",
    "\n",
    "if manifest_valid:\n",
    "    # Load local manifest file\n",
    "    with open(manifest_path, encoding = \"utf8\") as file:\n",
    "        load = json.load(file)    \n",
    "else:\n",
    "    # Download manifest if necessary\n",
    "    manifest = requests.get(url = manifest)\n",
    "    if manifest.status_code == 200:\n",
    "        load = json.loads(manifest.content)\n",
    "        dump = json.dumps(load, indent = 4, ensure_ascii = False)\n",
    "        # Save the manifest as json\n",
    "        with open(f'{data_path}\\\\manifest.json', 'wb') as file:\n",
    "            file.write(dump.encode())\n",
    "    else:\n",
    "        raise ConnectionError('Failed to download manifest!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>package_type</th>\n",
       "      <th>packages_num</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Component</td>\n",
       "      <td>477</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Msi</td>\n",
       "      <td>1549</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Exe</td>\n",
       "      <td>112</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Zip</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Vsix</td>\n",
       "      <td>10816</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Msu</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Group</td>\n",
       "      <td>83</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>WindowsFeature</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Product</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Workload</td>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Nupkg</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      package_type  packages_num\n",
       "0        Component           477\n",
       "1              Msi          1549\n",
       "2              Exe           112\n",
       "3              Zip            21\n",
       "4             Vsix         10816\n",
       "5              Msu            14\n",
       "6            Group            83\n",
       "7   WindowsFeature             3\n",
       "8          Product            10\n",
       "9         Workload            36\n",
       "10           Nupkg             1"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "# Read the manifest again\n",
    "# Do not modify the manifest! (read-only)\n",
    "# Manifest will be used many times\n",
    "manifest = LowerDict(load)\n",
    "\n",
    "# Get all types of package (for research)\n",
    "pkg_types = LowerDict()\n",
    "for i in manifest['packages']:\n",
    "    if i[\"type\"] not in pkg_types: # Lower dict\n",
    "        pkg_types[i['type']] = 1\n",
    "    else:\n",
    "        pkg_types[i['type']] += 1\n",
    "\n",
    "df = pd.DataFrame.from_dict(\n",
    "    pkg_types,\n",
    "    orient = 'index',\n",
    "    columns = ['packages_num']\n",
    ").rename_axis('package_type').reset_index()\n",
    "\n",
    "df"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get workloads info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>title</th>\n",
       "      <th>version</th>\n",
       "      <th>description</th>\n",
       "      <th>category</th>\n",
       "      <th>dependenciesNum</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Microsoft.VisualStudio.Workload.Azure</td>\n",
       "      <td>Azure development</td>\n",
       "      <td>17.7.33905.399</td>\n",
       "      <td>Azure SDKs, tools, and projects for developing...</td>\n",
       "      <td>Web &amp; Cloud</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Microsoft.VisualStudio.Workload.AzureBuildTools</td>\n",
       "      <td>Azure development build tools</td>\n",
       "      <td>17.7.33905.399</td>\n",
       "      <td>MSBuild tasks and targets for building Azure a...</td>\n",
       "      <td>Web &amp; Cloud</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Microsoft.VisualStudio.Workload.CoreEditor</td>\n",
       "      <td>Visual Studio core editor</td>\n",
       "      <td>17.7.33905.399</td>\n",
       "      <td>The Visual Studio core shell experience, inclu...</td>\n",
       "      <td>None</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Microsoft.VisualStudio.Workload.Data</td>\n",
       "      <td>Data storage and processing</td>\n",
       "      <td>17.7.33905.399</td>\n",
       "      <td>Connect, develop, and test data solutions with...</td>\n",
       "      <td>Other Toolsets</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Microsoft.VisualStudio.Workload.DataBuildTools</td>\n",
       "      <td>Data storage and processing build tools</td>\n",
       "      <td>17.7.33905.399</td>\n",
       "      <td>Build SQL Server Database Projects</td>\n",
       "      <td>Web &amp; Cloud</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Microsoft.VisualStudio.Workload.DataScience</td>\n",
       "      <td>Data science and analytical applications</td>\n",
       "      <td>17.7.33905.399</td>\n",
       "      <td>Languages and tooling for creating data scienc...</td>\n",
       "      <td>Other Toolsets</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Microsoft.VisualStudio.Workload.ManagedDesktop</td>\n",
       "      <td>.NET desktop development</td>\n",
       "      <td>17.7.34002.345</td>\n",
       "      <td>Build WPF, Windows Forms, and console applicat...</td>\n",
       "      <td>Desktop &amp; Mobile</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Microsoft.VisualStudio.Workload.ManagedDesktop</td>\n",
       "      <td>.NET desktop development</td>\n",
       "      <td>17.7.34002.345</td>\n",
       "      <td>Build WPF, Windows Forms, and console applicat...</td>\n",
       "      <td>Desktop &amp; Mobile</td>\n",
       "      <td>26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Microsoft.VisualStudio.Workload.ManagedDesktop...</td>\n",
       "      <td>.NET desktop build tools</td>\n",
       "      <td>17.7.33905.399</td>\n",
       "      <td>Tools for building WPF, Windows Forms, and con...</td>\n",
       "      <td>Desktop &amp; Mobile</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Microsoft.VisualStudio.Workload.ManagedGame</td>\n",
       "      <td>Game development with Unity</td>\n",
       "      <td>17.7.33905.399</td>\n",
       "      <td>Create 2D and 3D games with Unity, a powerful ...</td>\n",
       "      <td>Gaming</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Microsoft.VisualStudio.Workload.MSBuildTools</td>\n",
       "      <td>MSBuild Tools</td>\n",
       "      <td>17.7.33905.399</td>\n",
       "      <td>Provides the tools required to build MSBuild-b...</td>\n",
       "      <td>None</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Microsoft.VisualStudio.Workload.NativeCrossPlat</td>\n",
       "      <td>Linux and embedded development with C++</td>\n",
       "      <td>17.7.33905.399</td>\n",
       "      <td>Create and debug applications running in a Lin...</td>\n",
       "      <td>Other Toolsets</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Microsoft.VisualStudio.Workload.NativeCrossPlat</td>\n",
       "      <td>Linux and embedded development with C++</td>\n",
       "      <td>17.7.33905.399</td>\n",
       "      <td>Create and debug applications running in a Lin...</td>\n",
       "      <td>Other Toolsets</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Microsoft.VisualStudio.Workload.NativeDesktop</td>\n",
       "      <td>Desktop development with C++</td>\n",
       "      <td>17.7.33905.399</td>\n",
       "      <td>Build modern C++ apps for Windows using tools ...</td>\n",
       "      <td>Desktop &amp; Mobile</td>\n",
       "      <td>35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Microsoft.VisualStudio.Workload.NativeDesktop</td>\n",
       "      <td>Desktop development with C++</td>\n",
       "      <td>17.7.33905.399</td>\n",
       "      <td>Build modern C++ apps for Windows using tools ...</td>\n",
       "      <td>Desktop &amp; Mobile</td>\n",
       "      <td>31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Microsoft.VisualStudio.Workload.NativeGame</td>\n",
       "      <td>Game development with C++</td>\n",
       "      <td>17.7.33905.399</td>\n",
       "      <td>Use the full power of C++ to build professiona...</td>\n",
       "      <td>Gaming</td>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Microsoft.VisualStudio.Workload.NativeGame</td>\n",
       "      <td>Game development with C++</td>\n",
       "      <td>17.7.33905.399</td>\n",
       "      <td>Use the full power of C++ to build professiona...</td>\n",
       "      <td>Gaming</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Microsoft.VisualStudio.Workload.NativeMobile</td>\n",
       "      <td>Mobile development with C++</td>\n",
       "      <td>17.7.33905.399</td>\n",
       "      <td>Build cross-platform applications for iOS, And...</td>\n",
       "      <td>Desktop &amp; Mobile</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Microsoft.VisualStudio.Workload.NetCrossPlat</td>\n",
       "      <td>.NET Multi-platform App UI development</td>\n",
       "      <td>17.7.34009.333</td>\n",
       "      <td>Build Android, iOS, Windows, and Mac apps from...</td>\n",
       "      <td>Desktop &amp; Mobile</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Microsoft.VisualStudio.Workload.NetWeb</td>\n",
       "      <td>ASP.NET and web development</td>\n",
       "      <td>17.7.34002.345</td>\n",
       "      <td>Build web applications using ASP.NET Core, ASP...</td>\n",
       "      <td>Web &amp; Cloud</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Microsoft.VisualStudio.Workload.NetWeb</td>\n",
       "      <td>ASP.NET and web development</td>\n",
       "      <td>17.7.34002.345</td>\n",
       "      <td>Build web applications using ASP.NET Core, ASP...</td>\n",
       "      <td>Web &amp; Cloud</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Microsoft.VisualStudio.Workload.Node</td>\n",
       "      <td>Node.js development</td>\n",
       "      <td>17.7.33905.399</td>\n",
       "      <td>Build scalable network applications using Node...</td>\n",
       "      <td>Web &amp; Cloud</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Microsoft.VisualStudio.Workload.NodeBuildTools</td>\n",
       "      <td>Node.js build tools</td>\n",
       "      <td>17.7.33905.399</td>\n",
       "      <td>MSBuild tasks and targets for building scalabl...</td>\n",
       "      <td>None</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Microsoft.VisualStudio.Workload.Office</td>\n",
       "      <td>Office/SharePoint development</td>\n",
       "      <td>17.7.33905.399</td>\n",
       "      <td>Create Office and SharePoint add-ins, SharePoi...</td>\n",
       "      <td>Other Toolsets</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>Microsoft.VisualStudio.Workload.OfficeBuildTools</td>\n",
       "      <td>Office/SharePoint build tools</td>\n",
       "      <td>17.7.33905.399</td>\n",
       "      <td>Build Office and SharePoint add-ins, and VSTO ...</td>\n",
       "      <td>Web &amp; Cloud</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>Microsoft.VisualStudio.Workload.Python</td>\n",
       "      <td>Python development</td>\n",
       "      <td>17.7.33905.399</td>\n",
       "      <td>Editing, debugging, interactive development an...</td>\n",
       "      <td>Web &amp; Cloud</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>Microsoft.VisualStudio.Workload.TestAgent</td>\n",
       "      <td>Test Agent</td>\n",
       "      <td>17.7.33905.399</td>\n",
       "      <td>Supports running automated tests and load test...</td>\n",
       "      <td>Debugging and testing</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>Microsoft.VisualStudio.Workload.TestController</td>\n",
       "      <td>Test Controller</td>\n",
       "      <td>17.7.33905.399</td>\n",
       "      <td>Distribute automated tests to multiple machines</td>\n",
       "      <td>Debugging and testing</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>Microsoft.VisualStudio.Workload.Universal</td>\n",
       "      <td>Universal Windows Platform development</td>\n",
       "      <td>17.7.33905.399</td>\n",
       "      <td>Create applications for the Universal Windows ...</td>\n",
       "      <td>Desktop &amp; Mobile</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>Microsoft.VisualStudio.Workload.Universal</td>\n",
       "      <td>Universal Windows Platform development</td>\n",
       "      <td>17.7.33905.399</td>\n",
       "      <td>Create applications for the Universal Windows ...</td>\n",
       "      <td>Desktop &amp; Mobile</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>Microsoft.VisualStudio.Workload.UniversalBuild...</td>\n",
       "      <td>Universal Windows Platform build tools</td>\n",
       "      <td>17.7.33905.399</td>\n",
       "      <td>Provides the tools required to build Universal...</td>\n",
       "      <td>Desktop &amp; Mobile</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>Microsoft.VisualStudio.Workload.VCTools</td>\n",
       "      <td>Desktop development with C++</td>\n",
       "      <td>17.7.33905.399</td>\n",
       "      <td>Build modern C++ apps for Windows using tools ...</td>\n",
       "      <td>Desktop &amp; Mobile</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>Microsoft.VisualStudio.Workload.VisualStudioEx...</td>\n",
       "      <td>Visual Studio extension development</td>\n",
       "      <td>17.7.33905.399</td>\n",
       "      <td>Create add-ons and extensions for Visual Studi...</td>\n",
       "      <td>Other Toolsets</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>Microsoft.VisualStudio.Workload.VisualStudioEx...</td>\n",
       "      <td>Visual Studio extension development</td>\n",
       "      <td>17.7.33905.399</td>\n",
       "      <td>Tools for building add-ons and extensions for ...</td>\n",
       "      <td>Other Toolsets</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>Microsoft.VisualStudio.Workload.WebBuildTools</td>\n",
       "      <td>Web development build tools</td>\n",
       "      <td>17.7.34002.345</td>\n",
       "      <td>MSBuild tasks and targets for building web app...</td>\n",
       "      <td>Web &amp; Cloud</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>Microsoft.VisualStudio.Workload.XamarinBuildTools</td>\n",
       "      <td>.NET Multi-platform App UI development</td>\n",
       "      <td>17.7.33905.399</td>\n",
       "      <td>Build Android, iOS, Windows, and Mac apps from...</td>\n",
       "      <td>Desktop &amp; Mobile</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   id  \\\n",
       "0               Microsoft.VisualStudio.Workload.Azure   \n",
       "1     Microsoft.VisualStudio.Workload.AzureBuildTools   \n",
       "2          Microsoft.VisualStudio.Workload.CoreEditor   \n",
       "3                Microsoft.VisualStudio.Workload.Data   \n",
       "4      Microsoft.VisualStudio.Workload.DataBuildTools   \n",
       "5         Microsoft.VisualStudio.Workload.DataScience   \n",
       "6      Microsoft.VisualStudio.Workload.ManagedDesktop   \n",
       "7      Microsoft.VisualStudio.Workload.ManagedDesktop   \n",
       "8   Microsoft.VisualStudio.Workload.ManagedDesktop...   \n",
       "9         Microsoft.VisualStudio.Workload.ManagedGame   \n",
       "10       Microsoft.VisualStudio.Workload.MSBuildTools   \n",
       "11    Microsoft.VisualStudio.Workload.NativeCrossPlat   \n",
       "12    Microsoft.VisualStudio.Workload.NativeCrossPlat   \n",
       "13      Microsoft.VisualStudio.Workload.NativeDesktop   \n",
       "14      Microsoft.VisualStudio.Workload.NativeDesktop   \n",
       "15         Microsoft.VisualStudio.Workload.NativeGame   \n",
       "16         Microsoft.VisualStudio.Workload.NativeGame   \n",
       "17       Microsoft.VisualStudio.Workload.NativeMobile   \n",
       "18       Microsoft.VisualStudio.Workload.NetCrossPlat   \n",
       "19             Microsoft.VisualStudio.Workload.NetWeb   \n",
       "20             Microsoft.VisualStudio.Workload.NetWeb   \n",
       "21               Microsoft.VisualStudio.Workload.Node   \n",
       "22     Microsoft.VisualStudio.Workload.NodeBuildTools   \n",
       "23             Microsoft.VisualStudio.Workload.Office   \n",
       "24   Microsoft.VisualStudio.Workload.OfficeBuildTools   \n",
       "25             Microsoft.VisualStudio.Workload.Python   \n",
       "26          Microsoft.VisualStudio.Workload.TestAgent   \n",
       "27     Microsoft.VisualStudio.Workload.TestController   \n",
       "28          Microsoft.VisualStudio.Workload.Universal   \n",
       "29          Microsoft.VisualStudio.Workload.Universal   \n",
       "30  Microsoft.VisualStudio.Workload.UniversalBuild...   \n",
       "31            Microsoft.VisualStudio.Workload.VCTools   \n",
       "32  Microsoft.VisualStudio.Workload.VisualStudioEx...   \n",
       "33  Microsoft.VisualStudio.Workload.VisualStudioEx...   \n",
       "34      Microsoft.VisualStudio.Workload.WebBuildTools   \n",
       "35  Microsoft.VisualStudio.Workload.XamarinBuildTools   \n",
       "\n",
       "                                       title         version  \\\n",
       "0                          Azure development  17.7.33905.399   \n",
       "1              Azure development build tools  17.7.33905.399   \n",
       "2                  Visual Studio core editor  17.7.33905.399   \n",
       "3                Data storage and processing  17.7.33905.399   \n",
       "4    Data storage and processing build tools  17.7.33905.399   \n",
       "5   Data science and analytical applications  17.7.33905.399   \n",
       "6                   .NET desktop development  17.7.34002.345   \n",
       "7                   .NET desktop development  17.7.34002.345   \n",
       "8                   .NET desktop build tools  17.7.33905.399   \n",
       "9                Game development with Unity  17.7.33905.399   \n",
       "10                             MSBuild Tools  17.7.33905.399   \n",
       "11   Linux and embedded development with C++  17.7.33905.399   \n",
       "12   Linux and embedded development with C++  17.7.33905.399   \n",
       "13              Desktop development with C++  17.7.33905.399   \n",
       "14              Desktop development with C++  17.7.33905.399   \n",
       "15                 Game development with C++  17.7.33905.399   \n",
       "16                 Game development with C++  17.7.33905.399   \n",
       "17               Mobile development with C++  17.7.33905.399   \n",
       "18    .NET Multi-platform App UI development  17.7.34009.333   \n",
       "19               ASP.NET and web development  17.7.34002.345   \n",
       "20               ASP.NET and web development  17.7.34002.345   \n",
       "21                       Node.js development  17.7.33905.399   \n",
       "22                       Node.js build tools  17.7.33905.399   \n",
       "23             Office/SharePoint development  17.7.33905.399   \n",
       "24             Office/SharePoint build tools  17.7.33905.399   \n",
       "25                        Python development  17.7.33905.399   \n",
       "26                                Test Agent  17.7.33905.399   \n",
       "27                           Test Controller  17.7.33905.399   \n",
       "28    Universal Windows Platform development  17.7.33905.399   \n",
       "29    Universal Windows Platform development  17.7.33905.399   \n",
       "30    Universal Windows Platform build tools  17.7.33905.399   \n",
       "31              Desktop development with C++  17.7.33905.399   \n",
       "32       Visual Studio extension development  17.7.33905.399   \n",
       "33       Visual Studio extension development  17.7.33905.399   \n",
       "34               Web development build tools  17.7.34002.345   \n",
       "35    .NET Multi-platform App UI development  17.7.33905.399   \n",
       "\n",
       "                                          description               category  \\\n",
       "0   Azure SDKs, tools, and projects for developing...            Web & Cloud   \n",
       "1   MSBuild tasks and targets for building Azure a...            Web & Cloud   \n",
       "2   The Visual Studio core shell experience, inclu...                   None   \n",
       "3   Connect, develop, and test data solutions with...         Other Toolsets   \n",
       "4                  Build SQL Server Database Projects            Web & Cloud   \n",
       "5   Languages and tooling for creating data scienc...         Other Toolsets   \n",
       "6   Build WPF, Windows Forms, and console applicat...       Desktop & Mobile   \n",
       "7   Build WPF, Windows Forms, and console applicat...       Desktop & Mobile   \n",
       "8   Tools for building WPF, Windows Forms, and con...       Desktop & Mobile   \n",
       "9   Create 2D and 3D games with Unity, a powerful ...                 Gaming   \n",
       "10  Provides the tools required to build MSBuild-b...                   None   \n",
       "11  Create and debug applications running in a Lin...         Other Toolsets   \n",
       "12  Create and debug applications running in a Lin...         Other Toolsets   \n",
       "13  Build modern C++ apps for Windows using tools ...       Desktop & Mobile   \n",
       "14  Build modern C++ apps for Windows using tools ...       Desktop & Mobile   \n",
       "15  Use the full power of C++ to build professiona...                 Gaming   \n",
       "16  Use the full power of C++ to build professiona...                 Gaming   \n",
       "17  Build cross-platform applications for iOS, And...       Desktop & Mobile   \n",
       "18  Build Android, iOS, Windows, and Mac apps from...       Desktop & Mobile   \n",
       "19  Build web applications using ASP.NET Core, ASP...            Web & Cloud   \n",
       "20  Build web applications using ASP.NET Core, ASP...            Web & Cloud   \n",
       "21  Build scalable network applications using Node...            Web & Cloud   \n",
       "22  MSBuild tasks and targets for building scalabl...                   None   \n",
       "23  Create Office and SharePoint add-ins, SharePoi...         Other Toolsets   \n",
       "24  Build Office and SharePoint add-ins, and VSTO ...            Web & Cloud   \n",
       "25  Editing, debugging, interactive development an...            Web & Cloud   \n",
       "26  Supports running automated tests and load test...  Debugging and testing   \n",
       "27    Distribute automated tests to multiple machines  Debugging and testing   \n",
       "28  Create applications for the Universal Windows ...       Desktop & Mobile   \n",
       "29  Create applications for the Universal Windows ...       Desktop & Mobile   \n",
       "30  Provides the tools required to build Universal...       Desktop & Mobile   \n",
       "31  Build modern C++ apps for Windows using tools ...       Desktop & Mobile   \n",
       "32  Create add-ons and extensions for Visual Studi...         Other Toolsets   \n",
       "33  Tools for building add-ons and extensions for ...         Other Toolsets   \n",
       "34  MSBuild tasks and targets for building web app...            Web & Cloud   \n",
       "35  Build Android, iOS, Windows, and Mac apps from...       Desktop & Mobile   \n",
       "\n",
       "   dependenciesNum  \n",
       "0               18  \n",
       "1               15  \n",
       "2                1  \n",
       "3                4  \n",
       "4                3  \n",
       "5                4  \n",
       "6               24  \n",
       "7               26  \n",
       "8               12  \n",
       "9                5  \n",
       "10               3  \n",
       "11               6  \n",
       "12               8  \n",
       "13              35  \n",
       "14              31  \n",
       "15              25  \n",
       "16              24  \n",
       "17              11  \n",
       "18              17  \n",
       "19              23  \n",
       "20              24  \n",
       "21              11  \n",
       "22               2  \n",
       "23              13  \n",
       "24               7  \n",
       "25               7  \n",
       "26               2  \n",
       "27               2  \n",
       "28              16  \n",
       "29              17  \n",
       "30               9  \n",
       "31              21  \n",
       "32               9  \n",
       "33               7  \n",
       "34              18  \n",
       "35              17  "
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get how many workloads (specializations) are there\n",
    "num = pkg_types['Workload']\n",
    "\n",
    "# Initialize empty dataframe\n",
    "df = pd.DataFrame(\n",
    "    columns = ['id', 'title', 'version', 'description', 'category', 'dependenciesNum'],\n",
    "    index = range(num)\n",
    ")\n",
    "\n",
    "# Initialize workload as a dict list (workload id as key)\n",
    "# Each workload is derived from manifest \"packages\"\n",
    "workload = LowerDict()\n",
    "\n",
    "num = -1\n",
    "for i in manifest['packages']:\n",
    "    if lower(i['type']) == lower('Workload'):\n",
    "        # Save workload props (values) to \"workload\" variable\n",
    "        workload[i['id']] = i\n",
    "\n",
    "        num += 1\n",
    "        df.loc[num, 'id'] = i.get('id')\n",
    "        df.loc[num, 'version'] = i.get('version')\n",
    "        # How many dependencies that the workload have?\n",
    "        if i.get('dependencies'):\n",
    "            df.loc[num, 'dependenciesNum'] = len(i['dependencies'])\n",
    "        else: df.loc[num, 'dependenciesNum'] = 0\n",
    "        # Get descriptive info for each workload\n",
    "        if i.get('localizedResources'):\n",
    "            for j in i['localizedResources']:\n",
    "                # \"localizedResources\" doesn't have \"neutral\" language\n",
    "                # Prioritize preferred language(s) based on the index\n",
    "                for k in vs_lang:\n",
    "                    if lower(j['language']) == lower(k):\n",
    "                        df.loc[num, 'title'] = j.get('title')\n",
    "                        df.loc[num, 'description'] = j.get('description')\n",
    "                        # Category may not always exist\n",
    "                        df.loc[num, 'category'] = j.get('category')\n",
    "                        break\n",
    "\n",
    "# Save summarized info of workloads\n",
    "df.to_json(\n",
    "    f'{data_path}\\\\workload_summary.json',\n",
    "    indent = 4,\n",
    "    force_ascii = False,\n",
    "    orient = 'records'\n",
    ")\n",
    "\n",
    "# Save detailed info of workloads\n",
    "# Convert LowerDict to dict first before exporting\n",
    "dump = json.dumps(dict(workload), indent = 4, ensure_ascii = False)\n",
    "with open(f'{data_path}\\\\workload_details.json', 'wb') as file:\n",
    "    file.write(dump.encode())\n",
    "\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(workload)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Filter workloads and get main dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# List of chosen workload id(s)\n",
    "# Change it based on your needs\n",
    "wl_needed = [\n",
    "    # I think \"MSBuildTools\" is required for any build tools\n",
    "    'Microsoft.VisualStudio.Workload.MSBuildTools',\n",
    "    # Desktop development with C++ (build tools)\n",
    "    'Microsoft.VisualStudio.Workload.VCTools'\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>parentId</th>\n",
       "      <th>title</th>\n",
       "      <th>description</th>\n",
       "      <th>category</th>\n",
       "      <th>dependenciesNum</th>\n",
       "      <th>flag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Microsoft.Component.MSBuild</td>\n",
       "      <td>Microsoft.VisualStudio.Workload.MSBuildTools</td>\n",
       "      <td>MSBuild</td>\n",
       "      <td>MSBuild tasks and supported C#/VB compilers.</td>\n",
       "      <td>Compilers, build tools, and runtimes</td>\n",
       "      <td>5</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Microsoft.Build.Dependencies</td>\n",
       "      <td>Microsoft.VisualStudio.Workload.MSBuildTools</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Microsoft.VisualStudio.Component.CoreBuildTools</td>\n",
       "      <td>Microsoft.VisualStudio.Workload.MSBuildTools</td>\n",
       "      <td>Visual Studio Build Tools Core</td>\n",
       "      <td>Core features needed for supporting Visual Stu...</td>\n",
       "      <td>None</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Microsoft.VisualStudio.Component.VC.CoreBuildT...</td>\n",
       "      <td>Microsoft.VisualStudio.Workload.VCTools</td>\n",
       "      <td>C++ Build Tools core features</td>\n",
       "      <td>Core features needed for supporting Visual C++...</td>\n",
       "      <td>Development activities</td>\n",
       "      <td>8</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Microsoft.VisualStudio.Component.VC.Tools.x86.x64</td>\n",
       "      <td>Microsoft.VisualStudio.Workload.VCTools</td>\n",
       "      <td>MSVC v143 - VS 2022 C++ x64/x86 build tools (L...</td>\n",
       "      <td>Visual C++ (v143) latest compilers and librari...</td>\n",
       "      <td>Compilers, build tools, and runtimes</td>\n",
       "      <td>35</td>\n",
       "      <td>Recommended</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Microsoft.VisualStudio.Component.VC.Redist.14....</td>\n",
       "      <td>Microsoft.VisualStudio.Workload.VCTools</td>\n",
       "      <td>C++ 2022 Redistributable Update</td>\n",
       "      <td>Latest update for the Visual C++ Redistributable</td>\n",
       "      <td>Compilers, build tools, and runtimes</td>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Microsoft.VisualStudio.VC.UnitTest.Desktop.Bui...</td>\n",
       "      <td>Microsoft.VisualStudio.Workload.VCTools</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Microsoft.VisualStudio.Component.Windows11SDK....</td>\n",
       "      <td>Microsoft.VisualStudio.Workload.VCTools</td>\n",
       "      <td>Windows 11 SDK (10.0.22621.0)</td>\n",
       "      <td>Headers, libraries, and tools for developing a...</td>\n",
       "      <td>SDKs, libraries, and frameworks</td>\n",
       "      <td>1</td>\n",
       "      <td>Recommended</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Microsoft.VisualStudio.Component.VC.CMake.Project</td>\n",
       "      <td>Microsoft.VisualStudio.Workload.VCTools</td>\n",
       "      <td>C++ CMake tools for Windows</td>\n",
       "      <td>Visual C++ tools support for CMake.</td>\n",
       "      <td>Compilers, build tools, and runtimes</td>\n",
       "      <td>6</td>\n",
       "      <td>Recommended</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Microsoft.VisualStudio.Component.TestTools.Bui...</td>\n",
       "      <td>Microsoft.VisualStudio.Workload.VCTools</td>\n",
       "      <td>Testing tools core features - Build Tools</td>\n",
       "      <td>Includes core test tools to run tests from CLI.</td>\n",
       "      <td>Debugging and testing</td>\n",
       "      <td>1</td>\n",
       "      <td>Recommended</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Microsoft.VisualStudio.Component.VC.ATL</td>\n",
       "      <td>Microsoft.VisualStudio.Workload.VCTools</td>\n",
       "      <td>C++ ATL for latest v143 build tools (x86 &amp; x64)</td>\n",
       "      <td>Headers and libraries for the development of A...</td>\n",
       "      <td>SDKs, libraries, and frameworks</td>\n",
       "      <td>5</td>\n",
       "      <td>Optional</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Microsoft.VisualStudio.Component.VC.ATLMFC</td>\n",
       "      <td>Microsoft.VisualStudio.Workload.VCTools</td>\n",
       "      <td>C++ MFC for latest v143 build tools (x86 &amp; x64)</td>\n",
       "      <td>Includes headers and libraries for the develop...</td>\n",
       "      <td>SDKs, libraries, and frameworks</td>\n",
       "      <td>11</td>\n",
       "      <td>Optional</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Microsoft.VisualStudio.Component.VC.CLI.Support</td>\n",
       "      <td>Microsoft.VisualStudio.Workload.VCTools</td>\n",
       "      <td>C++/CLI support for v143 build tools (Latest)</td>\n",
       "      <td>Required tools and frameworks for code editing...</td>\n",
       "      <td>Compilers, build tools, and runtimes</td>\n",
       "      <td>8</td>\n",
       "      <td>Optional</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Microsoft.VisualStudio.Component.VC.ASAN</td>\n",
       "      <td>Microsoft.VisualStudio.Workload.VCTools</td>\n",
       "      <td>C++ AddressSanitizer</td>\n",
       "      <td>AddressSanitizer (ASAN) is a tool for detectin...</td>\n",
       "      <td>Debugging and testing</td>\n",
       "      <td>1</td>\n",
       "      <td>Recommended</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Microsoft.VisualStudio.Component.VC.Modules.x8...</td>\n",
       "      <td>Microsoft.VisualStudio.Workload.VCTools</td>\n",
       "      <td>C++ Modules for v143 build tools (x64/x86 – ex...</td>\n",
       "      <td>C++ Module interfaces for the standard librari...</td>\n",
       "      <td>Compilers, build tools, and runtimes</td>\n",
       "      <td>2</td>\n",
       "      <td>Optional</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Microsoft.VisualStudio.ComponentGroup.NativeDe...</td>\n",
       "      <td>Microsoft.VisualStudio.Workload.VCTools</td>\n",
       "      <td>C++ core desktop features</td>\n",
       "      <td>Core IDE features needed for Visual C++ deskto...</td>\n",
       "      <td>Development activities</td>\n",
       "      <td>7</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Microsoft.VisualStudio.ComponentGroup.NativeDe...</td>\n",
       "      <td>Microsoft.VisualStudio.Workload.VCTools</td>\n",
       "      <td>C++ Clang tools for Windows (16.0.5 - x64/x86)</td>\n",
       "      <td>Platform toolset, Clang compiler, and tools ne...</td>\n",
       "      <td>Compilers, build tools, and runtimes</td>\n",
       "      <td>2</td>\n",
       "      <td>Optional</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Microsoft.VisualStudio.Component.Windows11SDK....</td>\n",
       "      <td>Microsoft.VisualStudio.Workload.VCTools</td>\n",
       "      <td>Windows 11 SDK (10.0.22000.0)</td>\n",
       "      <td>Headers, libraries, and tools for developing a...</td>\n",
       "      <td>SDKs, libraries, and frameworks</td>\n",
       "      <td>1</td>\n",
       "      <td>Optional</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Microsoft.VisualStudio.Component.Windows10SDK....</td>\n",
       "      <td>Microsoft.VisualStudio.Workload.VCTools</td>\n",
       "      <td>Windows 10 SDK (10.0.20348.0)</td>\n",
       "      <td>Headers, libraries, and tools for developing a...</td>\n",
       "      <td>SDKs, libraries, and frameworks</td>\n",
       "      <td>1</td>\n",
       "      <td>Optional</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Microsoft.VisualStudio.Component.Windows10SDK....</td>\n",
       "      <td>Microsoft.VisualStudio.Workload.VCTools</td>\n",
       "      <td>Windows 10 SDK (10.0.19041.0)</td>\n",
       "      <td>Headers, libraries, and tools for developing a...</td>\n",
       "      <td>SDKs, libraries, and frameworks</td>\n",
       "      <td>1</td>\n",
       "      <td>Optional</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Microsoft.VisualStudio.Component.Windows10SDK....</td>\n",
       "      <td>Microsoft.VisualStudio.Workload.VCTools</td>\n",
       "      <td>Windows 10 SDK (10.0.18362.0)</td>\n",
       "      <td>Headers, libraries, and tools for developing a...</td>\n",
       "      <td>SDKs, libraries, and frameworks</td>\n",
       "      <td>1</td>\n",
       "      <td>Optional</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Microsoft.VisualStudio.ComponentGroup.VC.Tools...</td>\n",
       "      <td>Microsoft.VisualStudio.Workload.VCTools</td>\n",
       "      <td>MSVC v142 - VS 2019 C++ x64/x86 build tools (v...</td>\n",
       "      <td>Visual C++ (v142) compilers and libraries for ...</td>\n",
       "      <td>Compilers, build tools, and runtimes</td>\n",
       "      <td>39</td>\n",
       "      <td>Optional</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Microsoft.VisualStudio.Component.VC.v141.x86.x64</td>\n",
       "      <td>Microsoft.VisualStudio.Workload.VCTools</td>\n",
       "      <td>MSVC v141 - VS 2017 C++ x64/x86 build tools (v...</td>\n",
       "      <td>Visual Studio 15.9 Visual C++ v14.16 (v141) co...</td>\n",
       "      <td>Compilers, build tools, and runtimes</td>\n",
       "      <td>36</td>\n",
       "      <td>Optional</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Microsoft.VisualStudio.Component.VC.140</td>\n",
       "      <td>Microsoft.VisualStudio.Workload.VCTools</td>\n",
       "      <td>MSVC v140 - VS 2015 C++ build tools (v14.00)</td>\n",
       "      <td>Visual Studio 2015 C++ compilers and libraries...</td>\n",
       "      <td>Compilers, build tools, and runtimes</td>\n",
       "      <td>76</td>\n",
       "      <td>Optional</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   id  \\\n",
       "0                         Microsoft.Component.MSBuild   \n",
       "1                        Microsoft.Build.Dependencies   \n",
       "2     Microsoft.VisualStudio.Component.CoreBuildTools   \n",
       "3   Microsoft.VisualStudio.Component.VC.CoreBuildT...   \n",
       "4   Microsoft.VisualStudio.Component.VC.Tools.x86.x64   \n",
       "5   Microsoft.VisualStudio.Component.VC.Redist.14....   \n",
       "6   Microsoft.VisualStudio.VC.UnitTest.Desktop.Bui...   \n",
       "7   Microsoft.VisualStudio.Component.Windows11SDK....   \n",
       "8   Microsoft.VisualStudio.Component.VC.CMake.Project   \n",
       "9   Microsoft.VisualStudio.Component.TestTools.Bui...   \n",
       "10            Microsoft.VisualStudio.Component.VC.ATL   \n",
       "11         Microsoft.VisualStudio.Component.VC.ATLMFC   \n",
       "12    Microsoft.VisualStudio.Component.VC.CLI.Support   \n",
       "13           Microsoft.VisualStudio.Component.VC.ASAN   \n",
       "14  Microsoft.VisualStudio.Component.VC.Modules.x8...   \n",
       "15  Microsoft.VisualStudio.ComponentGroup.NativeDe...   \n",
       "16  Microsoft.VisualStudio.ComponentGroup.NativeDe...   \n",
       "17  Microsoft.VisualStudio.Component.Windows11SDK....   \n",
       "18  Microsoft.VisualStudio.Component.Windows10SDK....   \n",
       "19  Microsoft.VisualStudio.Component.Windows10SDK....   \n",
       "20  Microsoft.VisualStudio.Component.Windows10SDK....   \n",
       "21  Microsoft.VisualStudio.ComponentGroup.VC.Tools...   \n",
       "22   Microsoft.VisualStudio.Component.VC.v141.x86.x64   \n",
       "23            Microsoft.VisualStudio.Component.VC.140   \n",
       "\n",
       "                                        parentId  \\\n",
       "0   Microsoft.VisualStudio.Workload.MSBuildTools   \n",
       "1   Microsoft.VisualStudio.Workload.MSBuildTools   \n",
       "2   Microsoft.VisualStudio.Workload.MSBuildTools   \n",
       "3        Microsoft.VisualStudio.Workload.VCTools   \n",
       "4        Microsoft.VisualStudio.Workload.VCTools   \n",
       "5        Microsoft.VisualStudio.Workload.VCTools   \n",
       "6        Microsoft.VisualStudio.Workload.VCTools   \n",
       "7        Microsoft.VisualStudio.Workload.VCTools   \n",
       "8        Microsoft.VisualStudio.Workload.VCTools   \n",
       "9        Microsoft.VisualStudio.Workload.VCTools   \n",
       "10       Microsoft.VisualStudio.Workload.VCTools   \n",
       "11       Microsoft.VisualStudio.Workload.VCTools   \n",
       "12       Microsoft.VisualStudio.Workload.VCTools   \n",
       "13       Microsoft.VisualStudio.Workload.VCTools   \n",
       "14       Microsoft.VisualStudio.Workload.VCTools   \n",
       "15       Microsoft.VisualStudio.Workload.VCTools   \n",
       "16       Microsoft.VisualStudio.Workload.VCTools   \n",
       "17       Microsoft.VisualStudio.Workload.VCTools   \n",
       "18       Microsoft.VisualStudio.Workload.VCTools   \n",
       "19       Microsoft.VisualStudio.Workload.VCTools   \n",
       "20       Microsoft.VisualStudio.Workload.VCTools   \n",
       "21       Microsoft.VisualStudio.Workload.VCTools   \n",
       "22       Microsoft.VisualStudio.Workload.VCTools   \n",
       "23       Microsoft.VisualStudio.Workload.VCTools   \n",
       "\n",
       "                                                title  \\\n",
       "0                                             MSBuild   \n",
       "1                                                 NaN   \n",
       "2                      Visual Studio Build Tools Core   \n",
       "3                       C++ Build Tools core features   \n",
       "4   MSVC v143 - VS 2022 C++ x64/x86 build tools (L...   \n",
       "5                     C++ 2022 Redistributable Update   \n",
       "6                                                 NaN   \n",
       "7                       Windows 11 SDK (10.0.22621.0)   \n",
       "8                         C++ CMake tools for Windows   \n",
       "9           Testing tools core features - Build Tools   \n",
       "10    C++ ATL for latest v143 build tools (x86 & x64)   \n",
       "11    C++ MFC for latest v143 build tools (x86 & x64)   \n",
       "12      C++/CLI support for v143 build tools (Latest)   \n",
       "13                               C++ AddressSanitizer   \n",
       "14  C++ Modules for v143 build tools (x64/x86 – ex...   \n",
       "15                          C++ core desktop features   \n",
       "16     C++ Clang tools for Windows (16.0.5 - x64/x86)   \n",
       "17                      Windows 11 SDK (10.0.22000.0)   \n",
       "18                      Windows 10 SDK (10.0.20348.0)   \n",
       "19                      Windows 10 SDK (10.0.19041.0)   \n",
       "20                      Windows 10 SDK (10.0.18362.0)   \n",
       "21  MSVC v142 - VS 2019 C++ x64/x86 build tools (v...   \n",
       "22  MSVC v141 - VS 2017 C++ x64/x86 build tools (v...   \n",
       "23       MSVC v140 - VS 2015 C++ build tools (v14.00)   \n",
       "\n",
       "                                          description  \\\n",
       "0        MSBuild tasks and supported C#/VB compilers.   \n",
       "1                                                 NaN   \n",
       "2   Core features needed for supporting Visual Stu...   \n",
       "3   Core features needed for supporting Visual C++...   \n",
       "4   Visual C++ (v143) latest compilers and librari...   \n",
       "5    Latest update for the Visual C++ Redistributable   \n",
       "6                                                 NaN   \n",
       "7   Headers, libraries, and tools for developing a...   \n",
       "8                 Visual C++ tools support for CMake.   \n",
       "9     Includes core test tools to run tests from CLI.   \n",
       "10  Headers and libraries for the development of A...   \n",
       "11  Includes headers and libraries for the develop...   \n",
       "12  Required tools and frameworks for code editing...   \n",
       "13  AddressSanitizer (ASAN) is a tool for detectin...   \n",
       "14  C++ Module interfaces for the standard librari...   \n",
       "15  Core IDE features needed for Visual C++ deskto...   \n",
       "16  Platform toolset, Clang compiler, and tools ne...   \n",
       "17  Headers, libraries, and tools for developing a...   \n",
       "18  Headers, libraries, and tools for developing a...   \n",
       "19  Headers, libraries, and tools for developing a...   \n",
       "20  Headers, libraries, and tools for developing a...   \n",
       "21  Visual C++ (v142) compilers and libraries for ...   \n",
       "22  Visual Studio 15.9 Visual C++ v14.16 (v141) co...   \n",
       "23  Visual Studio 2015 C++ compilers and libraries...   \n",
       "\n",
       "                                category  dependenciesNum         flag  \n",
       "0   Compilers, build tools, and runtimes                5         None  \n",
       "1                                    NaN                1         None  \n",
       "2                                   None                3         None  \n",
       "3                 Development activities                8         None  \n",
       "4   Compilers, build tools, and runtimes               35  Recommended  \n",
       "5   Compilers, build tools, and runtimes                1         None  \n",
       "6                                    NaN                1         None  \n",
       "7        SDKs, libraries, and frameworks                1  Recommended  \n",
       "8   Compilers, build tools, and runtimes                6  Recommended  \n",
       "9                  Debugging and testing                1  Recommended  \n",
       "10       SDKs, libraries, and frameworks                5     Optional  \n",
       "11       SDKs, libraries, and frameworks               11     Optional  \n",
       "12  Compilers, build tools, and runtimes                8     Optional  \n",
       "13                 Debugging and testing                1  Recommended  \n",
       "14  Compilers, build tools, and runtimes                2     Optional  \n",
       "15                Development activities                7         None  \n",
       "16  Compilers, build tools, and runtimes                2     Optional  \n",
       "17       SDKs, libraries, and frameworks                1     Optional  \n",
       "18       SDKs, libraries, and frameworks                1     Optional  \n",
       "19       SDKs, libraries, and frameworks                1     Optional  \n",
       "20       SDKs, libraries, and frameworks                1     Optional  \n",
       "21  Compilers, build tools, and runtimes               39     Optional  \n",
       "22  Compilers, build tools, and runtimes               36     Optional  \n",
       "23  Compilers, build tools, and runtimes               76     Optional  "
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Continue (skip) 1st loop from (inner) 2nd loop\n",
    "class ContinueParent(Exception): pass\n",
    "\n",
    "# Some packages may only have 1 variant of chip/arch available\n",
    "# If less strict mode is set to true, even if that package variant doesn't\n",
    "# match our preferred chip/arch, it will still be included as dependency\n",
    "less_strict = False\n",
    "\n",
    "# Initial dependencies (packages) of workload\n",
    "# Does not include nested dependencies yet\n",
    "dp_init = LowerDict()\n",
    "\n",
    "def check_version(ver_range: str) -> bool:\n",
    "    \"\"\" Check if dependency version requirement is valid for current VS version \"\"\"\n",
    "    try:\n",
    "        # Assume that it supports all version\n",
    "        if ver_range in ('', None): return True\n",
    "        # Search for pattern like this: [17.0,18.0)\n",
    "        elif re.search(r'^\\[\\d+\\.\\d+,\\d+\\.\\d+\\)$', ver_range):\n",
    "            min_ver = re.search(r'^\\[(\\d+\\.\\d+)', ver_range).group(1) # 17.0\n",
    "            max_ver = re.search(r',(\\d+\\.\\d+)\\)$', ver_range).group(1) # 18.0\n",
    "        # Search for pattern like this: [17.0,)\n",
    "        elif re.search(r'^\\[\\d+\\.\\d+,\\)$', ver_range):\n",
    "            min_ver = re.search(r'^\\[(\\d+\\.\\d+)', ver_range).group(1) # 17.0\n",
    "            max_ver = float('inf') # Infinity\n",
    "        # Search for pattern like this: 17.0.0.0\n",
    "        # Or just until the minor version: 17.0\n",
    "        elif re.search(r'^\\d+(\\.(\\d+)){0,3}$', ver_range):\n",
    "            min_ver = re.search(r'^(\\d+\\.\\d+)', ver_range).group(1) # 17.0\n",
    "            max_ver = min_ver\n",
    "        else: warn(f'Version format \"{ver_range}\" is unsupported!')\n",
    "    except AttributeError:\n",
    "        warn(f'Pattern matching group is not found for \"{ver_range}\"!')\n",
    "        raise\n",
    "\n",
    "    vs_ver_2 = float(f'{vs_ver_major}.{vs_ver_minor}')\n",
    "    if vs_ver_2 >= float(min_ver) and vs_ver_2 <= float(max_ver): return True\n",
    "    return False\n",
    "\n",
    "def get_sub_dependencies(parent_id: str, parent_props: LowerDict, output_dp_list: LowerDict) -> bool:\n",
    "    \"\"\" List all dependencies one level below (not nested) and write to output.\n",
    "        Output will be written as a dict list (dependency id as key).\n",
    "        Should be used together with \"get_packages_info()\" \"\"\"\n",
    "\n",
    "    # Check if the parent has dependencies\n",
    "    if parent_props.get('dependencies'):\n",
    "        # MS write package (sub) dependencies as dict, not as list\n",
    "        # Therefore loop it as dict (with key and value)\n",
    "        for key, value in parent_props['dependencies'].items():\n",
    "            # If the dependency value is still a dict\n",
    "            if isinstance(value, dict):\n",
    "                # Some dependencies have id written as a subkey (e.g. AspNet)\n",
    "                # Otherwise, take the parent dict name (key) directly as id\n",
    "                d_id = value.get('id')\n",
    "                if not d_id: d_id = key\n",
    "                # VS version that's required to install the dependency\n",
    "                # If it doesn't support the current version then skip\n",
    "                d_version = value.get('version')\n",
    "                if not check_version(d_version): continue\n",
    "                # Get type and send a warning if the type is wrong (e.g. vsix, msi, exe)\n",
    "                d_type = value.get('type')\n",
    "                if lower(d_type) not in lower(['Recommended', 'Optional', '', None]):\n",
    "                    warn(f'Recommendation flag ({d_type}) is different than expected!')\n",
    "                # Sometimes the dependency chip/arch type is also explicitly stated\n",
    "                # It means that it's required but only if you have a specific chip/arch\n",
    "                # The available chip/arch keys are \"chip\", \"machineArch\", and \"productArch\"\n",
    "                # Though from my observation, \"productArch\" was never listed as a requirement\n",
    "                d_chip = value.get('chip')\n",
    "                if lower(d_chip) not in lower([vs_chip[0], '', None]): continue\n",
    "                d_machine_arch = value.get('machineArch')\n",
    "                if lower(d_machine_arch) not in lower([vs_host[0], '', None]): continue\n",
    "                # Only add the dependency if the product id matches (build tools)\n",
    "                # Note that dict \"get()\" will treat empty string/object as true,\n",
    "                # While Python logic will treat empty string/object as false\n",
    "                d_prod_id = value.get('when')\n",
    "                if d_prod_id and lower(vs_product_id) not in lower(d_prod_id): continue\n",
    "            # If the dependency value is a string (version number)\n",
    "            else:\n",
    "                d_id = key\n",
    "                d_version = value\n",
    "                d_type = None\n",
    "                d_chip = None\n",
    "                d_machine_arch = None\n",
    "\n",
    "            if not output_dp_list.get(d_id):\n",
    "                # If the dependency is not listed yet\n",
    "                output_dp_list[d_id] = { # Lower dict\n",
    "                    # Id of the parent element that needs this dependency\n",
    "                    'parentId': parent_id,\n",
    "                    # Requirement to be able to install the dependency (from the parent)\n",
    "                    'require': {\n",
    "                        # Recommendation flag (recommended or optional)\n",
    "                        # If the type is none it's usually required or platform-specific\n",
    "                        'flag': d_type,\n",
    "                        'version': d_version,\n",
    "                        'chip': d_chip,\n",
    "                        'machineArch': d_machine_arch\n",
    "                    },\n",
    "                    # Identifier that we only have basic info about this dependency\n",
    "                    # Once we call \"get_packages_info\", it will be removed\n",
    "                    'incomplete': True\n",
    "                }\n",
    "            # If the dependency is already listed, just check for important value(s)\n",
    "            # May need to review if language/chip/arch should also be checked or not\n",
    "            else:\n",
    "                # Change the optional flag if it becomes recommended or required\n",
    "                if lower(d_type) == lower('Recommended') or d_type == None:\n",
    "                    output_dp_list[d_id]['require']['flag'] = d_type\n",
    "\n",
    "        # Return true if it has dependencies, otherwise false\n",
    "        return True\n",
    "    else:\n",
    "        return False\n",
    "\n",
    "# Get dependencies from the workload(s) and save them as dict list\n",
    "# Dependency id is used as key, disallowing duplicates if there are multiple variants\n",
    "# May be a bad idea but I'm too lazy to rework the entire codes\n",
    "# However, it will also prevent infinite loop if child dependency refers to the parent again\n",
    "for id, props in workload.items():\n",
    "    if lower(id) in lower(wl_needed):\n",
    "        # Get dependencies from parent id and props, write/add the results to \"dp_init\"\n",
    "        get_sub_dependencies(id, props, dp_init)\n",
    "\n",
    "def get_packages_variants(src_manifest: LowerDict, dp_id: str, pkg_index: int = None, max_pkg: int = None) -> dict:\n",
    "    \"\"\" Check all available variants of a package id (chip, language, etc).\n",
    "        If `pkg_index` (start) and `max_pkg` (end) are valid, packages list will be sliced.\n",
    "        That will improve performance instead of looping the entire manifest packages \"\"\"\n",
    "    variants = {\n",
    "        'id': dp_id,\n",
    "        'language': [],\n",
    "        'chip': [],\n",
    "        'machineArch': [],\n",
    "        'productArch': []\n",
    "    }\n",
    "\n",
    "    # Get the number of packages (for index checking)\n",
    "    num_pkg = len(src_manifest['packages'])\n",
    "\n",
    "    if pkg_index and max_pkg:\n",
    "        # Start searching variants at \"pkg_index\" until \"max_pkg\" is reached\n",
    "        start = pkg_index\n",
    "        end = pkg_index + max_pkg + 1\n",
    "        # Make sure the index is not out of bound\n",
    "        if end > num_pkg: end = num_pkg\n",
    "    else:\n",
    "        start = 0\n",
    "        end = num_pkg\n",
    "\n",
    "    num = 0\n",
    "    for i in src_manifest['packages'][start:end]:\n",
    "        if lower(i['id']) == lower(dp_id):\n",
    "            # Track how many variants are there\n",
    "            num += 1\n",
    "            # Add variants only if the key exists\n",
    "            for key in variants:\n",
    "                if i.get(key):\n",
    "                    if lower(key) == 'id': variants[key] = i.get(key)\n",
    "                    else: variants[key].append(i[key])\n",
    "\n",
    "    for key in set(variants) - {'id'}:\n",
    "        # If the key value is not an empty list\n",
    "        if variants.get(key):\n",
    "            # Warn if key only exists in some variants\n",
    "            if len(variants[key]) != num:\n",
    "                # You can remove the warning if it's annoying\n",
    "                # Though this one can be used to check in case of missing file\n",
    "                warn(f'Not all variants of {variants[\"id\"]} have \"{key}\" key!')\n",
    "\n",
    "    return variants\n",
    "\n",
    "def get_packages_info(src_manifest: LowerDict, input_dp_list: LowerDict):\n",
    "    \"\"\" Fetch packages info from manifest to add that info to input.\n",
    "        If you modify \"get_sub_dependencies()\", make sure it doesn't break this one! \"\"\"\n",
    "    dp_variant = None\n",
    "    for index, i in enumerate(src_manifest['packages']):\n",
    "        if i['id'] in input_dp_list: # Lower dict\n",
    "            # Skip getting info for this dependency (package) if it has already been done\n",
    "            # By checking if the \"incomplete\" key exists or not\n",
    "            if not input_dp_list[i['id']].get('incomplete'): continue\n",
    "\n",
    "            # Check if current package has any of these keys\n",
    "            if any([i.get('language'), i.get('chip'), i.get('productArch'), i.get('machineArch')]):\n",
    "                # If package variants haven't been searched before then search it now\n",
    "                if not dp_variant or lower(dp_variant['id']) != lower(i['id']):\n",
    "                    if i.get('language'):\n",
    "                        # Language combined with arch may create many variants\n",
    "                        # 14 language * 3 machineArch * 3 productArch = 126 variants\n",
    "                        # Use a bigger number of max packages to search in case of bad math\n",
    "                        # Also as a fail-safe for the possibility of future values\n",
    "                        dp_variant = get_packages_variants(manifest, i['id'], index, 250)\n",
    "                    else:\n",
    "                        # 3 machineArch * 3 productArch = 9 variants\n",
    "                        dp_variant = get_packages_variants(manifest, i['id'], index, 30)\n",
    "\n",
    "            # Variant keys that need to be checked, priority is based on the index\n",
    "            # Only one final package will be chosen no matter how many package variants are found\n",
    "            # Check the json file manually to see if the final language/chip/arch is correct\n",
    "            # Compare the variant you get with variants from manifest (by searching package id)\n",
    "            variants = ['language', 'machineArch', 'chip', 'productArch']\n",
    "\n",
    "            try:\n",
    "                for key in variants:\n",
    "                    # Preferred variants for language, chip, etc\n",
    "                    if key == 'language': vs_spec = vs_lang\n",
    "                    elif key == 'chip': vs_spec = vs_chip\n",
    "                    # Assume it's arch if not the others\n",
    "                    else: vs_spec = vs_host\n",
    "\n",
    "                    # Note that dict \"get()\" will treat empty string/object as true,\n",
    "                    # While Python logic will treat empty string/object as false\n",
    "                    val = i.get(key)\n",
    "                    # Therefore, remove \"key\" if it's just an empty string/object\n",
    "                    if i.get(key) and not val: i.pop(key)\n",
    "\n",
    "                    # If \"key\" is not explicitly stated as a requirement from the parent\n",
    "                    if val and not input_dp_list[i['id']]['require'].get(key):\n",
    "                        # If the neutral variant or my preferred variant exists\n",
    "                        if any(lower(x) in lower(['neutral', *vs_spec]) for x in dp_variant[key]):\n",
    "                            # Prioritize variant based on \"vs_spec\" index order\n",
    "                            # The latter will be removed if an earlier index variant exists\n",
    "                            for x in vs_spec:\n",
    "                                if lower(x) in lower(dp_variant[key]):\n",
    "                                    # Remove the latter variants and set only this one\n",
    "                                    dp_variant[key] = [x]\n",
    "                                    break\n",
    "                            # Skip this one if it's not the wanted variant\n",
    "                            if lower(i[key]) not in lower(['neutral', *dp_variant[key]]):\n",
    "                                raise ContinueParent()\n",
    "                        else:\n",
    "                            # If it's the only variant available, should we skip it?\n",
    "                            if less_strict: pass\n",
    "                            else:\n",
    "                                input_dp_list.pop(i['id'])\n",
    "                                raise ContinueParent() # Skip\n",
    "                    # If \"key\" is explicitly stated as a requirement from the parent\n",
    "                    elif val and input_dp_list[i['id']]['require'].get(key):\n",
    "                        # If \"key\" is not the same as the needed requirement\n",
    "                        if lower(i[key]) != lower(input_dp_list[i['id']]['require'].get(key)):\n",
    "                            if len(dp_variant[key]) == 1:\n",
    "                                # If it's the only variant available, should we skip it?\n",
    "                                if less_strict: pass\n",
    "                                else:\n",
    "                                    input_dp_list.pop(i['id'])\n",
    "                                    raise ContinueParent() # Skip\n",
    "                            else: raise ContinueParent() # Skip\n",
    "            except ContinueParent:\n",
    "                continue\n",
    "\n",
    "            # Keys to ignore for all packages, add based on your needs\n",
    "            # You may want to ignore some keys if you want to process it manually later\n",
    "            # Or just skip it because you don't want it to clutter the final json file\n",
    "            keys_to_ignore = [\n",
    "                'id',\n",
    "                'localizedResources',\n",
    "                'dependencies'\n",
    "            ]\n",
    "\n",
    "            # Add all available keys from package id\n",
    "            for key in i:\n",
    "                if key not in keys_to_ignore:\n",
    "                    val = i.get(key)\n",
    "                    if val: input_dp_list[i['id']][key] = val\n",
    "\n",
    "            # Don't add all translations, just the preferred one\n",
    "            if i.get('localizedResources'):\n",
    "                for item in i['localizedResources']:\n",
    "                    # \"localizedResources\" doesn't have \"neutral\" language\n",
    "                    # Prioritize preferred language(s) based on the index\n",
    "                    for j in vs_lang:\n",
    "                        if lower(item['language']) == lower(j):\n",
    "                            input_dp_list[i['id']]['title'] = item.get('title')\n",
    "                            input_dp_list[i['id']]['description'] = item.get('description')\n",
    "                            input_dp_list[i['id']]['category'] = item.get('category')\n",
    "                            break\n",
    "\n",
    "            # Does the dependency have other (nested) dependencies?\n",
    "            if i.get('dependencies') not in ('', {}, None):\n",
    "                input_dp_list[i['id']]['dependencies'] = i.get('dependencies')\n",
    "                input_dp_list[i['id']]['dependenciesNum'] = len(i['dependencies'])\n",
    "            else:\n",
    "                input_dp_list[i['id']]['dependenciesNum'] = 0\n",
    "\n",
    "            # Mark it as complete once we get all important info (by deleting the key)\n",
    "            input_dp_list[i['id']].pop('incomplete')\n",
    "\n",
    "# Call the function after declaring\n",
    "get_packages_info(manifest, dp_init)\n",
    "\n",
    "df = pd.DataFrame.from_dict(\n",
    "    dp_init,\n",
    "    orient = 'index'\n",
    ").rename_axis('id').reset_index()\n",
    "\n",
    "cols_to_show = [\n",
    "    'id',\n",
    "    'parentId',\n",
    "    'require',\n",
    "    'title',\n",
    "    'description',\n",
    "    'category',\n",
    "    'dependenciesNum',\n",
    "]\n",
    "# Only show some columns, hide/drop the rest\n",
    "df = df.loc[:, df.columns.isin(cols_to_show)]\n",
    "\n",
    "# Save current workload(s) dependencies (based on \"wl_needed\")\n",
    "# These dependencies are only one level below, not fully nested\n",
    "df.to_json(\n",
    "    f'{data_path}\\\\dependency_summary.json',\n",
    "    indent = 4,\n",
    "    force_ascii = False,\n",
    "    orient = 'records'\n",
    ")\n",
    "\n",
    "# Just take recommendation \"flag\" from the \"require\" key\n",
    "df['flag'] = df['require'].apply(lambda x: x.get('flag'))\n",
    "df.drop('require', axis = 1, inplace = True)\n",
    "\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(dp_init)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Filter dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dependency id(s) you may want to remove (one time removal only)\n",
    "# If during later process the dependency is needed by other component, it will be added again\n",
    "# This list will also be used again later so just keep it even if empty\n",
    "dp_remove = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num of dependencies (before): 24\n",
      "Removed Microsoft.VisualStudio.Component.VC.ATL\n",
      "Removed Microsoft.VisualStudio.Component.VC.ATLMFC\n",
      "Removed Microsoft.VisualStudio.Component.VC.CLI.Support\n",
      "Removed Microsoft.VisualStudio.Component.VC.Modules.x86.x64\n",
      "Removed Microsoft.VisualStudio.ComponentGroup.NativeDesktop.Llvm.Clang\n",
      "Removed Microsoft.VisualStudio.Component.Windows11SDK.22000\n",
      "Removed Microsoft.VisualStudio.Component.Windows10SDK.20348\n",
      "Removed Microsoft.VisualStudio.Component.Windows10SDK.19041\n",
      "Removed Microsoft.VisualStudio.Component.Windows10SDK.18362\n",
      "Removed Microsoft.VisualStudio.ComponentGroup.VC.Tools.142.x86.x64\n",
      "Removed Microsoft.VisualStudio.Component.VC.v141.x86.x64\n",
      "Removed Microsoft.VisualStudio.Component.VC.140\n",
      "Num of dependencies (after): 12\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>parentId</th>\n",
       "      <th>title</th>\n",
       "      <th>description</th>\n",
       "      <th>category</th>\n",
       "      <th>dependenciesNum</th>\n",
       "      <th>flag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Microsoft.Component.MSBuild</td>\n",
       "      <td>Microsoft.VisualStudio.Workload.MSBuildTools</td>\n",
       "      <td>MSBuild</td>\n",
       "      <td>MSBuild tasks and supported C#/VB compilers.</td>\n",
       "      <td>Compilers, build tools, and runtimes</td>\n",
       "      <td>5</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Microsoft.Build.Dependencies</td>\n",
       "      <td>Microsoft.VisualStudio.Workload.MSBuildTools</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Microsoft.VisualStudio.Component.CoreBuildTools</td>\n",
       "      <td>Microsoft.VisualStudio.Workload.MSBuildTools</td>\n",
       "      <td>Visual Studio Build Tools Core</td>\n",
       "      <td>Core features needed for supporting Visual Stu...</td>\n",
       "      <td>None</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Microsoft.VisualStudio.Component.VC.CoreBuildT...</td>\n",
       "      <td>Microsoft.VisualStudio.Workload.VCTools</td>\n",
       "      <td>C++ Build Tools core features</td>\n",
       "      <td>Core features needed for supporting Visual C++...</td>\n",
       "      <td>Development activities</td>\n",
       "      <td>8</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Microsoft.VisualStudio.Component.VC.Tools.x86.x64</td>\n",
       "      <td>Microsoft.VisualStudio.Workload.VCTools</td>\n",
       "      <td>MSVC v143 - VS 2022 C++ x64/x86 build tools (L...</td>\n",
       "      <td>Visual C++ (v143) latest compilers and librari...</td>\n",
       "      <td>Compilers, build tools, and runtimes</td>\n",
       "      <td>35</td>\n",
       "      <td>Recommended</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Microsoft.VisualStudio.Component.VC.Redist.14....</td>\n",
       "      <td>Microsoft.VisualStudio.Workload.VCTools</td>\n",
       "      <td>C++ 2022 Redistributable Update</td>\n",
       "      <td>Latest update for the Visual C++ Redistributable</td>\n",
       "      <td>Compilers, build tools, and runtimes</td>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Microsoft.VisualStudio.VC.UnitTest.Desktop.Bui...</td>\n",
       "      <td>Microsoft.VisualStudio.Workload.VCTools</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Microsoft.VisualStudio.Component.Windows11SDK....</td>\n",
       "      <td>Microsoft.VisualStudio.Workload.VCTools</td>\n",
       "      <td>Windows 11 SDK (10.0.22621.0)</td>\n",
       "      <td>Headers, libraries, and tools for developing a...</td>\n",
       "      <td>SDKs, libraries, and frameworks</td>\n",
       "      <td>1</td>\n",
       "      <td>Recommended</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Microsoft.VisualStudio.Component.VC.CMake.Project</td>\n",
       "      <td>Microsoft.VisualStudio.Workload.VCTools</td>\n",
       "      <td>C++ CMake tools for Windows</td>\n",
       "      <td>Visual C++ tools support for CMake.</td>\n",
       "      <td>Compilers, build tools, and runtimes</td>\n",
       "      <td>6</td>\n",
       "      <td>Recommended</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Microsoft.VisualStudio.Component.TestTools.Bui...</td>\n",
       "      <td>Microsoft.VisualStudio.Workload.VCTools</td>\n",
       "      <td>Testing tools core features - Build Tools</td>\n",
       "      <td>Includes core test tools to run tests from CLI.</td>\n",
       "      <td>Debugging and testing</td>\n",
       "      <td>1</td>\n",
       "      <td>Recommended</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Microsoft.VisualStudio.Component.VC.ASAN</td>\n",
       "      <td>Microsoft.VisualStudio.Workload.VCTools</td>\n",
       "      <td>C++ AddressSanitizer</td>\n",
       "      <td>AddressSanitizer (ASAN) is a tool for detectin...</td>\n",
       "      <td>Debugging and testing</td>\n",
       "      <td>1</td>\n",
       "      <td>Recommended</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Microsoft.VisualStudio.ComponentGroup.NativeDe...</td>\n",
       "      <td>Microsoft.VisualStudio.Workload.VCTools</td>\n",
       "      <td>C++ core desktop features</td>\n",
       "      <td>Core IDE features needed for Visual C++ deskto...</td>\n",
       "      <td>Development activities</td>\n",
       "      <td>7</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   id  \\\n",
       "0                         Microsoft.Component.MSBuild   \n",
       "1                        Microsoft.Build.Dependencies   \n",
       "2     Microsoft.VisualStudio.Component.CoreBuildTools   \n",
       "3   Microsoft.VisualStudio.Component.VC.CoreBuildT...   \n",
       "4   Microsoft.VisualStudio.Component.VC.Tools.x86.x64   \n",
       "5   Microsoft.VisualStudio.Component.VC.Redist.14....   \n",
       "6   Microsoft.VisualStudio.VC.UnitTest.Desktop.Bui...   \n",
       "7   Microsoft.VisualStudio.Component.Windows11SDK....   \n",
       "8   Microsoft.VisualStudio.Component.VC.CMake.Project   \n",
       "9   Microsoft.VisualStudio.Component.TestTools.Bui...   \n",
       "10           Microsoft.VisualStudio.Component.VC.ASAN   \n",
       "11  Microsoft.VisualStudio.ComponentGroup.NativeDe...   \n",
       "\n",
       "                                        parentId  \\\n",
       "0   Microsoft.VisualStudio.Workload.MSBuildTools   \n",
       "1   Microsoft.VisualStudio.Workload.MSBuildTools   \n",
       "2   Microsoft.VisualStudio.Workload.MSBuildTools   \n",
       "3        Microsoft.VisualStudio.Workload.VCTools   \n",
       "4        Microsoft.VisualStudio.Workload.VCTools   \n",
       "5        Microsoft.VisualStudio.Workload.VCTools   \n",
       "6        Microsoft.VisualStudio.Workload.VCTools   \n",
       "7        Microsoft.VisualStudio.Workload.VCTools   \n",
       "8        Microsoft.VisualStudio.Workload.VCTools   \n",
       "9        Microsoft.VisualStudio.Workload.VCTools   \n",
       "10       Microsoft.VisualStudio.Workload.VCTools   \n",
       "11       Microsoft.VisualStudio.Workload.VCTools   \n",
       "\n",
       "                                                title  \\\n",
       "0                                             MSBuild   \n",
       "1                                                 NaN   \n",
       "2                      Visual Studio Build Tools Core   \n",
       "3                       C++ Build Tools core features   \n",
       "4   MSVC v143 - VS 2022 C++ x64/x86 build tools (L...   \n",
       "5                     C++ 2022 Redistributable Update   \n",
       "6                                                 NaN   \n",
       "7                       Windows 11 SDK (10.0.22621.0)   \n",
       "8                         C++ CMake tools for Windows   \n",
       "9           Testing tools core features - Build Tools   \n",
       "10                               C++ AddressSanitizer   \n",
       "11                          C++ core desktop features   \n",
       "\n",
       "                                          description  \\\n",
       "0        MSBuild tasks and supported C#/VB compilers.   \n",
       "1                                                 NaN   \n",
       "2   Core features needed for supporting Visual Stu...   \n",
       "3   Core features needed for supporting Visual C++...   \n",
       "4   Visual C++ (v143) latest compilers and librari...   \n",
       "5    Latest update for the Visual C++ Redistributable   \n",
       "6                                                 NaN   \n",
       "7   Headers, libraries, and tools for developing a...   \n",
       "8                 Visual C++ tools support for CMake.   \n",
       "9     Includes core test tools to run tests from CLI.   \n",
       "10  AddressSanitizer (ASAN) is a tool for detectin...   \n",
       "11  Core IDE features needed for Visual C++ deskto...   \n",
       "\n",
       "                                category  dependenciesNum         flag  \n",
       "0   Compilers, build tools, and runtimes                5         None  \n",
       "1                                    NaN                1         None  \n",
       "2                                   None                3         None  \n",
       "3                 Development activities                8         None  \n",
       "4   Compilers, build tools, and runtimes               35  Recommended  \n",
       "5   Compilers, build tools, and runtimes                1         None  \n",
       "6                                    NaN                1         None  \n",
       "7        SDKs, libraries, and frameworks                1  Recommended  \n",
       "8   Compilers, build tools, and runtimes                6  Recommended  \n",
       "9                  Debugging and testing                1  Recommended  \n",
       "10                 Debugging and testing                1  Recommended  \n",
       "11                Development activities                7         None  "
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from copy import deepcopy\n",
    "\n",
    "print(f'Num of dependencies (before): {len(dp_init)}')\n",
    "\n",
    "# Start excluding dependencies you don't need\n",
    "# For starter, I'm also removing all dependencies flagged as optional\n",
    "# Use deepcopy because we can't iterate while removing things from the iterator\n",
    "for i in deepcopy(dp_init):\n",
    "    if lower(dp_init[i]) in lower(dp_remove):\n",
    "        dp_init.pop(i)\n",
    "        lower_remove(dp_remove, i)\n",
    "        print(f'Removed {i}')\n",
    "    # Remove optional dependencies\n",
    "    elif lower(dp_init[i]['require']['flag']) == lower('Optional'):\n",
    "        dp_init.pop(i)\n",
    "        print(f'Removed {i}')\n",
    "\n",
    "print(f'Num of dependencies (after): {len(dp_init)}')\n",
    "\n",
    "df = pd.DataFrame.from_dict(\n",
    "    dp_init,\n",
    "    orient = 'index'\n",
    ").rename_axis('id').reset_index()\n",
    "\n",
    "# Only show some columns, hide/drop the rest\n",
    "df = df.loc[:, df.columns.isin(cols_to_show)]\n",
    "\n",
    "# Replace the previous file with filtered values\n",
    "# These dependencies are only one level below, not fully nested\n",
    "df.to_json(\n",
    "    f'{data_path}\\\\dependency_summary.json',\n",
    "    indent = 4,\n",
    "    force_ascii = False,\n",
    "    orient = 'records'\n",
    ")\n",
    "\n",
    "# Just take recommendation \"flag\" from the \"require\" key\n",
    "df['flag'] = df['require'].apply(lambda x: x.get('flag'))\n",
    "df.drop('require', axis = 1, inplace = True)\n",
    "\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(dp_init)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get all (nested) dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Include miscellaneous dependency id(s) that must be included\n",
    "# Sometimes, there may be important dependencies that are not included by default\n",
    "# Even if this list is empty, just keep it because it may be used later\n",
    "dp_misc = [\n",
    "    # Clang compiler, will increase the installation size massively\n",
    "    # 'Microsoft.VisualStudio.ComponentGroup.NativeDesktop.Llvm.Clang'\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Dhika\\AppData\\Local\\Temp\\VSCodePortableTemp\\ipykernel_5144\\2231605760.py:167: UserWarning: Not all variants of Microsoft.VisualCpp.RuntimeDebug.14 have \"machineArch\" key!\n",
      "  warn(f'Not all variants of {variants[\"id\"]} have \"{key}\" key!')\n",
      "C:\\Users\\Dhika\\AppData\\Local\\Temp\\VSCodePortableTemp\\ipykernel_5144\\2231605760.py:167: UserWarning: Not all variants of Microsoft.VisualStudio.Debugger.CollectionAgents have \"machineArch\" key!\n",
      "  warn(f'Not all variants of {variants[\"id\"]} have \"{key}\" key!')\n",
      "C:\\Users\\Dhika\\AppData\\Local\\Temp\\VSCodePortableTemp\\ipykernel_5144\\2231605760.py:167: UserWarning: Not all variants of Microsoft.VisualStudio.Debugger.Remote have \"machineArch\" key!\n",
      "  warn(f'Not all variants of {variants[\"id\"]} have \"{key}\" key!')\n",
      "C:\\Users\\Dhika\\AppData\\Local\\Temp\\VSCodePortableTemp\\ipykernel_5144\\2231605760.py:167: UserWarning: Not all variants of Microsoft.VisualStudio.Debugger.Concord have \"machineArch\" key!\n",
      "  warn(f'Not all variants of {variants[\"id\"]} have \"{key}\" key!')\n",
      "C:\\Users\\Dhika\\AppData\\Local\\Temp\\VSCodePortableTemp\\ipykernel_5144\\2231605760.py:167: UserWarning: Not all variants of Microsoft.VisualStudio.Debugger.Concord.Remote have \"machineArch\" key!\n",
      "  warn(f'Not all variants of {variants[\"id\"]} have \"{key}\" key!')\n",
      "C:\\Users\\Dhika\\AppData\\Local\\Temp\\VSCodePortableTemp\\ipykernel_5144\\2231605760.py:167: UserWarning: Not all variants of Microsoft.VisualStudio.MinShell.Targeted have \"machineArch\" key!\n",
      "  warn(f'Not all variants of {variants[\"id\"]} have \"{key}\" key!')\n",
      "C:\\Users\\Dhika\\AppData\\Local\\Temp\\VSCodePortableTemp\\ipykernel_5144\\2231605760.py:167: UserWarning: Not all variants of Microsoft.VisualStudio.CoreEditor have \"machineArch\" key!\n",
      "  warn(f'Not all variants of {variants[\"id\"]} have \"{key}\" key!')\n",
      "C:\\Users\\Dhika\\AppData\\Local\\Temp\\VSCodePortableTemp\\ipykernel_5144\\2231605760.py:167: UserWarning: Not all variants of Microsoft.VisualStudio.Debugger.Concord.Remote.Resources have \"machineArch\" key!\n",
      "  warn(f'Not all variants of {variants[\"id\"]} have \"{key}\" key!')\n",
      "C:\\Users\\Dhika\\AppData\\Local\\Temp\\VSCodePortableTemp\\ipykernel_5144\\2231605760.py:167: UserWarning: Not all variants of Microsoft.VisualStudio.ClientDiagnostics.Targeted have \"machineArch\" key!\n",
      "  warn(f'Not all variants of {variants[\"id\"]} have \"{key}\" key!')\n",
      "C:\\Users\\Dhika\\AppData\\Local\\Temp\\VSCodePortableTemp\\ipykernel_5144\\2231605760.py:167: UserWarning: Not all variants of Microsoft.VisualStudio.Community.VB.Resources.Targeted have \"machineArch\" key!\n",
      "  warn(f'Not all variants of {variants[\"id\"]} have \"{key}\" key!')\n"
     ]
    }
   ],
   "source": [
    "# Can be used to clear output if something went wrong (non-stop loop)\n",
    "from IPython.display import clear_output\n",
    "\n",
    "# List of dependencies we want to install/extract in the end\n",
    "# Python sometimes uses reference when assigning value to variable\n",
    "# Deepcopy will force it to copy as value instead of reference\n",
    "dp_final = deepcopy(dp_init) # Lower dict\n",
    "\n",
    "# Temporarily removed dependencies that have both payloads and (sub) dependencies\n",
    "# May also workaround some troublesome packages that list themselves as (sub) dependency\n",
    "# Can cause missing files if removed permanently\n",
    "dp_payloads = LowerDict()\n",
    "\n",
    "# Dependency id(s) that are \"component\" types\n",
    "# Not important at all but may be needed by \"vswhere\"\n",
    "dp_components = set()\n",
    "\n",
    "def still_has_dependencies(input_dp_list: LowerDict) -> bool:\n",
    "    \"\"\" Check if all dependencies are lowest level (no more sub dependencies) \"\"\"\n",
    "    for id, props in input_dp_list.items():\n",
    "        if props.get('incomplete') or props['dependenciesNum'] > 0:\n",
    "            return True\n",
    "    return False\n",
    "\n",
    "# Fetch all nested dependencies and add it directly to main list\n",
    "# This may take at worst a minute or two, but if it still doesn't stop, something is broken\n",
    "while still_has_dependencies(dp_final):\n",
    "    # Add our miscellaneous dependencies\n",
    "    for i in dp_misc:\n",
    "        # Match the keys written here with \"get_sub_dependencies\"\n",
    "        # Fill most values with none because we only know the id\n",
    "        dp_final[i] = {\n",
    "            'parentId': None,\n",
    "            'require': {\n",
    "                'flag': None,\n",
    "                'version': None,\n",
    "                'chip': None,\n",
    "                'machineArch': None\n",
    "            },\n",
    "            'incomplete': True\n",
    "        }\n",
    "\n",
    "    # Add all nested dependencies as same-level dependencies\n",
    "    for id, props in deepcopy(dp_final).items():\n",
    "        # Make sure we don't remove miscellaneous dependencies because we haven't got any info yet\n",
    "        # Once we make an exception (else), remove the id from \"dp_misc\"\n",
    "        # So all the dependencies info can be added/searched on the next iteration\n",
    "        if lower(id) not in lower(dp_misc):\n",
    "            # Get dependencies of parent id and props, write the results to \"dp_final\"\n",
    "            has_dependencies = get_sub_dependencies(id, props, dp_final)\n",
    "            # Move the parent to the removal list because we already have all the (sub) dependencies\n",
    "            # This is to avoid searching dependencies for the same parent again and again\n",
    "            # But we also must check if it has payloads or not before deleting\n",
    "            if has_dependencies:\n",
    "                dp_remove.append(id)\n",
    "        else:\n",
    "            lower_remove(dp_misc, id)\n",
    "\n",
    "    # Fetch description and other info (version, payloads, etc)\n",
    "    # This is for both miscellaneous dependencies and the newly added dependencies\n",
    "    get_packages_info(manifest, dp_final)\n",
    "\n",
    "    for id, props in deepcopy(dp_final).items():\n",
    "        # Save component id(s) before removing\n",
    "        if lower(props.get('type')) == 'component':\n",
    "            if lower(id) not in lower(dp_components):\n",
    "                dp_components.add(id)\n",
    "\n",
    "        # Start removing parent elements of our dependencies\n",
    "        # If the parent has payloads, it will be temporarily moved from \"dp_final\" to \"dp_payloads\"\n",
    "        # It will be restored later once we make sure there is no more nested dependency\n",
    "        if lower(id) in lower(dp_remove):\n",
    "            dp_final.pop(id)\n",
    "            # Parent id may be needed by \"vswhere\"\n",
    "            # lower_remove(dp_remove, id)\n",
    "\n",
    "            if props.get('payloads'):\n",
    "                dp_payloads[id] = props\n",
    "                # warn(f'{id} has dependencies and payloads!')\n",
    "                if lower(props.get('parentId')) == lower(id):\n",
    "                    warn(f'{id} has itself as a dependency!')\n",
    "\n",
    "    # clear_output(wait = True)\n",
    "\n",
    "# Start re-adding removed parent packages that contain payloads\n",
    "for id, props in dp_payloads.items():\n",
    "    dp_final[id] = props\n",
    "\n",
    "# Save all (nested) dependencies that contain payloads\n",
    "dump = json.dumps(dict(dp_final), indent = 4, ensure_ascii = False)\n",
    "with open(f'{data_path}\\\\dependency_details.json', 'wb') as file:\n",
    "    file.write(dump.encode())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(dp_final)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "vs_config = { \"version\": \"1.0\", \"components\": [] }\n",
    "for i in dp_components:\n",
    "    vs_config['components'].append(i)\n",
    "\n",
    "# Export configuration (can be used by build tools installer)\n",
    "dump = json.dumps(vs_config, indent = 4)\n",
    "with open(f'{data_path}\\\\.vsconfig', 'wb') as file:\n",
    "    file.write(dump.encode())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>package_type</th>\n",
       "      <th>packages_num</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Vsix</td>\n",
       "      <td>295</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Msi</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Exe</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Zip</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  package_type  packages_num\n",
       "0         Vsix           295\n",
       "1          Msi            19\n",
       "2          Exe             6\n",
       "3          Zip             2"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total files (payloads) to download: 598\n",
      "Total download size: 2.03 GiB\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>payload_ext</th>\n",
       "      <th>payloads_num</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>vsix</td>\n",
       "      <td>295</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>msi</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>cab</td>\n",
       "      <td>192</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>exe</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>zip</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>ps1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  payload_ext  payloads_num\n",
       "0        vsix           295\n",
       "1         msi           100\n",
       "2         cab           192\n",
       "3         exe             7\n",
       "4         zip             2\n",
       "5         ps1             2"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import math\n",
    "\n",
    "def convert_size(size_bytes: int, precision: int = 2):\n",
    "    if size_bytes == 0: return '0 B'\n",
    "    size_name = ('B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB')\n",
    "    index = int(math.floor(math.log(size_bytes, 1024)))\n",
    "    power = math.pow(1024, index)\n",
    "    size = round(size_bytes / power, precision)\n",
    "    return f'{size} {size_name[index]}'\n",
    "\n",
    "total_files = 0\n",
    "download_size = 0\n",
    "file_ext = LowerDict()\n",
    "pkg_type = LowerDict()\n",
    "\n",
    "for i in dp_final.values():\n",
    "    # Get all package types\n",
    "    if i[\"type\"] not in pkg_type: # Lower dict\n",
    "        pkg_type[i['type']] = 1\n",
    "    else:\n",
    "        pkg_type[i['type']] += 1\n",
    "\n",
    "    # Get package payloads info\n",
    "    for j in i['payloads']:\n",
    "        # Add total files to download\n",
    "        total_files += 1\n",
    "        # Add total download size\n",
    "        if j.get('size'):\n",
    "            download_size += j['size']\n",
    "        # Get all file (payload) extensions\n",
    "        ext = j['fileName'].split(\".\")[-1]\n",
    "        if ext not in file_ext: # Lower dict\n",
    "            file_ext[ext] = 1\n",
    "        else: file_ext[ext] += 1\n",
    "\n",
    "df = pd.DataFrame.from_dict(\n",
    "    pkg_type,\n",
    "    orient = 'index',\n",
    "    columns = ['packages_num']\n",
    ").rename_axis('package_type').reset_index()\n",
    "display(df)\n",
    "\n",
    "print(f'Total files (payloads) to download: {total_files}')\n",
    "print(f'Total download size: {convert_size(download_size)}')\n",
    "\n",
    "df = pd.DataFrame.from_dict(\n",
    "    file_ext,\n",
    "    orient = 'index',\n",
    "    columns = ['payloads_num']\n",
    ").rename_axis('payload_ext').reset_index()\n",
    "display(df)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Download payloads\n",
    "- Linux users should be wary that each path is case-sensitive\n",
    "- Recommended to change target path to a NTFS/FAT partition\n",
    "- Also, backslashes in path may cause issues on Linux"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>packageId</th>\n",
       "      <th>packageType</th>\n",
       "      <th>url</th>\n",
       "      <th>fileName</th>\n",
       "      <th>fileType</th>\n",
       "      <th>targetDir</th>\n",
       "      <th>filePath</th>\n",
       "      <th>sha256</th>\n",
       "      <th>size</th>\n",
       "      <th>extensionDir</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Microsoft.VisualStudio.NativeImageSupport</td>\n",
       "      <td>Vsix</td>\n",
       "      <td>https://download.visualstudio.microsoft.com/do...</td>\n",
       "      <td>payload.vsix</td>\n",
       "      <td>vsix</td>\n",
       "      <td>c:\\Users\\Dhika\\Documents\\Projects\\Proto\\Python...</td>\n",
       "      <td>c:\\Users\\Dhika\\Documents\\Projects\\Proto\\Python...</td>\n",
       "      <td>5627e2cae4f5ec3be71a5303d1d070d39f0f1cbed41d59...</td>\n",
       "      <td>24140</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Microsoft.NuGet.Build.Tasks.Setup</td>\n",
       "      <td>Vsix</td>\n",
       "      <td>https://download.visualstudio.microsoft.com/do...</td>\n",
       "      <td>Microsoft.NuGet.Build.Tasks.Setup.vsix</td>\n",
       "      <td>vsix</td>\n",
       "      <td>c:\\Users\\Dhika\\Documents\\Projects\\Proto\\Python...</td>\n",
       "      <td>c:\\Users\\Dhika\\Documents\\Projects\\Proto\\Python...</td>\n",
       "      <td>5635b7c143b0f022cdd8f79494718f4c86df8ed0b03624...</td>\n",
       "      <td>289165</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Microsoft.PythonTools.BuildCore.Vsix</td>\n",
       "      <td>Vsix</td>\n",
       "      <td>https://download.visualstudio.microsoft.com/do...</td>\n",
       "      <td>Microsoft.PythonTools.BuildCore.Vsix.vsix</td>\n",
       "      <td>vsix</td>\n",
       "      <td>c:\\Users\\Dhika\\Documents\\Projects\\Proto\\Python...</td>\n",
       "      <td>c:\\Users\\Dhika\\Documents\\Projects\\Proto\\Python...</td>\n",
       "      <td>0693de9e5ad73ab55ebb6597ae327c0fe18826fcec4e8f...</td>\n",
       "      <td>338178</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Microsoft.Build.FileTracker.Msi</td>\n",
       "      <td>Msi</td>\n",
       "      <td>https://download.visualstudio.microsoft.com/do...</td>\n",
       "      <td>Microsoft.Build.FileTracker.Msi.msi</td>\n",
       "      <td>msi</td>\n",
       "      <td>c:\\Users\\Dhika\\Documents\\Projects\\Proto\\Python...</td>\n",
       "      <td>c:\\Users\\Dhika\\Documents\\Projects\\Proto\\Python...</td>\n",
       "      <td>c3a2fb55efb35848271854c5a6d0e02e295bbefa5d7e5a...</td>\n",
       "      <td>49152</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Microsoft.Build.FileTracker.Msi</td>\n",
       "      <td>Msi</td>\n",
       "      <td>https://download.visualstudio.microsoft.com/do...</td>\n",
       "      <td>cab1.cab</td>\n",
       "      <td>cab</td>\n",
       "      <td>c:\\Users\\Dhika\\Documents\\Projects\\Proto\\Python...</td>\n",
       "      <td>c:\\Users\\Dhika\\Documents\\Projects\\Proto\\Python...</td>\n",
       "      <td>9f841ec081a33069c1db6c2ab93617ab4381cb1f69e205...</td>\n",
       "      <td>404202</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>593</th>\n",
       "      <td>Microsoft.VisualStudio.PerformanceProvider</td>\n",
       "      <td>Vsix</td>\n",
       "      <td>https://download.visualstudio.microsoft.com/do...</td>\n",
       "      <td>payload.vsix</td>\n",
       "      <td>vsix</td>\n",
       "      <td>c:\\Users\\Dhika\\Documents\\Projects\\Proto\\Python...</td>\n",
       "      <td>c:\\Users\\Dhika\\Documents\\Projects\\Proto\\Python...</td>\n",
       "      <td>6b498931bb9fc43e0188a62ab54702db428a1ff3fdadd2...</td>\n",
       "      <td>40714</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>594</th>\n",
       "      <td>Microsoft.VisualStudio.WebSiteProject.DTE</td>\n",
       "      <td>Vsix</td>\n",
       "      <td>https://download.visualstudio.microsoft.com/do...</td>\n",
       "      <td>payload.vsix</td>\n",
       "      <td>vsix</td>\n",
       "      <td>c:\\Users\\Dhika\\Documents\\Projects\\Proto\\Python...</td>\n",
       "      <td>c:\\Users\\Dhika\\Documents\\Projects\\Proto\\Python...</td>\n",
       "      <td>0ebe982d538da2474a7bb9a49d0f63a24aa8b54de64299...</td>\n",
       "      <td>49013</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>595</th>\n",
       "      <td>Microsoft.VisualStudio.Community.ProductArch.R...</td>\n",
       "      <td>Vsix</td>\n",
       "      <td>https://download.visualstudio.microsoft.com/do...</td>\n",
       "      <td>payload.vsix</td>\n",
       "      <td>vsix</td>\n",
       "      <td>c:\\Users\\Dhika\\Documents\\Projects\\Proto\\Python...</td>\n",
       "      <td>c:\\Users\\Dhika\\Documents\\Projects\\Proto\\Python...</td>\n",
       "      <td>991245a81865ef38bc2df8e2273fd428d94e59cf8284c4...</td>\n",
       "      <td>41073</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>596</th>\n",
       "      <td>Microsoft.VisualStudio.Debugger.Script.Remote....</td>\n",
       "      <td>Vsix</td>\n",
       "      <td>https://download.visualstudio.microsoft.com/do...</td>\n",
       "      <td>payload.vsix</td>\n",
       "      <td>vsix</td>\n",
       "      <td>c:\\Users\\Dhika\\Documents\\Projects\\Proto\\Python...</td>\n",
       "      <td>c:\\Users\\Dhika\\Documents\\Projects\\Proto\\Python...</td>\n",
       "      <td>73f1ad57fe434a73571937a51b6988ffc55b68055b23b3...</td>\n",
       "      <td>10609</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>597</th>\n",
       "      <td>Microsoft.VisualStudio.Diagnostics.AspNetHelper</td>\n",
       "      <td>Vsix</td>\n",
       "      <td>https://download.visualstudio.microsoft.com/do...</td>\n",
       "      <td>payload.vsix</td>\n",
       "      <td>vsix</td>\n",
       "      <td>c:\\Users\\Dhika\\Documents\\Projects\\Proto\\Python...</td>\n",
       "      <td>c:\\Users\\Dhika\\Documents\\Projects\\Proto\\Python...</td>\n",
       "      <td>14fccf1d222e1ec07f4006d7a23f8d6f573a64a496a493...</td>\n",
       "      <td>1197</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>598 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             packageId packageType  \\\n",
       "0            Microsoft.VisualStudio.NativeImageSupport        Vsix   \n",
       "1                    Microsoft.NuGet.Build.Tasks.Setup        Vsix   \n",
       "2                 Microsoft.PythonTools.BuildCore.Vsix        Vsix   \n",
       "3                      Microsoft.Build.FileTracker.Msi         Msi   \n",
       "4                      Microsoft.Build.FileTracker.Msi         Msi   \n",
       "..                                                 ...         ...   \n",
       "593         Microsoft.VisualStudio.PerformanceProvider        Vsix   \n",
       "594          Microsoft.VisualStudio.WebSiteProject.DTE        Vsix   \n",
       "595  Microsoft.VisualStudio.Community.ProductArch.R...        Vsix   \n",
       "596  Microsoft.VisualStudio.Debugger.Script.Remote....        Vsix   \n",
       "597    Microsoft.VisualStudio.Diagnostics.AspNetHelper        Vsix   \n",
       "\n",
       "                                                   url  \\\n",
       "0    https://download.visualstudio.microsoft.com/do...   \n",
       "1    https://download.visualstudio.microsoft.com/do...   \n",
       "2    https://download.visualstudio.microsoft.com/do...   \n",
       "3    https://download.visualstudio.microsoft.com/do...   \n",
       "4    https://download.visualstudio.microsoft.com/do...   \n",
       "..                                                 ...   \n",
       "593  https://download.visualstudio.microsoft.com/do...   \n",
       "594  https://download.visualstudio.microsoft.com/do...   \n",
       "595  https://download.visualstudio.microsoft.com/do...   \n",
       "596  https://download.visualstudio.microsoft.com/do...   \n",
       "597  https://download.visualstudio.microsoft.com/do...   \n",
       "\n",
       "                                      fileName fileType  \\\n",
       "0                                 payload.vsix     vsix   \n",
       "1       Microsoft.NuGet.Build.Tasks.Setup.vsix     vsix   \n",
       "2    Microsoft.PythonTools.BuildCore.Vsix.vsix     vsix   \n",
       "3          Microsoft.Build.FileTracker.Msi.msi      msi   \n",
       "4                                     cab1.cab      cab   \n",
       "..                                         ...      ...   \n",
       "593                               payload.vsix     vsix   \n",
       "594                               payload.vsix     vsix   \n",
       "595                               payload.vsix     vsix   \n",
       "596                               payload.vsix     vsix   \n",
       "597                               payload.vsix     vsix   \n",
       "\n",
       "                                             targetDir  \\\n",
       "0    c:\\Users\\Dhika\\Documents\\Projects\\Proto\\Python...   \n",
       "1    c:\\Users\\Dhika\\Documents\\Projects\\Proto\\Python...   \n",
       "2    c:\\Users\\Dhika\\Documents\\Projects\\Proto\\Python...   \n",
       "3    c:\\Users\\Dhika\\Documents\\Projects\\Proto\\Python...   \n",
       "4    c:\\Users\\Dhika\\Documents\\Projects\\Proto\\Python...   \n",
       "..                                                 ...   \n",
       "593  c:\\Users\\Dhika\\Documents\\Projects\\Proto\\Python...   \n",
       "594  c:\\Users\\Dhika\\Documents\\Projects\\Proto\\Python...   \n",
       "595  c:\\Users\\Dhika\\Documents\\Projects\\Proto\\Python...   \n",
       "596  c:\\Users\\Dhika\\Documents\\Projects\\Proto\\Python...   \n",
       "597  c:\\Users\\Dhika\\Documents\\Projects\\Proto\\Python...   \n",
       "\n",
       "                                              filePath  \\\n",
       "0    c:\\Users\\Dhika\\Documents\\Projects\\Proto\\Python...   \n",
       "1    c:\\Users\\Dhika\\Documents\\Projects\\Proto\\Python...   \n",
       "2    c:\\Users\\Dhika\\Documents\\Projects\\Proto\\Python...   \n",
       "3    c:\\Users\\Dhika\\Documents\\Projects\\Proto\\Python...   \n",
       "4    c:\\Users\\Dhika\\Documents\\Projects\\Proto\\Python...   \n",
       "..                                                 ...   \n",
       "593  c:\\Users\\Dhika\\Documents\\Projects\\Proto\\Python...   \n",
       "594  c:\\Users\\Dhika\\Documents\\Projects\\Proto\\Python...   \n",
       "595  c:\\Users\\Dhika\\Documents\\Projects\\Proto\\Python...   \n",
       "596  c:\\Users\\Dhika\\Documents\\Projects\\Proto\\Python...   \n",
       "597  c:\\Users\\Dhika\\Documents\\Projects\\Proto\\Python...   \n",
       "\n",
       "                                                sha256    size extensionDir  \n",
       "0    5627e2cae4f5ec3be71a5303d1d070d39f0f1cbed41d59...   24140          NaN  \n",
       "1    5635b7c143b0f022cdd8f79494718f4c86df8ed0b03624...  289165          NaN  \n",
       "2    0693de9e5ad73ab55ebb6597ae327c0fe18826fcec4e8f...  338178          NaN  \n",
       "3    c3a2fb55efb35848271854c5a6d0e02e295bbefa5d7e5a...   49152          NaN  \n",
       "4    9f841ec081a33069c1db6c2ab93617ab4381cb1f69e205...  404202          NaN  \n",
       "..                                                 ...     ...          ...  \n",
       "593  6b498931bb9fc43e0188a62ab54702db428a1ff3fdadd2...   40714          NaN  \n",
       "594  0ebe982d538da2474a7bb9a49d0f63a24aa8b54de64299...   49013          NaN  \n",
       "595  991245a81865ef38bc2df8e2273fd428d94e59cf8284c4...   41073          NaN  \n",
       "596  73f1ad57fe434a73571937a51b6988ffc55b68055b23b3...   10609          NaN  \n",
       "597  14fccf1d222e1ec07f4006d7a23f8d6f573a64a496a493...    1197          NaN  \n",
       "\n",
       "[598 rows x 10 columns]"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pyl_list = []\n",
    "\n",
    "for id, i in dp_final.items():\n",
    "    # Folder name is composed based on these keys\n",
    "    # Don't reorder the index unless you know what you're doing\n",
    "    folder_keys = ['version', 'chip', 'language', 'productArch', 'machineArch']\n",
    "    # Need to be reset each iteration\n",
    "    folder_name = id\n",
    "\n",
    "    for key in folder_keys:\n",
    "        # Note that dict \"get()\" will treat empty string/object as true,\n",
    "        # While Python logic will treat empty string/object as false\n",
    "        val = i.get(key)\n",
    "        if val and isinstance(val, str):\n",
    "            folder_name += f',{key}={val}'\n",
    "\n",
    "    for j in i['payloads']:\n",
    "        # Not sure why, but the first added item will be written as tuple if dict is empty\n",
    "        # I'm initializing non-empty dict directly as a workaround\n",
    "        pyl = {'packageId': id}\n",
    "        pyl['packageType'] = i['type']\n",
    "        pyl['url'] = j['url']\n",
    "        pyl['fileName'] = j['fileName']\n",
    "        pyl['fileType'] = j['fileName'].split('.')[-1]\n",
    "        pyl['targetDir'] = cache_path + '\\\\' + folder_name\n",
    "        pyl['filePath'] = pyl['targetDir'] + '\\\\' + pyl['fileName']\n",
    "        pyl['sha256'] = j['sha256']\n",
    "        pyl['size'] = j['size']\n",
    "\n",
    "        # For vsix packages that don't have \"Contents\" folder\n",
    "        # Key is case-insensitive (lower dict)\n",
    "        if lower(i['type']) == 'vsix':\n",
    "            # Note that dict \"get()\" will treat empty string/object as true,\n",
    "            # While Python logic will treat empty string/object as false\n",
    "            ext_dir = i.get('extensionDir')\n",
    "            if ext_dir: pyl['extensionDir'] = ext_dir\n",
    "\n",
    "        pyl_list.append(pyl)\n",
    "\n",
    "df = pd.DataFrame(pyl_list)\n",
    "\n",
    "# Save payload list as csv and json\n",
    "# May be useful if you want to use external download manager\n",
    "df.to_csv(\n",
    "    f'{data_path}\\\\payloads.csv',\n",
    "    index = False,\n",
    "    sep = ';'\n",
    ")\n",
    "df.to_json(\n",
    "    f'{data_path}\\\\payloads.json',\n",
    "    indent = 4,\n",
    "    force_ascii = False,\n",
    "    orient = 'records'\n",
    ")\n",
    "\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not vs_package_download:\n",
    "    # Choosing yes will also verify all local packages\n",
    "    # If the package already exist and valid, it won't be redownloaded\n",
    "    vs_package_download = input('Do want to download and/or verify all needed packages (Y/N)?')\n",
    "    if lower(vs_package_download) != 'y':\n",
    "        vs_package_download = False\n",
    "        # raise KeyboardInterrupt('Not continuing because there is no package to download/extract')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "** Microsoft.VisualStudio.Diagnostics.AspNetHelper **\n",
      "payload.vsix checksum is valid!\n"
     ]
    }
   ],
   "source": [
    "from urllib import request\n",
    "from tqdm import tqdm\n",
    "\n",
    "# I'm just copying the code from here\n",
    "# https://github.com/tqdm/tqdm#hooks-and-callbacks\n",
    "class TqdmUpTo(tqdm):\n",
    "    def progress_bar(self, b = 1, bsize = 1, tsize = None):\n",
    "        if tsize is not None: self.total = tsize\n",
    "        return self.update(b * bsize - self.n)\n",
    "\n",
    "# Doesn't support pause and resume!\n",
    "# Download is also single thread only for now\n",
    "def download_file(url, file_path, desc = None):\n",
    "    tqdm_kwargs = {\n",
    "        'unit': 'B',\n",
    "        'unit_scale': True,\n",
    "        'unit_divisor': 1024,\n",
    "        'miniters': 1,\n",
    "        'desc': desc\n",
    "    }\n",
    "\n",
    "    with TqdmUpTo(**tqdm_kwargs) as t:\n",
    "        request.urlretrieve(url, file_path, reporthook = t.progress_bar)\n",
    "        t.total = t.n\n",
    "\n",
    "if vs_package_download:\n",
    "    # Package id of the current payload\n",
    "    pkg_id = None\n",
    "    # Auto-clear cell output\n",
    "    # Will be disabled if an error is found\n",
    "    auto_clear = True\n",
    "\n",
    "    for i in pyl_list:\n",
    "        # Quick access\n",
    "        package_id = i['packageId']\n",
    "        url = i['url']\n",
    "        file_name = i['fileName']\n",
    "        target_dir = i['targetDir']\n",
    "        file_path = i['filePath']\n",
    "        sha256 = i['sha256']\n",
    "\n",
    "        # A single package can have multiple payloads\n",
    "        # Don't reprint package id for each payload \n",
    "        if package_id != pkg_id or auto_clear:\n",
    "            pkg_id = package_id\n",
    "            print(f'\\n** {pkg_id} **')\n",
    "\n",
    "        # File checksum\n",
    "        correct_file = False\n",
    "        # Maximum redownload tries\n",
    "        tries = 3\n",
    "\n",
    "        # Check file existence\n",
    "        if os.path.isfile(file_path):\n",
    "            correct_file = check_sha256sum(file_path, sha256)\n",
    "        else:\n",
    "            parent_dir = os.path.dirname(file_path)\n",
    "            # Create (nested) directories for parent dir\n",
    "            os.makedirs(parent_dir, exist_ok = True)\n",
    "\n",
    "        # Don't redownload file if the checksum is correct\n",
    "        while not correct_file and tries > 0:\n",
    "            print(f'Downloading {file_name}... (tries left: {tries})')\n",
    "            download_file(url, file_path, desc = file_name)\n",
    "            correct_file = check_sha256sum(file_path, sha256)\n",
    "            if not correct_file: tries -= 1\n",
    "\n",
    "        if not correct_file:\n",
    "            print(f'No tries left, {file_name} checksum is incorrect!')\n",
    "            # Disable auto-clear\n",
    "            auto_clear = False\n",
    "        else:\n",
    "            print(f'{file_name} checksum is valid!')\n",
    "\n",
    "        if auto_clear: clear_output(wait = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For moving or deleting files and folders\n",
    "import shutil\n",
    "\n",
    "# Payload target directories\n",
    "pyl_dir = { i['targetDir'] for i in pyl_list }\n",
    "\n",
    "# Check for outdated packages\n",
    "# By comparing existing folders with payloads list\n",
    "if vs_package_download and vs_package_del_outdated:\n",
    "    for i in os.scandir(cache_path):\n",
    "        if i.is_dir():\n",
    "            if i.path not in pyl_dir:\n",
    "                print(f'Removed {i.name}')\n",
    "                # This may be dangerous\n",
    "                shutil.rmtree(i.path)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Extract files (installation)\n",
    "- Generic `vsix` files only\n",
    "- Some `msi` files will be treated specially later"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# \"Microsoft.Build.FileTracker.Msi\" type (Msi) is unsupported!\n",
      "* \"Microsoft.VS.VC.vcvars.X64.Shortcuts\" payload (payload.vsix) doesn't have \"Contents\" folder\n",
      "* \"Microsoft.VS.VC.vcvars.X86.Shortcuts\" payload (payload.vsix) doesn't have \"Contents\" folder\n",
      "# \"Microsoft.VisualCpp.CRT.ClickOnce.Msi\" type (Msi) is unsupported!\n",
      "# \"Win11SDK_10.0.22621\" type (Exe) is unsupported!\n",
      "# \"Microsoft.Build.UnGAC\" type (Exe) is unsupported!\n",
      "# \"Microsoft.VisualStudio.Setup.Configuration\" type (Msi) is unsupported!\n",
      "# \"Microsoft.VisualStudio.Setup.WMIProvider\" type (Msi) is unsupported!\n",
      "# \"Microsoft.Windows.UniversalCRT.Redistributable.Msi\" type (Msi) is unsupported!\n",
      "* \"Microsoft.VisualCpp.Tools.Core.Resources\" payload (Microsoft.VisualCpp.Tools.Core.Resources.enu.vsix) doesn't have \"Contents\" folder\n",
      "* \"Microsoft.VisualStudio.StaticAnalysis.auxil.Resources\" payload (payload.vsix) doesn't have \"Contents\" folder\n",
      "* \"Microsoft.VisualStudio.StaticAnalysis.FxCop.Resources\" payload (payload.vsix) doesn't have \"Contents\" folder\n",
      "* \"Microsoft.VisualCpp.CRT.Redist.Resources\" payload (Microsoft.VisualCpp.CRT.Redist.Resources.enu.vsix) doesn't have \"Contents\" folder\n",
      "* \"Microsoft.VisualStudio.TextTemplating.Integration.Resources\" payload (payload.vsix) doesn't have \"Contents\" folder\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# \"Microsoft.VisualStudio.Initializer\" type (Exe) is unsupported!\n",
      "# \"Microsoft.VisualStudio.GitHubProtocolHandler.Msi\" type (Msi) is unsupported!\n",
      "* \"Microsoft.VisualStudio.VisualC.Utilities.Resources\" payload (payload.vsix) doesn't have \"Contents\" folder\n",
      "* \"Microsoft.VisualStudio.Debugger.Concord.Managed.Resources\" payload (payload.vsix) doesn't have \"Contents\" folder\n",
      "# \"Microsoft.VisualStudio.Debugger.Script.Msi\" type (Msi) is unsupported!\n",
      "# \"CoreEditorFonts\" type (Msi) is unsupported!\n",
      "# \"Microsoft.WebView2\" type (Exe) is unsupported!\n",
      "# \"Microsoft.VisualStudio.MinShell.Interop.Shared.Msi\" type (Msi) is unsupported!\n",
      "# \"Microsoft.VisualStudio.Community.Msi.Resources\" type (Msi) is unsupported!\n",
      "# \"Microsoft.Net.6.Runtime\" type (Zip) is unsupported!\n",
      "# \"Microsoft.VisualStudio.MinShell.Msi.Resources\" type (Msi) is unsupported!\n",
      "# \"Microsoft.VisualStudio.Community.Shared.Msi\" type (Msi) is unsupported!\n",
      "* \"Microsoft.VisualCpp.CRT.Redist.x64.OneCore.Desktop\" payload (Microsoft.VisualCpp.CRT.Redist.x64.OneCore.Desktop.vsix) doesn't have \"Contents\" folder\n",
      "* \"Microsoft.VisualCpp.CRT.Redist.x86.OneCore.Desktop\" payload (Microsoft.VisualCpp.CRT.Redist.x86.OneCore.Desktop.vsix) doesn't have \"Contents\" folder\n",
      "* \"Microsoft.VisualCpp.CRT.x64.OneCore.Desktop\" payload (Microsoft.VisualCpp.CRT.x64.OneCore.Desktop.vsix) doesn't have \"Contents\" folder\n",
      "* \"Microsoft.VisualCpp.CRT.x64.Store\" payload (Microsoft.VisualCpp.CRT.x64.Store.vsix) doesn't have \"Contents\" folder\n",
      "* \"Microsoft.VisualCpp.CRT.x86.OneCore.Desktop\" payload (Microsoft.VisualCpp.CRT.x86.OneCore.Desktop.vsix) doesn't have \"Contents\" folder\n",
      "* \"Microsoft.VisualCpp.CRT.x86.Store\" payload (Microsoft.VisualCpp.CRT.x86.Store.vsix) doesn't have \"Contents\" folder\n",
      "* \"Microsoft.VisualCpp.PGO.Headers\" payload (Microsoft.VisualCpp.PGO.Headers.vsix) doesn't have \"Contents\" folder\n",
      "* \"Microsoft.VisualCpp.PGO.x64\" payload (Microsoft.VisualCpp.PGO.X64.vsix) doesn't have \"Contents\" folder\n",
      "* \"Microsoft.VisualCpp.PGO.x86\" payload (Microsoft.VisualCpp.PGO.X86.vsix) doesn't have \"Contents\" folder\n",
      "* \"Microsoft.VisualCpp.premium.Tools.HostX64.TargetX64\" payload (Microsoft.VisualCpp.Premium.Tools.HostX64.TargetX64.vsix) doesn't have \"Contents\" folder\n",
      "* \"Microsoft.VisualCpp.premium.Tools.HostX64.TargetX86\" payload (Microsoft.VisualCpp.Premium.Tools.HostX64.TargetX86.vsix) doesn't have \"Contents\" folder\n",
      "* \"Microsoft.VisualCpp.premium.Tools.HostX86.TargetX86\" payload (Microsoft.VisualCpp.Premium.Tools.HostX86.TargetX86.vsix) doesn't have \"Contents\" folder\n",
      "* \"Microsoft.VisualCpp.premium.Tools.HostX86.TargetX64\" payload (Microsoft.VisualCpp.Premium.Tools.HostX86.TargetX64.vsix) doesn't have \"Contents\" folder\n",
      "# \"Microsoft.VisualCpp.Redist.14\" type (Exe) is unsupported!\n",
      "* \"Microsoft.VisualCpp.CA.Ext.Hostx64.TargetX86\" payload (Microsoft.VisualCpp.CA.Ext.Hostx64.Targetx86.vsix) doesn't have \"Contents\" folder\n",
      "* \"Microsoft.VisualCpp.CA.Ext.Hostx64.TargetX64\" payload (Microsoft.VisualCpp.CA.Ext.Hostx64.Targetx64.vsix) doesn't have \"Contents\" folder\n",
      "* \"Microsoft.VisualCpp.CA.Ext.Hostx86.Targetx86\" payload (Microsoft.VisualCpp.CA.Ext.Hostx86.Targetx86.vsix) doesn't have \"Contents\" folder\n",
      "* \"Microsoft.VisualCpp.CA.Ext.Hostx86.Targetx64\" payload (Microsoft.VisualCpp.CA.Ext.Hostx86.Targetx64.vsix) doesn't have \"Contents\" folder\n",
      "# \"Microsoft.VisualCpp.Redist.14.Latest\" type (Exe) is unsupported!\n",
      "* \"Microsoft.VisualCpp.ASAN.X86\" payload (Microsoft.VisualCpp.ASAN.X86.vsix) doesn't have \"Contents\" folder\n",
      "* \"Microsoft.VisualCpp.CRT.Headers\" payload (Microsoft.VisualCpp.CRT.Headers.vsix) doesn't have \"Contents\" folder\n",
      "# \"Microsoft.VisualCpp.RuntimeDebug.14\" type (Msi) is unsupported!\n",
      "* \"Microsoft.VisualCpp.CRT.source\" payload (Microsoft.VisualCpp.CRT.Source.vsix) doesn't have \"Contents\" folder\n",
      "* \"Microsoft.VisualCpp.CRT.x64.Desktop\" payload (Microsoft.VisualCpp.CRT.x64.Desktop.vsix) doesn't have \"Contents\" folder\n",
      "* \"Microsoft.VisualCpp.CRT.x86.Desktop\" payload (Microsoft.VisualCpp.CRT.x86.Desktop.vsix) doesn't have \"Contents\" folder\n",
      "* \"Microsoft.VisualCpp.Tools.HostX86.TargetX86.Res\" payload (Microsoft.VisualCpp.Tools.HostX86.TargetX86.Res.enu.vsix) doesn't have \"Contents\" folder\n",
      "* \"Microsoft.VisualCpp.Tools.HostX86.TargetX64.Res\" payload (Microsoft.VisualCpp.Tools.Hostx86.Targetx64.Res.enu.vsix) doesn't have \"Contents\" folder\n",
      "* \"Microsoft.VisualStudio.VC.Ide.Pro\" payload (payload.vsix) doesn't have \"Contents\" folder\n",
      "# \"Microsoft.VisualStudio.MinShell.Msi\" type (Msi) is unsupported!\n",
      "# \"Microsoft.VisualStudio.MinShell.Interop.Msi\" type (Msi) is unsupported!\n",
      "# \"Microsoft.Net.6.WindowsDesktop.Runtime\" type (Zip) is unsupported!\n",
      "# \"Microsoft.VisualStudio.VsWebProtocolSelector.Msi\" type (Msi) is unsupported!\n",
      "# \"Microsoft.VisualStudio.FileHandler.Msi\" type (Msi) is unsupported!\n",
      "# \"Microsoft.VisualStudio.MinShell.Shared.Msi\" type (Msi) is unsupported!\n",
      "# \"Microsoft.VisualStudio.Community.Msi\" type (Msi) is unsupported!\n",
      "* \"Microsoft.VisualStudio.ProjectSystem.Full\" payload (Microsoft.VisualStudio.ProjectSystem.Full.vsix) doesn't have \"Contents\" folder\n",
      "* \"Microsoft.VisualStudio.Diagnostics.AspNetHelper\" payload (payload.vsix) doesn't have \"Contents\" folder\n"
     ]
    }
   ],
   "source": [
    "from urllib import parse\n",
    "import zipfile\n",
    "\n",
    "# Recursively delete the old build tools files\n",
    "# Disabled by default because it's too dangerous\n",
    "# if vs_package_extract:\n",
    "#     shutil.rmtree(install_path)\n",
    "\n",
    "def write_file(file_bytes: bytes, file_path: str, package_id: str):\n",
    "    # Replace forward slash with backslash (if any)\n",
    "    # \"makedirs\" and \"write\" may fail on Windows if forward slash exist(s)\n",
    "    file_path = file_path.replace('/', '\\\\')\n",
    "    # Ensure parent directories exist\n",
    "    file_dir = os.path.dirname(file_path)\n",
    "    os.makedirs(file_dir, exist_ok = True)\n",
    "    # Existing file will be overwritten\n",
    "    with open(file_path, 'wb') as f:\n",
    "        f.write(file_bytes)\n",
    "\n",
    "# Root files/folders to ignore if vsix package has \"extensionDir\"\n",
    "# All case should be lower case to avoid missmatch\n",
    "vsix_ext_ignore = (\n",
    "    'package/',\n",
    "    '_rels/',\n",
    "    'manifest.json',\n",
    "    lower('[Content_Types].xml'),\n",
    "    lower('Contents/')\n",
    ")\n",
    "\n",
    "pkg_id = None\n",
    "for i in pyl_list:\n",
    "    package_id = i['packageId']\n",
    "    package_type = i['packageType']\n",
    "    file_name = i['fileName']\n",
    "    file_path = i['filePath']\n",
    "    file_type = i['fileType']\n",
    "\n",
    "    if lower(package_type) == 'vsix' and vs_package_download:\n",
    "        with zipfile.ZipFile(file_path) as z:\n",
    "            has_contents = False\n",
    "            for name in z.namelist():\n",
    "                if lower(name).startswith('contents/'):\n",
    "                    has_contents = True\n",
    "                    if vs_package_extract:\n",
    "                        # Decode URL like path (e.g. %20 to space)\n",
    "                        d_name = parse.unquote(name)\n",
    "                        # Remove \"contents\" (parent folder)\n",
    "                        d_name = d_name[len(\"contents/\"):]\n",
    "                        # Can't be extracted directly due to URL like path\n",
    "                        file = z.read(name)\n",
    "                        write_file(file, f'{install_path}\\\\{d_name}', package_id)\n",
    "        if not has_contents:\n",
    "            if i.get('extensionDir'):\n",
    "                ext_dir = i['extensionDir']\n",
    "                if ext_dir == None: ext_dir = ''\n",
    "\n",
    "                if lower(ext_dir).startswith('[installdir]'):\n",
    "                    # Correct installation directory\n",
    "                    ext_dir = ext_dir.replace('[installdir]', install_path)\n",
    "                    with zipfile.ZipFile(file_path) as z:\n",
    "                        for name in z.namelist():\n",
    "                            if not lower(name).startswith(vsix_ext_ignore):\n",
    "                                if vs_package_extract:\n",
    "                                    d_name = parse.unquote(name)\n",
    "                                    # Use \"extensionDir\" as install path\n",
    "                                    file = z.read(name)\n",
    "                                    write_file(file, f'{ext_dir}\\\\{d_name}', package_id)\n",
    "                else:\n",
    "                    print(f'$ \"{package_id}\" payload ({file_name}) doesn\\'t have \"installdir\" path')\n",
    "            else:\n",
    "                # Remove this message if it's annoying\n",
    "                # You can re-enable it to troubleshoot if you miss some important files\n",
    "                print(f'* \"{package_id}\" payload ({file_name}) doesn\\'t have \"Contents\" folder')\n",
    "                # pass\n",
    "    else:\n",
    "        # A single package id can have multiple payloads\n",
    "        # Only reprint info if the previous id doesn't match\n",
    "        if package_id != pkg_id:\n",
    "            pkg_id = package_id\n",
    "            # This package may need to be inspected and extracted manually\n",
    "            # You can use some tools like Sandboxie or Universal Extractor\n",
    "            print(f'# \"{package_id}\" type ({package_type}) is unsupported!')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Install Windows SDK\n",
    "- Windows SDK is universal for all archs, there may be arch files you want to delete manually after install"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "* Found Win11SDK_10.0.22621 *\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Parameters splitting (designed for Unix shell)\n",
    "# May broke Windows path, check result manually\n",
    "import shlex\n",
    "# For running installer\n",
    "import subprocess\n",
    "\n",
    "# Regex may incorrectly interpret some characters as special search pattern\n",
    "# It's bad for replacing Windows path which contains a lot of backslashes\n",
    "# Code taken from: https://stackoverflow.com/a/4773614\n",
    "def ireplace(text, old, new):\n",
    "    \"\"\" Case-insensitive string replace \"\"\"\n",
    "    idx = 0\n",
    "    while idx < len(text):\n",
    "        index_l = text.lower().find(old.lower(), idx)\n",
    "        if index_l == -1: return text\n",
    "        text = text[:index_l] + new + text[index_l + len(old):]\n",
    "        idx = index_l + len(new) \n",
    "    return text\n",
    "\n",
    "def run_sdk_installer(installer_path: str, installer_params: str):\n",
    "    \"\"\" Parse the parameters and run Windows SDK (un)installer \"\"\"\n",
    "\n",
    "    # Known alias in parameters\n",
    "    replace_params = {\n",
    "        '[LogFile]': f'{install_path}\\\\temp\\\\win_sdk_setup.log',\n",
    "        '[CEIPConsentOnOff]': 'off',\n",
    "        '[ProgramFilesOrSharedDrive]': f'{install_path}\\\\temp'\n",
    "    }\n",
    "\n",
    "    # Replace known alias\n",
    "    for key, val in replace_params.items():\n",
    "        installer_params = ireplace(installer_params, key, val)\n",
    "\n",
    "    print(f'Installer: {installer_path}')\n",
    "    print(f'Params: {shlex.split(installer_params)}')\n",
    "    print(f'Please wait, (un)installer is already running...\\n')\n",
    "\n",
    "    # Run installer/uninstaller and wait until finished\n",
    "    # Though using \"shlex\" is not recommended on Windows, I haven't noticed any issue\n",
    "    os.makedirs(f'{install_path}\\\\temp', exist_ok = True)\n",
    "    subprocess.check_call([installer_path, *shlex.split(installer_params)])\n",
    "\n",
    "def rm_auto(full_path: str, recursive = False):\n",
    "    \"\"\" Remove file or folder only if exists \"\"\"\n",
    "    if os.path.exists(full_path):\n",
    "        if os.path.isfile(full_path):\n",
    "            os.remove(full_path)\n",
    "        elif os.path.isdir(full_path):\n",
    "            if recursive:\n",
    "                shutil.rmtree(full_path)\n",
    "            else:\n",
    "                os.rmdir(full_path)\n",
    "\n",
    "sdk_ver = set()\n",
    "pkg_id = None\n",
    "\n",
    "for i in pyl_list:\n",
    "    # Search for pattern like this: Win11SDK_10.0.22621\n",
    "    sdk_id = re.match(r'^(win\\d+sdk_\\d+\\.\\d+\\.\\d+)$', lower(i['packageId']))\n",
    "    if sdk_id: sdk_id = sdk_id.group(1)\n",
    "    else: continue\n",
    "\n",
    "    # Set values only if it's Windows SDK\n",
    "    package_id = i['packageId']\n",
    "    package_type = i['packageType']\n",
    "    file_name = i['fileName']\n",
    "    file_path = i['filePath']\n",
    "    file_type = i['fileType']\n",
    "\n",
    "    if package_id != pkg_id:\n",
    "        pkg_id = package_id\n",
    "        print(f'* Found {package_id} *\\n')\n",
    "\n",
    "    # \"pyl_info\" doesn't contain all necessary info\n",
    "    # Get full info from \"dp_final\" (like manifest)\n",
    "    package_info = dp_final[sdk_id]\n",
    "    sdk_ver.add(package_info['version'])\n",
    "\n",
    "    if vs_package_win_sdk:\n",
    "        # Installer file and parameters\n",
    "        ins_file_name = package_info['installParams']['fileName']\n",
    "        ins_params = package_info['installParams']['parameters']\n",
    "        # Skip if payload file name doesn't match installer file name\n",
    "        if not lower(ins_file_name) == lower(file_name): continue\n",
    "        # Install Windows SDK\n",
    "        run_sdk_installer(file_path, ins_params)\n",
    "\n",
    "        # Move Windows SDK files from temporary directory\n",
    "        # Currently, destination directory is set to \"install_path\"\n",
    "        print('Moving Windows SDK files...\\n')\n",
    "        sdk_files = os.listdir(f'{install_path}\\\\temp')\n",
    "        for file in sdk_files:\n",
    "            dest = f'{install_path}\\\\{file}'\n",
    "            # Remove destination file/folder if already exists\n",
    "            if os.path.exists(dest):\n",
    "                if os.path.isfile(dest): os.remove(dest)\n",
    "                else:\n",
    "                    del_dir = input(f'\"{dest}\" already exists, delete it (Y/N)?')\n",
    "                    if lower(del_dir) == 'y': shutil.rmtree(dest)\n",
    "            # Move file/folder to destination directory\n",
    "            shutil.move(f'{install_path}\\\\temp\\\\{file}', install_path)\n",
    "\n",
    "        # Uninstaller file and parameters\n",
    "        unins_file_name = package_info['uninstallParams']['fileName']\n",
    "        unins_params = package_info['uninstallParams']['parameters']\n",
    "        # Skip if payload file name doesn't match uninstaller file name\n",
    "        if not lower(unins_file_name) == lower(file_name): continue\n",
    "        # Uninstall Windows SDK\n",
    "        run_sdk_installer(file_path, unins_params)\n",
    "\n",
    "# Delete log files and temp directory\n",
    "rm_auto(f'{install_path}\\\\temp\\\\win_sdk_setup.log')\n",
    "rm_auto(f'{install_path}\\\\win_sdk_setup.log')\n",
    "rm_auto(f'{install_path}\\\\temp', recursive = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Extract and copy miscellaneous files\n",
    "- Fix for some weird errors related to building/debugging process\n",
    "- Copy UCRT debug dlls (`vcruntimeXXXd.dll`, etc)\n",
    "- Copy DIA SDK dll (`msdiaXXX.dll`)\n",
    "- Chosen SDK and MSVC version will also be used again later"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Selected SDK v10.0.22621.0!\n"
     ]
    }
   ],
   "source": [
    "# FIXME Check SDK versions based on path existence rather than using (latest) payloads list\n",
    "# When using payloads list, there are chances that installed outdated SDKs will not be detected\n",
    "# May need to use \"glob\" (wildcard) to support multiple Windows versions (8.1, 10, etc)\n",
    "# However, not all paths contain full SDK files, so some paths may need to be excluded\n",
    "\n",
    "if isinstance(sdk_ver, set):\n",
    "    sdk_ver = list(sdk_ver)\n",
    "\n",
    "if len(sdk_ver) == 1:\n",
    "    sdk_ver = sdk_ver[0]\n",
    "# Don't check length if SDK version is already a string\n",
    "elif isinstance(sdk_ver, list) and len(sdk_ver) > 1:\n",
    "    print('Multiple SDK versions detected!')\n",
    "    num = 1\n",
    "    for i in sdk_ver:\n",
    "        print(f'{num} - SDK v{i}')\n",
    "        num += 1\n",
    "    print(f'Select the number (1-{len(sdk_ver)}):')\n",
    "    choose = int(input())\n",
    "    sdk_ver = sdk_ver[choose-1]\n",
    "elif not isinstance(sdk_ver, str):\n",
    "    raise KeyError('No SDK version detected!')\n",
    "\n",
    "sdk_ver_major = re.search(r'^(\\d+)\\.', sdk_ver).group(1)\n",
    "sdk_ver_minor = re.search(r'^\\d+\\.(\\d+)\\.', sdk_ver).group(1)\n",
    "# Just in case there is something like Windows 8.1\n",
    "sdk_ver_auto = sdk_ver_major if sdk_ver_minor == '0' else f'{sdk_ver_major}.{sdk_ver_minor}'\n",
    "# Get Windows SDK path based on SDK version\n",
    "sdk_path = f'{install_path}\\\\Windows Kits\\\\{sdk_ver_auto}'\n",
    "\n",
    "# Semantic versioning (A.B.C instead of A.B.C.D)\n",
    "sdk_ver_patch = re.search(r'^\\d+\\.\\d+\\.(\\d+)', sdk_ver).group(1)\n",
    "sdk_ver_sem = f'{sdk_ver_major}.{sdk_ver_minor}.{sdk_ver_patch}'\n",
    "\n",
    "# Try changing SDK version to A.B.C.0 instead of A.B.C.D if path doesn't exist\n",
    "if not os.path.exists(f'{sdk_path}\\\\bin\\\\{sdk_ver}'):\n",
    "    if os.path.exists(f'{sdk_path}\\\\bin\\\\{sdk_ver_sem}.0'):\n",
    "        sdk_ver = f'{sdk_ver_sem}.0'\n",
    "    else:\n",
    "        warn(f'\"{sdk_path}\\\\bin\\\\{sdk_ver}\" doesn\\'t exist!')\n",
    "\n",
    "print(f'Selected SDK v{sdk_ver}!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Selected MSVC v14.37.32822!\n"
     ]
    }
   ],
   "source": [
    "# Checking MSVC versions based on package ids or payloads list is unreliable\n",
    "# They are many package ids, and each of them may have different purpose which I'm too lazy to check\n",
    "# Luckily, MSVC versions are not affected by Windows version, so it's simpler to just check the path\n",
    "\n",
    "msvc_path = f'{install_path}\\\\VC\\\\Tools\\\\MSVC'\n",
    "try:\n",
    "    msvc_ver = [ x for x in os.listdir(msvc_path) if os.path.isdir(f'{msvc_path}\\\\{x}') ]\n",
    "except FileNotFoundError:\n",
    "    msvc_ver = []\n",
    "\n",
    "if len(msvc_ver) == 1:\n",
    "    msvc_ver = msvc_ver[0]\n",
    "elif len(msvc_ver) > 1:\n",
    "    print('Multiple MSVC versions detected!')\n",
    "    num = 1\n",
    "    for i in msvc_ver:\n",
    "        print(f'{num} - MSVC v{i}')\n",
    "        num += 1\n",
    "    print(f'Select the number (1-{len(msvc_ver)}):')\n",
    "    choose = int(input())\n",
    "    msvc_ver = msvc_ver[choose-1]\n",
    "else:\n",
    "    msvc_ver = '0.0.0'\n",
    "    warn('No MSVC version detected!')\n",
    "\n",
    "msvc_ver_major = re.search(r'^(\\d+)\\.', msvc_ver).group(1)\n",
    "\n",
    "print(f'Selected MSVC v{msvc_ver}!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Arch to use (mainly used for path checking)\n",
    "arch = lower(vs_host[0])\n",
    "# Full path to MSVC \"bin\" folder\n",
    "msvc_bin = f'{msvc_path}\\\\{msvc_ver}\\\\bin\\\\Host{arch}\\\\{arch}'\n",
    "\n",
    "ucrt_debug_pkg_id = [\n",
    "    f'microsoft.visualcpp.runtimedebug.{msvc_ver_major}',\n",
    "    f'microsoft.visualcpp.runtimedebug.{msvc_ver_major}.{arch}'\n",
    "]\n",
    "\n",
    "if vs_package_extract:\n",
    "    for i in pyl_list:\n",
    "        package_id = i['packageId']\n",
    "        package_type = i['packageType']\n",
    "        file_name = i['fileName']\n",
    "        file_path = i['filePath']\n",
    "        file_type = i['fileType']\n",
    "\n",
    "        # Extract UCRT debug dlls\n",
    "        if lower(package_id) in ucrt_debug_pkg_id:\n",
    "            if lower(file_type) == 'msi':\n",
    "                subprocess.check_call(\n",
    "                # FIXME Will fail if there is space in path\n",
    "                ['msiexec.exe', '/a', file_path, '/quiet', '/qn', f'TARGETDIR={install_path}\\\\temp']\n",
    "            )\n",
    "\n",
    "    # Copy UCRT debug dlls\n",
    "    for dir_path, _, file_name in os.walk(f'{install_path}\\\\temp'):\n",
    "        for file in [ f for f in file_name if f.endswith('.dll') ]:\n",
    "            # Will overwrite file in destination if exist\n",
    "            shutil.copy(f'{dir_path}\\\\{file}', msvc_bin)\n",
    "            print(f'Found \"{dir_path}\\\\{file}\" (UCRT)')\n",
    "\n",
    "    # Copy DIA SDK dll\n",
    "    msdia_path = f'{install_path}\\\\DIA SDK\\\\bin'\n",
    "\n",
    "    if arch == 'x86': msdia_dll = f'msdia{msvc_ver_major}0.dll'\n",
    "    elif arch == 'x64': msdia_dll = f'amd64\\\\msdia{msvc_ver_major}0.dll'\n",
    "    else: msdia_dll = f'{arch}\\\\msdia{msvc_ver_major}0.dll'\n",
    "\n",
    "    if os.path.isfile(f'{msdia_path}\\\\{msdia_dll}'):\n",
    "        shutil.copy(f'{msdia_path}\\\\{msdia_dll}', msvc_bin)\n",
    "        print(f'Found \"{msdia_path}\\\\{msdia_dll}\" (DIA SDK)')\n",
    "    else:\n",
    "        warn(f'\"{msdia_path}\\\\{msdia_dll}\" doesn\\'t exist!')\n",
    "\n",
    "# Delete temp directory\n",
    "rm_auto(f'{install_path}\\\\temp', recursive = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Set environment variables\n",
    "- May not fit everyone's needs\n",
    "- Potentially dangerous if added recklessly"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C:\\BuildTools\\vs_env_setup.bat\n"
     ]
    }
   ],
   "source": [
    "# https://gist.github.com/mmozeiko/7f3162ec2988e81e56d5c4e22cde9977\n",
    "# https://github.com/Data-Oriented-House/PortableBuildTools/blob/master/src/install.odin\n",
    "# https://renenyffenegger.ch/notes/Windows/development/Visual-Studio/environment-variables/index\n",
    "# https://gitlab.kitware.com/cmake/cmake\n",
    "\n",
    "# Run \"vcvarsall.bat ARCH SDK_VER\" to see all VS environment variables\n",
    "# Compare the variables with CMake source code or other tools to see which are the most important\n",
    "# Backslash before end quote must be escaped, otherwise the backslash will be ignored and the quote will added instead\n",
    "# FIXME Migrate to PowerShell because cmd is unpredictable as fuck\n",
    "\n",
    "batch_setup = fr\"\"\"@echo off\n",
    "echo This will create/modify some environment variables PERMANENTLY.\n",
    "echo You CAN'T undo this action if something goes wrong.\n",
    "choice /c yn /m \"Are you sure you want to continue\"\n",
    "if %ERRORLEVEL% == 2 exit /b\n",
    "\n",
    "set ROOT=%~dp0\n",
    "\n",
    "set MSVC_ARCH={arch}\n",
    "set MSVC_VERSION={msvc_ver}\n",
    "set MSVC_ROOT=%ROOT%VC\\Tools\\MSVC\\%MSVC_VERSION%\n",
    "set MSVC_BIN=%MSVC_ROOT%\\bin\\Host%MSVC_ARCH%\\%MSVC_ARCH%\n",
    "set MSVC_INCLUDE=%MSVC_ROOT%\\include\n",
    "set MSVC_LIB=%MSVC_ROOT%\\lib\\%MSVC_ARCH%\n",
    "\n",
    "set SDK_ARCH={arch}\n",
    "set SDK_VERSION={sdk_ver}\n",
    "set SDK_ROOT=%ROOT%Windows Kits\\{sdk_ver_auto}\n",
    "set SDK_BIN=%SDK_ROOT%\\bin\\%SDK_VERSION%\\%SDK_ARCH%;%SDK_ROOT%\\bin\\%SDK_VERSION%\\%SDK_ARCH%\\ucrt\n",
    "set SDK_INCLUDE=%SDK_ROOT%\\Include\\%SDK_VERSION%\n",
    "set SDK_LIB=%SDK_ROOT%\\Lib\\%SDK_VERSION%\n",
    "\n",
    "set VS_PATH=%MSVC_BIN%;%SDK_BIN%\n",
    "\n",
    "setx VSINSTALLDIR \"{install_path}\\\\\"\n",
    "setx VS{vs_ver_major}0COMNTOOLS \"%ROOT%Common7\\Tools\\\\\"\n",
    "setx VisualStudioVersion \"{vs_ver_major}.0\"\n",
    "setx VCToolsInstallDir \"%MSVC_ROOT%\\\\\"\n",
    "setx VCTOOLSVERSION \"{msvc_ver}\"\n",
    "setx WINDOWSSDKDIR \"%SDK_ROOT%\\\\\"\n",
    "setx WINDOWSSDKVERSION \"{sdk_ver}\\\\\"\n",
    "\n",
    "setx INCLUDE \"%MSVC_INCLUDE%;%SDK_INCLUDE%\\ucrt;%SDK_INCLUDE%\\shared;%SDK_INCLUDE%\\um;%SDK_INCLUDE%\\winrt;%SDK_INCLUDE%\\cppwinrt\"\n",
    "setx LIB \"%MSVC_LIB%;%SDK_LIB%\\ucrt\\%SDK_ARCH%;%SDK_LIB%\\um\\%SDK_ARCH%\"\n",
    "\n",
    "echo %PATH% | find /c /i \"%VS_PATH%\" > nul || if \"%PATH:~-1%\" == \"\\\" (setx PATH \"%VS_PATH%;%PATH%\\\") else (setx PATH \"%VS_PATH%;%PATH%\")\n",
    "\"\"\"\n",
    "\n",
    "# Setup files can still be created even if package download/extraction is set to false\n",
    "# This allows to quickly recreate setup files in case something minor has changed (path, etc)\n",
    "# This can also be used to register build tools created by other similar projects\n",
    "# However, the compatibility is obviously not guaranteed and should be rechecked\n",
    "if not os.path.exists(install_path):\n",
    "    force_create_setup = input('Nothing is extracted, do you still want to create setup files (Y/N)?')\n",
    "    if lower(force_create_setup) != 'y':\n",
    "        raise KeyboardInterrupt('Not continuing because there is nothing to create')\n",
    "    else:\n",
    "        warn(f'Forcibly creating \"{install_path}\", set \"vs_package_extract\" to true if this is not intended!')\n",
    "        os.makedirs(install_path)\n",
    "\n",
    "with open(f'{install_path}\\\\vs_env_setup.bat', 'w') as f:\n",
    "    f.write(batch_setup)\n",
    "\n",
    "print(f'{install_path}\\\\vs_env_setup.bat')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Register Visual Studio and Windows SDK\n",
    "- Just a minimal setup to trick `vswhere`\n",
    "- Will write some values to Windows registry\n",
    "- Make sure Visual Studio is not installed to prevent accidental overwrite"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "These files must be run as admin\n",
      "C:\\BuildTools\\vs_register.bat\n",
      "C:\\BuildTools\\vs_unregister.bat\n"
     ]
    }
   ],
   "source": [
    "# Use only x86 version of the dll and regsvr32\n",
    "# It seems that \"vswhere\" may ignore the x64 version\n",
    "regsvr_path = '%SYSTEMROOT%\\\\SysWoW64\\\\regsvr32.exe'\n",
    "regsvr_path = '\"%SYSTEMROOT%\\\\SysWoW64\\\\regsvr32\"' if os.path.isfile(os.path.expandvars(regsvr_path)) else 'regsvr32'\n",
    "setupdll_path = f'VisualStudio\\\\Setup\\\\x86\\\\Microsoft.VisualStudio.Setup.Configuration.Native.dll'\n",
    "\n",
    "if vs_package_download:\n",
    "    for i in pyl_list:\n",
    "        package_id = i['packageId']\n",
    "        file_path = i['filePath']\n",
    "\n",
    "        if lower(package_id) == lower('Microsoft.VisualStudio.Setup.Configuration'):\n",
    "            # Extract setup dlls (needed by \"vswhere\")\n",
    "            subprocess.check_call(\n",
    "                # FIXME Will fail if there is space in path\n",
    "                ['msiexec.exe', '/a', file_path, '/quiet', '/qn', f'TARGETDIR={install_path}\\\\temp']\n",
    "            )\n",
    "            # Copy setup dlls and delete the temp folder\n",
    "            shutil.copytree(\n",
    "                src = f'{install_path}\\\\temp\\\\ProgramData\\\\Microsoft\\\\VisualStudio',\n",
    "                dst = f'{install_path}\\\\VisualStudio',\n",
    "                dirs_exist_ok = True\n",
    "            )\n",
    "            rm_auto(f'{install_path}\\\\temp', recursive = True)\n",
    "            break\n",
    "\n",
    "# Most tools will search Windows SDK location from registry\n",
    "hklm_win_sdk = \"HKLM\\\\SOFTWARE\\\\Microsoft\\\\Microsoft SDKs\"\n",
    "# To resolve \"urctd.lib\" not found error (still need UCRT debug dlls)\n",
    "hklm_win_kits = \"HKLM\\\\SOFTWARE\\\\Microsoft\\\\Windows Kits\"\n",
    "\n",
    "# Registry values to be written and removed\n",
    "# Will be dangerous if you already have VS or Windows SDK installed\n",
    "# FIXME Migrate to PowerShell and don't use cmd tools (reg.exe, regsvr32.exe)\n",
    "reg_add = fr\"\"\"@echo off\n",
    "echo This will create/modify some registry values PERMANENTLY.\n",
    "echo You CAN'T undo this action if something goes wrong.\n",
    "choice /c yn /m \"Are you sure you want to continue\"\n",
    "if %ERRORLEVEL% == 2 exit /b\n",
    "\n",
    "set ROOT=%~dp0\n",
    "\n",
    "{regsvr_path} \"%ROOT%{setupdll_path}\"\n",
    "reg add \"HKLM\\SOFTWARE\\Microsoft\\VisualStudio\\Setup\" /v CachePath /d \"%ROOT%VisualStudio\\Packages\" /f\n",
    "reg add \"{hklm_win_sdk}\\Windows\\v{sdk_ver_major}.{sdk_ver_minor}\" /v InstallationFolder /d \"%ROOT%Windows Kits\\{sdk_ver_auto}\\\\\" /f /reg:32\n",
    "reg add \"{hklm_win_sdk}\\Windows\\v{sdk_ver_major}.{sdk_ver_minor}\" /v ProductName /d \"Microsoft Windows SDK for Windows {sdk_ver_sem}\" /f /reg:32\n",
    "reg add \"{hklm_win_sdk}\\Windows\\v{sdk_ver_major}.{sdk_ver_minor}\" /v ProductVersion /d \"{sdk_ver_sem}\" /f /reg:32\n",
    "reg add \"{hklm_win_kits}\\Installed Roots\" /v KitsRoot{sdk_ver_auto.replace('.', '')} /d \"%ROOT%Windows Kits\\{sdk_ver_auto}\\\\\" /f /reg:32\n",
    "\"\"\"\n",
    "reg_rm = fr\"\"\"@echo off\n",
    "echo This will remove some registry keys PERMANENTLY.\n",
    "echo You CAN'T undo this action if something goes wrong.\n",
    "choice /c yn /m \"Are you sure you want to continue\"\n",
    "if %ERRORLEVEL% == 2 exit /b\n",
    "\n",
    "set ROOT=%~dp0\n",
    "\n",
    "{regsvr_path} /u \"%ROOT%{setupdll_path}\"\n",
    "reg delete \"HKLM\\SOFTWARE\\Microsoft\\VisualStudio\" /f\n",
    "reg delete \"{hklm_win_sdk}\" /f /reg:32\n",
    "reg delete \"{hklm_win_kits}\" /f /reg:32\n",
    "\"\"\"\n",
    "\n",
    "if not os.path.isfile(f'{install_path}\\\\{setupdll_path}'):\n",
    "    warn(f'\"{setupdll_path}\" does not exist, \"DllRegisterServer\" will fail!')\n",
    "\n",
    "# The batch file will need to be run as admin\n",
    "# It's possible to automatically elevate as admin by adding garbage lines,\n",
    "# But I don't really like it and that may not work correctly on Linux\n",
    "with open(f'{install_path}\\\\vs_register.bat', 'w') as file:\n",
    "    file.write(reg_add)\n",
    "\n",
    "with open(f'{install_path}\\\\vs_unregister.bat', 'w') as file:\n",
    "    file.write(reg_rm)\n",
    "\n",
    "print('These files must be run as admin')\n",
    "print(f'{install_path}\\\\vs_register.bat')\n",
    "print(f'{install_path}\\\\vs_unregister.bat')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C:\\BuildTools\\VisualStudio\\Packages\\_Instances\\ab123de45\\state.json\n"
     ]
    }
   ],
   "source": [
    "instance_id = 'ab123de45'\n",
    "instances_path = f'{install_path}\\\\VisualStudio\\\\Packages\\\\_Instances'\n",
    "\n",
    "# Make directories for current instance\n",
    "os.makedirs(f'{instances_path}\\\\{instance_id}', exist_ok = True)\n",
    "\n",
    "# There are more than 1 files, but \"state.json\" is the most important one\n",
    "# There is also \"state.packages.json\" that's probably needed if using \"require\" on \"vswhere\"\n",
    "state_json = {\n",
    "    \"icon\": {\n",
    "\t\t\"mimeType\": \"image/svg+xml\",\n",
    "\t\t\"fileName\": \"product.svg\"\n",
    "\t},\n",
    "\t\"updateDate\": \"1970-01-01T00:00:01.0000000Z\",\n",
    "\t\"installDate\": \"1970-01-01T00:00:01.0000000Z\",\n",
    "\t\"enginePath\": \"\",\n",
    "    \"installationName\": manifest['info']['id'],\n",
    "    \"catalogInfo\": manifest['info'],\n",
    "    # Installation path must exist in order to be detected by \"vswhere\"\n",
    "    # The path is always absolute so don't forget to change it when moving directory\n",
    "    \"installationPath\": install_path,\n",
    "    \"launchParams\": {\n",
    "\t\t\"fileName\": \"Common7\\\\Tools\\\\LaunchDevCmd.bat\"\n",
    "\t},\n",
    "    \"installationVersion\": manifest['info']['buildVersion'],\n",
    "    \"channelId\": channel_info['info']['manifestName'],\n",
    "    \"installedChannelId\": channel_info['info']['manifestName'],\n",
    "    \"channelUri\": f\"https://aka.ms/vs/{vs_ver_major}/{vs_channel}/channel\",\n",
    "\t\"installedChannelUri\": f\"https://aka.ms/vs/{vs_ver_major}/{vs_channel}/channel\",\n",
    "    \"seed\": {\n",
    "        \"language\": [ i for i in vs_lang ]\n",
    "    },\n",
    "    \"localizedResources\": [ j for i in channel_info['channelItems'] if i['id'] == vs_product_id for j in i['localizedResources'] ],\n",
    "    # Not sure where's this from, but no way I'm copying it all\n",
    "    \"channelResources\": [\n",
    "        {\n",
    "\t\t\t\"language\": i,\n",
    "\t\t\t\"title\": 'Current' if lower(vs_channel) == 'release' else 'Preview',\n",
    "\t\t\t\"description\": \"\",\n",
    "\t\t\t\"channelSuffix\": \"\"\n",
    "\t\t} for i in vs_lang\n",
    "    ],\n",
    "    \"product\": {\n",
    "        # Build tools may not appear on \"vswhere\" output unless \"products *\" is specified\n",
    "        # For example, CMake will use \"vswhere -format json\" which by default ignore build tools\n",
    "        # If you have problems, change \"vs_product_id\" to \"Microsoft.VisualStudio.Product.Community\"\n",
    "        \"id\": \"Microsoft.VisualStudio.Product.Community\",\n",
    "\t\t\"version\": manifest['info']['buildVersion'],\n",
    "\t\t\"type\": \"Product\",\n",
    "\t\t\"installed\": True,\n",
    "\t\t\"supportsExtensions\": False\n",
    "\t},\n",
    "    \"properties\": {\n",
    "\t\t\"campaignId\": \"\",\n",
    "\t\t\"setupEngineFilePath\": \"\",\n",
    "\t\t\"nickname\": \"\",\n",
    "\t\t\"channelManifestId\": channel_info['info']['id']\n",
    "\t},\n",
    "    \"selectedPackages\": [\n",
    "        {\n",
    "            \"id\": i['id'],\n",
    "            \"version\": i['version'],\n",
    "            \"type\": \"Component\",\n",
    "            \"selectedState\": \"GroupSelected\",\n",
    "            \"userSelectedState\": \"Implicit\"\n",
    "        } for i in manifest['packages'] if lower(i['id']) in lower(dp_components)\n",
    "    ],\n",
    "    # Not sure where's this from, I'm currently using generic link\n",
    "    \"releaseNotes\": \"https://docs.microsoft.com/en-us/visualstudio/releases\",\n",
    "    \"thirdPartyNotices\": \"https://visualstudio.microsoft.com/license-terms\"\n",
    "}\n",
    "\n",
    "with open(f'{instances_path}\\\\{instance_id}\\\\state.json', 'w', encoding = 'utf-8') as file:\n",
    "    state_json = json.dumps(state_json, indent = 4, ensure_ascii = False)\n",
    "    file.write(state_json)\n",
    "\n",
    "print(f'{instances_path}\\\\{instance_id}\\\\state.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C:\\BuildTools\\VisualStudio\\Packages\\_Instances\\ab123de45\\state.packages.json\n"
     ]
    }
   ],
   "source": [
    "state_packages_json = {\n",
    "    'packages': []\n",
    "}\n",
    "\n",
    "# Get final dependencies\n",
    "# All dependencies that have payloads\n",
    "for id, props in dp_final.items():\n",
    "    packages = {\n",
    "        'id': id,\n",
    "        'version': props['version'],\n",
    "        'type': props['type']\n",
    "    }\n",
    "\n",
    "    # Optional keys (may not always exist)\n",
    "    if props.get('language'): packages['language'] = props.get('language')\n",
    "    if props.get('chip'): packages['chip'] = props.get('chip')\n",
    "    if props.get('machineArch'): packages['machineArch'] = props.get('machineArch')\n",
    "    if props.get('productArch'): packages['productArch'] = props.get('productArch')\n",
    "\n",
    "    state_packages_json['packages'].append(packages)\n",
    "\n",
    "# Get grouped dependencies\n",
    "# They're either a workload or component\n",
    "for i in manifest['packages']:\n",
    "    if lower(i['id']) in lower(dp_components) or lower(i['id']) in lower(wl_needed):\n",
    "        packages = {\n",
    "            'id': i['id'],\n",
    "            'version': i['version'],\n",
    "            'type': i['type']\n",
    "        }\n",
    "\n",
    "        state_packages_json['packages'].append(packages)\n",
    "\n",
    "with open(f'{instances_path}\\\\{instance_id}\\\\state.packages.json', 'w', encoding = 'utf-8') as file:\n",
    "    state_packages_json = json.dumps(state_packages_json, indent = 4, ensure_ascii = False)\n",
    "    file.write(state_packages_json)\n",
    "\n",
    "print(f'{instances_path}\\\\{instance_id}\\\\state.packages.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C:\\BuildTools\\VisualStudio\\Packages\\_Instances\\ab123de45\\catalog.json\n"
     ]
    }
   ],
   "source": [
    "# \"catalog.json\" is basically a cached manifest\n",
    "# Not important but I'm including it for historical purpose\n",
    "dump = json.dumps(dict(manifest), indent = 4, ensure_ascii = False)\n",
    "with open(f'{instances_path}\\\\{instance_id}\\\\catalog.json', 'wb') as file:\n",
    "    file.write(dump.encode())\n",
    "\n",
    "print(f'{instances_path}\\\\{instance_id}\\\\catalog.json')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}