FEAT: Support tcp store for rendezvous in collective communication

.clang-format

@@ -0,0 +1,21 @@
+# See all possible options and defaults with:
+# clang-format --style=llvm --dump-config
+BasedOnStyle: LLVM
+AccessModifierOffset: -4
+AllowShortLambdasOnASingleLine: Inline
+AlwaysBreakTemplateDeclarations: Yes
+BinPackArguments: false
+BinPackParameters: false
+BreakBeforeBinaryOperators: All
+BreakConstructorInitializers: BeforeColon
+ColumnLimit: 80
+SpacesBeforeTrailingComments: 2
+IncludeBlocks: Regroup
+IndentCaseLabels: true
+IndentPPDirectives: AfterHash
+IndentWidth: 4
+Language: Cpp
+SpaceAfterCStyleCast: true
+Standard: c++20
+StatementMacros: ['PyObject_HEAD']
+TabWidth: 4

.cmake-format.yaml

@@ -0,0 +1,73 @@
+parse:
+  additional_commands:
+    pybind11_add_module:
+      flags:
+        - THIN_LTO
+        - MODULE
+        - SHARED
+        - NO_EXTRAS
+        - EXCLUDE_FROM_ALL
+        - SYSTEM
+
+format:
+  line_width: 99
+  tab_size: 2
+
+  # If an argument group contains more than this many sub-groups
+  # (parg or kwarg groups) then force it to a vertical layout.
+  max_subgroups_hwrap: 2
+
+  # If a positional argument group contains more than this many
+  # arguments, then force it to a vertical layout.
+  max_pargs_hwrap: 6
+
+  # If a cmdline positional group consumes more than this many
+  # lines without nesting, then invalidate the layout (and nest)
+  max_rows_cmdline: 2
+  separate_ctrl_name_with_space: false
+  separate_fn_name_with_space: false
+  dangle_parens: false
+
+  # If the trailing parenthesis must be 'dangled' on its on
+  # 'line, then align it to this reference: `prefix`: the start'
+  # 'of the statement,  `prefix-indent`: the start of the'
+  # 'statement, plus one indentation  level, `child`: align to'
+  # the column of the arguments
+  dangle_align: prefix
+  # If the statement spelling length (including space and
+  # parenthesis) is smaller than this amount, then force reject
+  # nested layouts.
+  min_prefix_chars: 4
+
+  # If the statement spelling length (including space and
+  # parenthesis) is larger than the tab width by more than this
+  # amount, then force reject un-nested layouts.
+  max_prefix_chars: 10
+
+  # If a candidate layout is wrapped horizontally but it exceeds
+  # this many lines, then reject the layout.
+  max_lines_hwrap: 2
+
+  line_ending: unix
+
+  # Format command names consistently as 'lower' or 'upper' case
+  command_case: canonical
+
+  # Format keywords consistently as 'lower' or 'upper' case
+  # unchanged is valid too
+  keyword_case: 'upper'
+
+  # A list of command names which should always be wrapped
+  always_wrap: []
+
+  # If true, the argument lists which are known to be sortable
+  # will be sorted lexicographically
+  enable_sort: true
+
+  # If true, the parsers may infer whether or not an argument
+  # list is sortable (without annotation).
+  autosort: false
+
+# Causes a few issues - can be solved later, possibly.
+markup:
+  enable_markup: false

.github/workflows/python.yaml

@@ -49,6 +49,21 @@ jobs:
       - name: codespell
         run: pip install codespell && cd python && codespell xoscar
 
+      - name: clang-format
+        uses: jidicula/[email protected]
+        with:
+          clang-format-version: '14'
+          fallback-style: 'LLVM'
+          check-path: 'cpp'
+      - name: cpplint
+        run: |
+          pip install cpplint
+          cpplint --recursive cpp
+      - name: cmake-format
+        run: |
+          pip install cmakelang[YAML]
+          find . -name "CMakeLists.txt" -not -path "*third_party/*" | xargs cmake-format -c .cmake-format.yaml --check
+
   build_test_job:
     runs-on: ${{ matrix.os }}
     needs: lint
@@ -85,6 +100,12 @@ jobs:
         python-version: ${{ matrix.python-version }}
         activate-environment: ${{ env.CONDA_ENV }}
 
+    # Fix "version `GLIBCXX_3.4.30' not found (required by xoscar_store.cpython-311-x86_64-linux-gnu.so)" issue in Python 3.11
+    - name: Install libstdcxx-ng for Python 3.11
+      if: ${{ (matrix.module != 'gpu') && (matrix.os == 'ubuntu-latest') && (matrix.python-version == '3.11') }}
+      run: |
+        conda install -c conda-forge libstdcxx-ng
+
     - name: Install dependencies
       env:
         MODULE: ${{ matrix.module }}

.gitignore

@@ -132,3 +132,10 @@ dmypy.json
 
 # cython compiled files
 python/xoscar/**/*.c*
+
+# cmake
+cmake-*
+CMakeFiles
+CMakeCache.txt
+*.cmake
+Makefile

.gitmodules

@@ -0,0 +1,6 @@
+[submodule "third_party/fmt"]
+	path = third_party/fmt
+	url = https://github.com/fmtlib/fmt.git
+[submodule "third_party/pybind11"]
+	path = third_party/pybind11
+	url = https://github.com/pybind/pybind11.git

.pre-commit-config.yaml

@@ -1,32 +1,58 @@
-files: python/xoscar
 repos:
   - repo: https://github.com/psf/black
     rev: 23.1.0
     hooks:
       - id: black
+        files: python/xoscar
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.4.0
     hooks:
       - id: end-of-file-fixer
+        files: python/xoscar
       - id: trailing-whitespace
+        files: python/xoscar
   - repo: https://github.com/PyCQA/flake8
     rev: 6.0.0
     hooks:
       - id: flake8
         args: [--config, python/setup.cfg]
+        files: python/xoscar
   - repo: https://github.com/pycqa/isort
     rev: 5.12.0
     hooks:
       - id: isort
         args: [--sp, python/setup.cfg]
+        files: python/xoscar
   - repo: https://github.com/pre-commit/mirrors-mypy
     rev: v1.3.0
     hooks:
       - id: mypy
         additional_dependencies: [tokenize-rt==3.2.0]
         args: [--ignore-missing-imports, --follow-imports, skip]
+        files: python/xoscar
   - repo: https://github.com/codespell-project/codespell
     rev: v2.2.2
     hooks:
       - id: codespell
         args: [ --config, python/setup.cfg]
+        files: python/xoscar
+
+  - repo: https://github.com/pre-commit/mirrors-clang-format
+    rev: "v15.0.7"
+    hooks:
+      - id: clang-format
+        files: cpp
+
+  - repo: https://github.com/cheshirekow/cmake-format-precommit
+    rev: "v0.6.13"
+    hooks:
+      - id: cmake-format
+        additional_dependencies: [ pyyaml ]
+        types: [ file ]
+        files: (\.cmake|CMakeLists.txt)(.in)?$
+
+  - repo: https://github.com/pocc/pre-commit-hooks
+    rev: v1.3.5
+    hooks:
+      - id: cpplint
+        files: cpp

.readthedocs.yaml

@@ -8,6 +8,8 @@ build:
   os: ubuntu-20.04
   tools:
     python: "3.9"
+  apt_packages:
+    - cmake
 
 python:
   install:

CMakeLists.txt

@@ -0,0 +1,18 @@
+cmake_minimum_required(VERSION 3.11...3.21)
+
+project(XoscarCollective)
+
+if(NOT DEFINED PYTHON_PATH)
+  find_package(Python COMPONENTS Interpreter Development)
+else()
+  set(PYTHON_EXECUTABLE ${PYTHON_PATH})
+endif()
+
+set(CMAKE_CXX_STANDARD 20)
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC")
+
+include_directories(${CMAKE_SOURCE_DIR})
+
+add_subdirectory(third_party/fmt)
+add_subdirectory(third_party/pybind11)
+add_subdirectory(cpp)

CPPLINT.cfg

@@ -0,0 +1 @@
+filter=-build/c++11,-build/include_subdir,-build/include_order,-build/include_what_you_use,-readability/todo,-readability/nolint,-runtime/int,-runtime/references,-whitespace/indent

cpp/CMakeLists.txt

@@ -0,0 +1,16 @@
+cmake_minimum_required(VERSION 3.11...3.21)
+
+project(XoscarCollective)
+set(CMAKE_CXX_STANDARD 20)
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC")
+if(NOT DEFINED LIBRARY_OUTPUT_DIRECTORY)
+  set(LIBRARY_OUTPUT_DIRECTORY ${CMAKE_SOURCE_DIR}/python/xoscar/collective/rendezvous)
+endif()
+
+include_directories(${CMAKE_SOURCE_DIR}/cpp/collective/rendezvous/include)
+
+add_subdirectory(collective/rendezvous)
+
+pybind11_add_module(xoscar_store collective/rendezvous/src/bind_tcp_store.cpp)
+target_link_libraries(xoscar_store PRIVATE StoreLib fmt::fmt)
+set_target_properties(xoscar_store PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${LIBRARY_OUTPUT_DIRECTORY})

cpp/collective/rendezvous/CMakeLists.txt

@@ -0,0 +1,25 @@
+cmake_minimum_required(VERSION 3.11...3.21)
+
+project(
+  XoscarRendezvous
+  VERSION 0.0.1
+  LANGUAGES CXX)
+
+set(CMAKE_CXX_STANDARD 20)
+
+include_directories(include)
+include_directories(../../../third_party/fmt/include)
+
+add_library(
+  StoreLib
+  include/error.h
+  include/exception.h
+  src/exception.cpp
+  include/socket.h
+  src/socket.cpp
+  include/store.hpp
+  src/store.cpp
+  include/tcp_store.hpp
+  src/tcp_store.cpp
+  include/unix_sock_utils.hpp
+  include/utils.hpp)

cpp/collective/rendezvous/LICENSE

@@ -0,0 +1,83 @@
+Apache 2.0 License with Code from PyTorch Repository
+
+This software includes code primarily sourced from the PyTorch repository, which is governed by its own original license.
+The original license for the PyTorch repository, as well as the additional terms below, apply to the code in this software.
+
+-----------------------------------------------------------------------------
+From PyTorch:
+
+Copyright (c) 2016-     Facebook, Inc            (Adam Paszke)
+Copyright (c) 2014-     Facebook, Inc            (Soumith Chintala)
+Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert)
+Copyright (c) 2012-2014 Deepmind Technologies    (Koray Kavukcuoglu)
+Copyright (c) 2011-2012 NEC Laboratories America (Koray Kavukcuoglu)
+Copyright (c) 2011-2013 NYU                      (Clement Farabet)
+Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou, Iain Melvin, Jason Weston)
+Copyright (c) 2006      Idiap Research Institute (Samy Bengio)
+Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert, Samy Bengio, Johnny Mariethoz)
+
+From Caffe2:
+
+Copyright (c) 2016-present, Facebook Inc. All rights reserved.
+
+All contributions by Facebook:
+Copyright (c) 2016 Facebook Inc.
+
+All contributions by Google:
+Copyright (c) 2015 Google Inc.
+All rights reserved.
+
+All contributions by Yangqing Jia:
+Copyright (c) 2015 Yangqing Jia
+All rights reserved.
+
+All contributions by Kakao Brain:
+Copyright 2019-2020 Kakao Brain
+
+All contributions by Cruise LLC:
+Copyright (c) 2022 Cruise LLC.
+All rights reserved.
+
+All contributions from Caffe:
+Copyright(c) 2013, 2014, 2015, the respective contributors
+All rights reserved.
+
+All other contributions:
+Copyright(c) 2015, 2016 the respective contributors
+All rights reserved.
+
+Caffe2 uses a copyright model similar to Caffe: each contributor holds
+copyright over their contributions to Caffe2. The project versioning records
+all such contribution and copyright details. If a contributor wants to further
+mark their specific copyright on a particular contribution, they should
+indicate their copyright solely in the commit message of the change when it is
+committed.
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+
+3. Neither the names of Facebook, Deepmind Technologies, NYU, NEC Laboratories America
+   and IDIAP Research Institute nor the names of its contributors may be
+   used to endorse or promote products derived from this software without
+   specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.