From c1a262681dd2ecb9a0dadc8a0da04cd6b69fdff0 Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Wed, 27 Mar 2024 17:13:50 -0700 Subject: [PATCH] Release for NSDI'24 --- .github/workflows/build.yml | 23 + .gitignore | 47 + LICENSE | 201 ++ Makefile | 197 ++ README.md | 94 + apps/FeatureExtraction/.gitignore | 9 + apps/FeatureExtraction/Makefile | 49 + apps/FeatureExtraction/inc/constants.hpp | 21 + apps/FeatureExtraction/inc/fake_backend.hpp | 75 + apps/FeatureExtraction/inc/feat_extractor.hpp | 36 + apps/FeatureExtraction/inc/midas_utils.hpp | 44 + apps/FeatureExtraction/inc/redis_utils.hpp | 36 + apps/FeatureExtraction/inc/socket.hpp | 182 ++ apps/FeatureExtraction/inc/utils.hpp | 76 + apps/FeatureExtraction/inc/zipf.hpp | 97 + apps/FeatureExtraction/run.sh | 14 + apps/FeatureExtraction/server/server.py | 82 + .../server/vision_model_loader.py | 195 ++ apps/FeatureExtraction/src/client.cpp | 29 + apps/FeatureExtraction/src/feat_extractor.cpp | 193 ++ apps/FeatureExtraction/tools/gen_feats.cpp | 33 + apps/HDSearch/.gitignore | 4 + apps/HDSearch/Makefile | 47 + apps/HDSearch/inc/constants.hpp | 28 + apps/HDSearch/inc/fake_backend.hpp | 78 + apps/HDSearch/inc/feat_extractor.hpp | 62 + apps/HDSearch/inc/utils.hpp | 75 + apps/HDSearch/run.sh | 7 + apps/HDSearch/src/feat_extractor.cpp | 189 ++ apps/HDSearch/src/main.cpp | 138 + apps/OneRF/.gitignore | 2 + apps/OneRF/Makefile | 44 + apps/OneRF/inc/backend.hpp | 17 + apps/OneRF/inc/fback.hpp | 59 + apps/OneRF/inc/mysql.hpp | 35 + apps/OneRF/inc/nuapp.hpp | 11 + apps/OneRF/inc/query_types.hpp | 59 + apps/OneRF/inc/requestor.hpp | 58 + apps/OneRF/inc/semaphore.hpp | 20 + apps/OneRF/rh_server/main.cpp | 12 + apps/OneRF/rh_server/rh_nuapp.cpp | 265 ++ apps/OneRF/rh_server/rh_nuapp.hpp | 95 + apps/OneRF/src/fback.cpp | 131 + apps/OneRF/src/mysql.cpp | 66 + apps/OneRF/src/requestor.cpp | 213 ++ apps/OneRF/src/semaphore.cpp | 26 + apps/build_all.sh | 13 + apps/storage/.gitignore | 2 + apps/storage/Makefile | 36 + apps/storage/run.sh | 5 + apps/storage/server.cpp | 136 + apps/storage/server.hpp | 79 + apps/storage/setup_disk.sh | 6 + apps/synthetic/.gitignore | 1 + apps/synthetic/Makefile | 36 + apps/synthetic/main.cpp | 131 + bin/.gitkeep | 0 bindings/c/.gitignore | 45 + bindings/c/Makefile | 67 + bindings/c/cache_manager_ffi.cpp | 85 + bindings/c/include/cache_manager.h | 38 + bindings/c/include/midas.h | 17 + bindings/c/include/midas_utils.h | 27 + bindings/c/include/resource_manager.h | 16 + bindings/c/include/softmem.h | 32 + bindings/c/resource_manager_ffi.cpp | 6 + bindings/c/softmem_ffi.cpp | 123 + bindings/c/test.c | 16 + config/.gitkeep | 0 daemon/daemon_types.cpp | 1008 +++++++ daemon/inc/daemon_types.hpp | 166 ++ daemon/inc/impl/daemon_types.ipp | 32 + daemon/main.cpp | 17 + exp/harvest_additional_memory/antagonist.sh | 21 + exp/harvest_additional_memory/set_memratio.sh | 6 + exp/intense_pressure/antagonist2.sh | 25 + exp/intense_pressure/set_memratio.sh | 6 + exp/moderate_pressure/antagonist3.sh | 28 + exp/moderate_pressure/set_memratio.sh | 6 + exp/softptr_cost/.gitignore | 1 + exp/softptr_cost/Makefile | 38 + exp/softptr_cost/cached_deref_cost.cpp | 85 + exp/softptr_cost/run-large.sh | 32 + exp/softptr_cost/run.sh | 10 + exp/softptr_cost/softptr_read_large.cpp | 115 + exp/softptr_cost/softptr_read_small.cpp | 106 + exp/softptr_cost/softptr_write_large.cpp | 111 + exp/softptr_cost/softptr_write_small.cpp | 107 + exp/softptr_cost/unique_ptr_read_large.cpp | 95 + exp/softptr_cost/unique_ptr_read_small.cpp | 88 + exp/softptr_cost/unique_ptr_write_large.cpp | 96 + exp/softptr_cost/unique_ptr_write_small.cpp | 90 + inc/array.hpp | 25 + inc/cache_manager.hpp | 138 + inc/construct_args.hpp | 12 + inc/evacuator.hpp | 64 + inc/fs_shim.hpp | 124 + inc/impl/array.ipp | 64 + inc/impl/cache_manager.ipp | 155 + inc/impl/evacuator.ipp | 32 + inc/impl/fs_shim.ipp | 102 + inc/impl/log.ipp | 155 + inc/impl/obj_locker.ipp | 54 + inc/impl/object.ipp | 512 ++++ inc/impl/qpair.ipp | 142 + inc/impl/resilient_func.ipp | 38 + inc/impl/resource_manager.ipp | 47 + inc/impl/sig_handler.ipp | 22 + inc/impl/slab.ipp | 39 + inc/impl/sync_hashmap.ipp | 259 ++ inc/impl/sync_kv.ipp | 784 +++++ inc/impl/sync_list.ipp | 107 + inc/impl/time.ipp | 46 + inc/impl/transient_ptr.ipp | 149 + inc/impl/victim_cache.ipp | 100 + inc/impl/zipf.ipp | 47 + inc/log.hpp | 130 + inc/logging.hpp | 83 + inc/obj_locker.hpp | 37 + inc/object.hpp | 263 ++ inc/perf.hpp | 87 + inc/qpair.hpp | 85 + inc/resilient_func.hpp | 26 + inc/resource_manager.hpp | 154 + inc/robinhood.h | 2544 +++++++++++++++++ inc/shm_types.hpp | 77 + inc/sig_handler.hpp | 34 + inc/slab.hpp | 120 + inc/sync_hashmap.hpp | 56 + inc/sync_kv.hpp | 181 ++ inc/sync_list.hpp | 42 + inc/time.hpp | 33 + inc/transient_ptr.hpp | 61 + inc/utils.hpp | 67 + inc/victim_cache.hpp | 61 + inc/zipf.hpp | 59 + koord/.gitignore | 3 + koord/Kbuild | 1 + koord/Makefile | 15 + koord/koord.c | 287 ++ koord/koord.h | 36 + koord/setup.sh | 8 + koord/user.c | 66 + scripts/build.sh | 15 + scripts/gen_compile_commands.sh | 10 + scripts/run_daemon.sh | 8 + scripts/set_memory_limit.sh | 10 + src/cache_manager.cpp | 106 + src/evacuator.cpp | 612 ++++ src/fs_shim.cpp | 490 ++++ src/log.cpp | 260 ++ src/object.cpp | 297 ++ src/perf.cpp | 304 ++ src/resilient_func.cpp | 130 + src/resource_manager.cpp | 527 ++++ src/sig_handler.cpp | 100 + src/slab.cpp | 118 + src/stacktrace.cpp | 70 + test/boost_shm_apis.cpp | 76 + test/test_batched_kv.cpp | 283 ++ test/test_cache_manager.cpp | 34 + test/test_concurrent_evacuator.cpp | 117 + test/test_concurrent_evacuator2.cpp | 118 + test/test_concurrent_evacuator3.cpp | 173 ++ test/test_feat_extractor.cpp | 534 ++++ test/test_feat_extractor_kv.cpp | 428 +++ test/test_fs_shim.cpp | 109 + test/test_hashmap_clear.cpp | 270 ++ test/test_large_alloc.cpp | 135 + test/test_log.cpp | 86 + test/test_memcpy.cpp | 116 + test/test_object.cpp | 41 + test/test_ordered_set.cpp | 295 ++ test/test_parallel_evacuator.cpp | 115 + test/test_resource_manager.cpp | 30 + test/test_sighandler.cpp | 50 + test/test_sighandler_proto.cpp | 62 + test/test_skewed_hashmap.cpp | 255 ++ test/test_slab.cpp | 47 + test/test_softptr_read_cost.cpp | 234 ++ test/test_softptr_write_cost.cpp | 231 ++ test/test_sync_hashmap.cpp | 277 ++ test/test_sync_kv.cpp | 282 ++ test/test_sync_list.cpp | 193 ++ test/test_victim_cache.cpp | 45 + 185 files changed, 22206 insertions(+) create mode 100644 .github/workflows/build.yml create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 README.md create mode 100644 apps/FeatureExtraction/.gitignore create mode 100644 apps/FeatureExtraction/Makefile create mode 100644 apps/FeatureExtraction/inc/constants.hpp create mode 100644 apps/FeatureExtraction/inc/fake_backend.hpp create mode 100644 apps/FeatureExtraction/inc/feat_extractor.hpp create mode 100644 apps/FeatureExtraction/inc/midas_utils.hpp create mode 100644 apps/FeatureExtraction/inc/redis_utils.hpp create mode 100644 apps/FeatureExtraction/inc/socket.hpp create mode 100644 apps/FeatureExtraction/inc/utils.hpp create mode 100644 apps/FeatureExtraction/inc/zipf.hpp create mode 100755 apps/FeatureExtraction/run.sh create mode 100644 apps/FeatureExtraction/server/server.py create mode 100644 apps/FeatureExtraction/server/vision_model_loader.py create mode 100644 apps/FeatureExtraction/src/client.cpp create mode 100644 apps/FeatureExtraction/src/feat_extractor.cpp create mode 100644 apps/FeatureExtraction/tools/gen_feats.cpp create mode 100644 apps/HDSearch/.gitignore create mode 100644 apps/HDSearch/Makefile create mode 100644 apps/HDSearch/inc/constants.hpp create mode 100644 apps/HDSearch/inc/fake_backend.hpp create mode 100644 apps/HDSearch/inc/feat_extractor.hpp create mode 100644 apps/HDSearch/inc/utils.hpp create mode 100755 apps/HDSearch/run.sh create mode 100644 apps/HDSearch/src/feat_extractor.cpp create mode 100644 apps/HDSearch/src/main.cpp create mode 100644 apps/OneRF/.gitignore create mode 100644 apps/OneRF/Makefile create mode 100644 apps/OneRF/inc/backend.hpp create mode 100644 apps/OneRF/inc/fback.hpp create mode 100644 apps/OneRF/inc/mysql.hpp create mode 100644 apps/OneRF/inc/nuapp.hpp create mode 100644 apps/OneRF/inc/query_types.hpp create mode 100644 apps/OneRF/inc/requestor.hpp create mode 100644 apps/OneRF/inc/semaphore.hpp create mode 100644 apps/OneRF/rh_server/main.cpp create mode 100644 apps/OneRF/rh_server/rh_nuapp.cpp create mode 100644 apps/OneRF/rh_server/rh_nuapp.hpp create mode 100644 apps/OneRF/src/fback.cpp create mode 100644 apps/OneRF/src/mysql.cpp create mode 100644 apps/OneRF/src/requestor.cpp create mode 100644 apps/OneRF/src/semaphore.cpp create mode 100755 apps/build_all.sh create mode 100644 apps/storage/.gitignore create mode 100644 apps/storage/Makefile create mode 100755 apps/storage/run.sh create mode 100644 apps/storage/server.cpp create mode 100644 apps/storage/server.hpp create mode 100755 apps/storage/setup_disk.sh create mode 100644 apps/synthetic/.gitignore create mode 100644 apps/synthetic/Makefile create mode 100644 apps/synthetic/main.cpp create mode 100644 bin/.gitkeep create mode 100644 bindings/c/.gitignore create mode 100644 bindings/c/Makefile create mode 100644 bindings/c/cache_manager_ffi.cpp create mode 100644 bindings/c/include/cache_manager.h create mode 100644 bindings/c/include/midas.h create mode 100644 bindings/c/include/midas_utils.h create mode 100644 bindings/c/include/resource_manager.h create mode 100644 bindings/c/include/softmem.h create mode 100644 bindings/c/resource_manager_ffi.cpp create mode 100644 bindings/c/softmem_ffi.cpp create mode 100644 bindings/c/test.c create mode 100644 config/.gitkeep create mode 100644 daemon/daemon_types.cpp create mode 100644 daemon/inc/daemon_types.hpp create mode 100644 daemon/inc/impl/daemon_types.ipp create mode 100644 daemon/main.cpp create mode 100644 exp/harvest_additional_memory/antagonist.sh create mode 100755 exp/harvest_additional_memory/set_memratio.sh create mode 100755 exp/intense_pressure/antagonist2.sh create mode 100755 exp/intense_pressure/set_memratio.sh create mode 100755 exp/moderate_pressure/antagonist3.sh create mode 100755 exp/moderate_pressure/set_memratio.sh create mode 100644 exp/softptr_cost/.gitignore create mode 100644 exp/softptr_cost/Makefile create mode 100644 exp/softptr_cost/cached_deref_cost.cpp create mode 100755 exp/softptr_cost/run-large.sh create mode 100755 exp/softptr_cost/run.sh create mode 100644 exp/softptr_cost/softptr_read_large.cpp create mode 100644 exp/softptr_cost/softptr_read_small.cpp create mode 100644 exp/softptr_cost/softptr_write_large.cpp create mode 100644 exp/softptr_cost/softptr_write_small.cpp create mode 100644 exp/softptr_cost/unique_ptr_read_large.cpp create mode 100644 exp/softptr_cost/unique_ptr_read_small.cpp create mode 100644 exp/softptr_cost/unique_ptr_write_large.cpp create mode 100644 exp/softptr_cost/unique_ptr_write_small.cpp create mode 100644 inc/array.hpp create mode 100644 inc/cache_manager.hpp create mode 100644 inc/construct_args.hpp create mode 100644 inc/evacuator.hpp create mode 100644 inc/fs_shim.hpp create mode 100644 inc/impl/array.ipp create mode 100644 inc/impl/cache_manager.ipp create mode 100644 inc/impl/evacuator.ipp create mode 100644 inc/impl/fs_shim.ipp create mode 100644 inc/impl/log.ipp create mode 100644 inc/impl/obj_locker.ipp create mode 100644 inc/impl/object.ipp create mode 100644 inc/impl/qpair.ipp create mode 100644 inc/impl/resilient_func.ipp create mode 100644 inc/impl/resource_manager.ipp create mode 100644 inc/impl/sig_handler.ipp create mode 100644 inc/impl/slab.ipp create mode 100644 inc/impl/sync_hashmap.ipp create mode 100644 inc/impl/sync_kv.ipp create mode 100644 inc/impl/sync_list.ipp create mode 100644 inc/impl/time.ipp create mode 100644 inc/impl/transient_ptr.ipp create mode 100644 inc/impl/victim_cache.ipp create mode 100644 inc/impl/zipf.ipp create mode 100644 inc/log.hpp create mode 100644 inc/logging.hpp create mode 100644 inc/obj_locker.hpp create mode 100644 inc/object.hpp create mode 100644 inc/perf.hpp create mode 100644 inc/qpair.hpp create mode 100644 inc/resilient_func.hpp create mode 100644 inc/resource_manager.hpp create mode 100644 inc/robinhood.h create mode 100644 inc/shm_types.hpp create mode 100644 inc/sig_handler.hpp create mode 100644 inc/slab.hpp create mode 100644 inc/sync_hashmap.hpp create mode 100644 inc/sync_kv.hpp create mode 100644 inc/sync_list.hpp create mode 100644 inc/time.hpp create mode 100644 inc/transient_ptr.hpp create mode 100644 inc/utils.hpp create mode 100644 inc/victim_cache.hpp create mode 100644 inc/zipf.hpp create mode 100644 koord/.gitignore create mode 100644 koord/Kbuild create mode 100644 koord/Makefile create mode 100644 koord/koord.c create mode 100644 koord/koord.h create mode 100755 koord/setup.sh create mode 100644 koord/user.c create mode 100755 scripts/build.sh create mode 100755 scripts/gen_compile_commands.sh create mode 100755 scripts/run_daemon.sh create mode 100755 scripts/set_memory_limit.sh create mode 100644 src/cache_manager.cpp create mode 100644 src/evacuator.cpp create mode 100644 src/fs_shim.cpp create mode 100644 src/log.cpp create mode 100644 src/object.cpp create mode 100644 src/perf.cpp create mode 100644 src/resilient_func.cpp create mode 100644 src/resource_manager.cpp create mode 100644 src/sig_handler.cpp create mode 100644 src/slab.cpp create mode 100644 src/stacktrace.cpp create mode 100644 test/boost_shm_apis.cpp create mode 100644 test/test_batched_kv.cpp create mode 100644 test/test_cache_manager.cpp create mode 100644 test/test_concurrent_evacuator.cpp create mode 100644 test/test_concurrent_evacuator2.cpp create mode 100644 test/test_concurrent_evacuator3.cpp create mode 100644 test/test_feat_extractor.cpp create mode 100644 test/test_feat_extractor_kv.cpp create mode 100644 test/test_fs_shim.cpp create mode 100644 test/test_hashmap_clear.cpp create mode 100644 test/test_large_alloc.cpp create mode 100644 test/test_log.cpp create mode 100644 test/test_memcpy.cpp create mode 100644 test/test_object.cpp create mode 100644 test/test_ordered_set.cpp create mode 100644 test/test_parallel_evacuator.cpp create mode 100644 test/test_resource_manager.cpp create mode 100644 test/test_sighandler.cpp create mode 100644 test/test_sighandler_proto.cpp create mode 100644 test/test_skewed_hashmap.cpp create mode 100644 test/test_slab.cpp create mode 100644 test/test_softptr_read_cost.cpp create mode 100644 test/test_softptr_write_cost.cpp create mode 100644 test/test_sync_hashmap.cpp create mode 100644 test/test_sync_kv.cpp create mode 100644 test/test_sync_list.cpp create mode 100644 test/test_victim_cache.cpp diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..56264a0 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,23 @@ +name: Build + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - uses: actions/cache@v2 + with: + key: ${{ github.ref }} + path: .cache + - name: Install dependencies + run: sudo apt-get update && sudo apt-get install -y libboost-all-dev gcc g++ + - name: Build + run: make -j diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..234e1ea --- /dev/null +++ b/.gitignore @@ -0,0 +1,47 @@ +**/libmidas++.a +**/libmidas++.so + +config/mem.config +config/policy.config + +# Prerequisites +*.d + +# Compiled Object files +*.slo +*.lo +*.o +*.obj + +# Precompiled Headers +*.gch +*.pch + +# Compiled Dynamic libraries +*.so +*.dylib +*.dll + +# Fortran module files +*.mod +*.smod + +# Compiled Static libraries +*.lai +*.la +*.a +*.lib + +# Executables +bin/* +*.exe +*.out +*.app + +# Clangd index +**/.cache/* +**/compile_commands.json +**/.clangd/* + +# vscode +**/.vscode/* \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..3c55cab --- /dev/null +++ b/Makefile @@ -0,0 +1,197 @@ +CXXFLAGS += -std=c++1z -O2 +CXXFLAGS += -march=native # required for avx-enhanced rmemcpy +CXXFLAGS += -fPIC +# CXXFLAGS += -Wall -Wextra +# CXXFLAGS += -g -O0 +CXX = g++ +LDXX = g++ + +INC += -Iinc + +override LDFLAGS += -lrt -lpthread +# For stacktrace logging +override LDFLAGS += -rdynamic +override LDFLAGS += -ldl + +lib_src = $(wildcard src/*.cpp) +lib_src := $(filter-out $(wildcard src/*main.cpp),$(lib_src)) +lib_obj = $(lib_src:.cpp=.o) + +test_src = $(wildcard test/test_*.cpp) +# test_src := $(filter-out $(wildcard test/boost*.cpp),$(test_src)) +test_target = $(test_src:.cpp=) + +daemon_src = $(wildcard daemon/*.cpp) +daemon_obj = $(daemon_src:.cpp=.o) + +src = $(lib_src) $(test_src) $(daemon_src) +obj = $(src:.cpp=.o) +dep = $(obj:.o=.d) + +daemon_main_src = $(daemon_src) +daemon_main_obj = $(daemon_obj) +test_resource_manager_src = test/test_resource_manager.cpp +test_resource_manager_obj = $(test_resource_manager_src:.cpp=.o) +test_object_src = test/test_object.cpp +test_object_obj = $(test_object_src:.cpp=.o) +test_slab_src = test/test_slab.cpp +test_slab_obj = $(test_slab_src:.cpp=.o) +test_sync_hashmap_src = test/test_sync_hashmap.cpp +test_sync_hashmap_obj = $(test_sync_hashmap_src:.cpp=.o) +test_sync_list_src = test/test_sync_list.cpp +test_sync_list_obj = $(test_sync_list_src:.cpp=.o) +test_hashmap_clear_src = test/test_hashmap_clear.cpp +test_hashmap_clear_obj = $(test_hashmap_clear_src:.cpp=.o) +test_log_src = test/test_log.cpp +test_log_obj = $(test_log_src:.cpp=.o) +test_large_alloc_src = test/test_large_alloc.cpp +test_large_alloc_obj = $(test_large_alloc_src:.cpp=.o) +test_parallel_evacuator_src = test/test_parallel_evacuator.cpp +test_parallel_evacuator_obj = $(test_parallel_evacuator_src:.cpp=.o) +test_concurrent_evacuator_src = test/test_concurrent_evacuator.cpp +test_concurrent_evacuator_obj = $(test_concurrent_evacuator_src:.cpp=.o) +test_concurrent_evacuator2_src = test/test_concurrent_evacuator2.cpp +test_concurrent_evacuator2_obj = $(test_concurrent_evacuator2_src:.cpp=.o) +test_concurrent_evacuator3_src = test/test_concurrent_evacuator3.cpp +test_concurrent_evacuator3_obj = $(test_concurrent_evacuator3_src:.cpp=.o) +test_skewed_hashmap_src = test/test_skewed_hashmap.cpp +test_skewed_hashmap_obj = $(test_skewed_hashmap_src:.cpp=.o) +test_sighandler_src = test/test_sighandler.cpp +test_sighandler_obj = $(test_sighandler_src:.cpp=.o) +test_memcpy_src = test/test_memcpy.cpp +test_memcpy_obj = $(test_memcpy_src:.cpp=.o) +test_cache_manager_src = test/test_cache_manager.cpp +test_cache_manager_obj = $(test_cache_manager_src:.cpp=.o) +test_victim_cache_src = test/test_victim_cache.cpp +test_victim_cache_obj = $(test_victim_cache_src:.cpp=.o) +test_sync_kv_src = test/test_sync_kv.cpp +test_sync_kv_obj = $(test_sync_kv_src:.cpp=.o) +test_ordered_set_src = test/test_ordered_set.cpp +test_ordered_set_obj = $(test_ordered_set_src:.cpp=.o) +test_batched_kv_src = test/test_batched_kv.cpp +test_batched_kv_obj = $(test_batched_kv_src:.cpp=.o) +test_fs_shim_src = test/test_fs_shim.cpp +test_fs_shim_obj = $(test_fs_shim_src:.cpp=.o) +test_softptr_read_cost_src = test/test_softptr_read_cost.cpp +test_softptr_read_cost_obj = $(test_softptr_read_cost_src:.cpp=.o) +test_softptr_write_cost_src = test/test_softptr_write_cost.cpp +test_softptr_write_cost_obj = $(test_softptr_write_cost_src:.cpp=.o) + +test_feat_extractor_src = test/test_feat_extractor.cpp +test_feat_extractor_obj = $(test_feat_extractor_src:.cpp=.o) +test_feat_extractor_kv_src = test/test_feat_extractor_kv.cpp +test_feat_extractor_kv_obj = $(test_feat_extractor_kv_src:.cpp=.o) + +.PHONY: all bin lib daemon clean + +all: bin lib daemon + +lib: lib/libmidas++.a + +bin: bin/test_resource_manager bin/test_object bin/test_parallel_evacuator \ + bin/test_log bin/test_large_alloc \ + bin/test_sync_hashmap bin/test_hashmap_clear bin/test_sync_list \ + bin/test_cache_manager bin/test_victim_cache \ + bin/test_sync_kv bin/test_ordered_set bin/test_batched_kv \ + bin/test_skewed_hashmap \ + bin/test_fs_shim \ + bin/test_sighandler \ + bin/test_memcpy \ + bin/test_softptr_read_cost bin/test_softptr_write_cost + +# bin/test_feat_extractor bin/test_feat_extractor_kv +# bin/test_concurrent_evacuator bin/test_concurrent_evacuator2 bin/test_concurrent_evacuator3 + +daemon: bin/daemon_main + +bin/daemon_main: $(daemon_main_obj) + $(LDXX) -o $@ $^ $(LDFLAGS) + +bin/test_resource_manager: $(test_resource_manager_obj) $(lib_obj) + $(LDXX) -o $@ $^ $(LDFLAGS) + +bin/test_object: $(test_object_obj) $(lib_obj) + $(LDXX) -o $@ $^ $(LDFLAGS) + +bin/test_slab: $(test_slab_obj) $(lib_obj) + $(LDXX) -o $@ $^ $(LDFLAGS) + +bin/test_sync_hashmap: $(test_sync_hashmap_obj) $(lib_obj) + $(LDXX) -o $@ $^ $(LDFLAGS) + +bin/test_sync_list: $(test_sync_list_obj) $(lib_obj) + $(LDXX) -o $@ $^ $(LDFLAGS) + +bin/test_hashmap_clear: $(test_hashmap_clear_obj) $(lib_obj) + $(LDXX) -o $@ $^ $(LDFLAGS) + +bin/test_log: $(test_log_obj) $(lib_obj) + $(LDXX) -o $@ $^ $(LDFLAGS) + +bin/test_large_alloc: $(test_large_alloc_obj) $(lib_obj) + $(LDXX) -o $@ $^ $(LDFLAGS) + +bin/test_parallel_evacuator: $(test_parallel_evacuator_obj) $(lib_obj) + $(LDXX) -o $@ $^ $(LDFLAGS) + +bin/test_concurrent_evacuator: $(test_concurrent_evacuator_obj) $(lib_obj) + $(LDXX) -o $@ $^ $(LDFLAGS) + +bin/test_concurrent_evacuator2: $(test_concurrent_evacuator2_obj) $(lib_obj) + $(LDXX) -o $@ $^ $(LDFLAGS) + +bin/test_concurrent_evacuator3: $(test_concurrent_evacuator3_obj) $(lib_obj) + $(LDXX) -o $@ $^ $(LDFLAGS) + +bin/test_skewed_hashmap: $(test_skewed_hashmap_obj) $(lib_obj) + $(LDXX) -o $@ $^ $(LDFLAGS) + +bin/test_sighandler: $(test_sighandler_obj) $(lib_obj) + $(LDXX) -o $@ $^ $(LDFLAGS) + +bin/test_memcpy: $(test_memcpy_obj) $(lib_obj) + $(LDXX) -o $@ $^ $(LDFLAGS) + +bin/test_cache_manager: $(test_cache_manager_obj) $(lib_obj) + $(LDXX) -o $@ $^ $(LDFLAGS) + +bin/test_victim_cache: $(test_victim_cache_obj) $(lib_obj) + $(LDXX) -o $@ $^ $(LDFLAGS) + +bin/test_sync_kv: $(test_sync_kv_obj) $(lib_obj) + $(LDXX) -o $@ $^ $(LDFLAGS) + +bin/test_ordered_set: $(test_ordered_set_obj) $(lib_obj) + $(LDXX) -o $@ $^ $(LDFLAGS) + +bin/test_batched_kv: $(test_batched_kv_obj) $(lib_obj) + $(LDXX) -o $@ $^ $(LDFLAGS) + +bin/test_fs_shim: $(test_fs_shim_obj) $(lib_obj) + $(LDXX) -o $@ $^ $(LDFLAGS) + +bin/test_softptr_read_cost: $(test_softptr_read_cost_obj) $(lib_obj) + $(LDXX) -o $@ $^ $(LDFLAGS) + +bin/test_softptr_write_cost: $(test_softptr_write_cost_obj) $(lib_obj) + $(LDXX) -o $@ $^ $(LDFLAGS) + +lib/libmidas++.a: $(lib_obj) + mkdir -p lib + $(AR) rcs $@ $^ + +# bin/test_feat_extractor: $(test_feat_extractor_obj) $(lib_obj) +# $(LDXX) -o $@ $^ -lcrypto $(LDFLAGS) + +# bin/test_feat_extractor_kv: $(test_feat_extractor_kv_obj) $(lib_obj) +# $(LDXX) -o $@ $^ -lcrypto $(LDFLAGS) + +%.o: %.cpp Makefile + $(CXX) $(CXXFLAGS) $(INC) -MMD -MP -c $< -o $@ + +ifneq ($(MAKECMDGOALS),clean) +-include $(dep) +endif + +clean: + $(RM) $(dep) $(obj) bin/* diff --git a/README.md b/README.md new file mode 100644 index 0000000..a834887 --- /dev/null +++ b/README.md @@ -0,0 +1,94 @@ +# Midas +![Status](https://img.shields.io/badge/Version-Experimental-green.svg) +[![Build](https://github.com/uclasystem/midas/actions/workflows/build.yml/badge.svg)](https://github.com/uclasystem/midas/actions/workflows/build.yml) +[![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) + + +Midas is a memory management system that enables applications to efficiently and safely harvest idle memory to store their soft state (e.g., caches, memoization results, etc.). It frees the developers from the burden of manually managing applications' soft state and stores the soft state that is most beneficial to each application, thereby improving both application performance and memory utilization. + +Midas currently supports C and C++ applications on Linux, with several real-world applications we have ported serving as practical references to demonstrate its simplicity and effectiveness. +For technical details about Midas, please refer to the [NSDI'24 paper](https://www.usenix.org/conference/nsdi24/presentation/qiao). + + +## Getting Started + +### System Requirements +Midas' code base assumes an x86-64 CPU architecture and the system is designed to run on Linux. Midas has been extensively tested on Ubuntu 18.04, 20.04, and 22.04, but it should also be compatible with other Linux distributions. + +### Dependencies + +Midas is developed in C++ and requires compilers with C++ 17 support (`std=c++1z`). To deploy Midas, the following dependencies are required: + +* gcc (>= 11) +* g++ (>= 11) +* Boost (>= 1.79.0) + +### Compile Midas +We have provided a push-button script to build Midas: +```bash +./scripts/build.sh +``` +The script will build the Midas library, the Midas coordinator (daemon), and all associated unit tests. This compilation typically completes within less than one minute. + +Users can also build Midas manually: +```bash +# Midas C++ static library, Midas coordinator (daemon), and unit tests. +make -j + +# Midas C library and bindings +cd bindings/c +make -j +make install # this will install the lib into bindings/c/lib +``` + +### Compile Ported Applications +We have also ported several applications to Midas under the `apps` directory. We have also offered the `apps/build_all.sh` script for automated compilation. + +```bash +cd apps/ +./build_all.sh +# The compiled executables are under each application's directory, respectively. +``` + +### Run +Before running any application, one needs to start the Midas coordinator on the host machine. The coordinator is responsible for managing the memory allocation and coordinate all running applications. + +```bash +./scripts/run_daemon.sh +# Example terminal outputs: +# [kInfo]: Daemon starts listening... +``` +By default, the Midas coordinator is configured to utilize all available memory on the server. This is recommended because it maximized the utility for applications. However, for users who wish to manage system resources more conservatively, there is an option to set a hard limit on the maximum memory capacity that Midas can harvest: +```bash +./scripts/set_memory_limit.sh +``` + +With the Midas coordinator running, users can launch applications and take advantage of soft state. Below is an example of how to start a storage server application ported to work with Midas: +```bash +cd apps/storage +./storage_server +``` +Users can run more applications following a similar process. + +## Repo Structure + +```txt +Github Repo Root +├── apps # Ported applications. +├── bin # Compiled unit tests. +├── bindings # Bindings for the other languages. Currently only C is supported. +├── config # Configuration files for the Midas coordinator(daemon). +├── daemon # Midas coordinator that runs as a daemon process. +├── exp # Scripts and instructions to reproduce results in the paper. +├── inc # Midas header files. +├── koord # Midas coordinator kernel module. +├── lib # Compiled Midas library. +├── LICENSE # Midas is open-sourced under the Apache-2.0 License. +├── Makefile # For building the Midas library and unit tests. +├── scripts # Scripts to faciliate developing and using Midas. +├── src # Midas source code. +└── test # Unit tests for individual Midas components. +``` + +## Contact +Please contact [Yifan Qiao](mailto:yifanqiao@g.ucla.edu) for any questions. diff --git a/apps/FeatureExtraction/.gitignore b/apps/FeatureExtraction/.gitignore new file mode 100644 index 0000000..3f8cdc3 --- /dev/null +++ b/apps/FeatureExtraction/.gitignore @@ -0,0 +1,9 @@ +img2vec +gen_feats + +build/* +data/* + +md5.txt +val_img_names.txt +*.data \ No newline at end of file diff --git a/apps/FeatureExtraction/Makefile b/apps/FeatureExtraction/Makefile new file mode 100644 index 0000000..70a58dc --- /dev/null +++ b/apps/FeatureExtraction/Makefile @@ -0,0 +1,49 @@ +CXX = g++ +LDXX = g++ +CXXFLAGS += -std=c++1z -O2 +CXXFLAGS += -march=native # required for avx-enhanced rmemcpy + +BUILD_DIR = build + +INC += -Iinc -I../../inc +LIBS += -lredis++ -lhiredis -lcrypto -pthread + +# libmidas_lib = $(wildcard ../../src/*.o) +# libmidas_lib := $(filter-out $(wildcard ../../src/daemon_main.o),$(libmidas_lib)) +libmidas_lib := ../../lib/libmidas++.a -lrt -ldl +LIBS += $(libmidas_lib) + +APP_BIN = img2vec +APP_SRC = $(wildcard src/*.cpp) +APP_OBJ = $(APP_SRC:%.cpp=$(BUILD_DIR)/%.o) +APP_DEP = $(APP_OBJ:.o=.d) + +TOOL_BIN = gen_feats +TOOL_SRC = $(wildcard tools/*.cpp) +TOOL_OBJ = $(TOOL_SRC:%.cpp=$(BUILD_DIR)/%.o) +TOOL_DEP = $(TOOL_OBJ:.o=.d) + +all: $(APP_BIN) $(TOOL_BIN) + +$(APP_BIN) : $(BUILD_DIR)/$(APP_BIN) +$(TOOL_BIN) : $(BUILD_DIR)/$(TOOL_BIN) + +$(BUILD_DIR)/$(APP_BIN) : $(APP_OBJ) + -mkdir -p $(@D) + $(CXX) $(CXXFLAGS) $^ -o $@ $(LIBS) + +$(BUILD_DIR)/$(TOOL_BIN) : $(TOOL_OBJ) + -mkdir -p $(@D) + $(CXX) $(CXXFLAGS) $^ -o $@ $(LIBS) + +$(BUILD_DIR)/%.o : %.cpp Makefile + -mkdir -p $(@D) + $(CXX) $(CXXFLAGS) $(INC) -MMD -MP -c $< -o $@ + +ifneq ($(MAKECMDGOALS),clean) +-include $(dep) +endif + +.PHONY : clean +clean : + -rm -rf $(BUILD_DIR)/$(BIN) $(OBJ) $(DEP) \ No newline at end of file diff --git a/apps/FeatureExtraction/inc/constants.hpp b/apps/FeatureExtraction/inc/constants.hpp new file mode 100644 index 0000000..a2aacab --- /dev/null +++ b/apps/FeatureExtraction/inc/constants.hpp @@ -0,0 +1,21 @@ +#pragma once + +#include + +namespace FeatExt { +constexpr static int kMD5Len = 32; +constexpr static int kFeatDim = 2048; + +constexpr static int kMissPenalty = 12; // ms +constexpr static int kNrThd = 24; +constexpr static int KPerThdLoad = 10000; + +constexpr static bool kSkewedDist = true; // false for uniform distribution +constexpr static double kSkewness = 0.9; // zipf + +const static std::string data_dir = + "/mnt/ssd/yifan/code/cachebank/apps/FeatureExtraction/"; + +constexpr bool kSimulate = true; +constexpr bool kUseRedis = false; +} // namespace FeatExt diff --git a/apps/FeatureExtraction/inc/fake_backend.hpp b/apps/FeatureExtraction/inc/fake_backend.hpp new file mode 100644 index 0000000..a5092de --- /dev/null +++ b/apps/FeatureExtraction/inc/fake_backend.hpp @@ -0,0 +1,75 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace FeatExt { +class FakeBackend { +public: + FakeBackend() : _arrival_req_id(-1), _processed_req_id(-1), _alive(true) { + int kProcessors = 2; + for (int i = 0; i < kProcessors; i++) { + processor_thds.push_back(std::thread([&]() { processor(); })); + } + } + ~FakeBackend() { + _alive = false; + { + std::unique_lock lk(_p_mtx); + _p_cv.notify_all(); + } + for (auto &thd : processor_thds) + thd.join(); + } + void serve_req() { + int req_id = 0; + { + std::unique_lock plk(_p_mtx); + req_id = _arrival_req_id.fetch_add(1) + 1; + } + _p_cv.notify_all(); + while (_processed_req_id.load() < req_id) + std::this_thread::sleep_for(std::chrono::microseconds(100)); + } + +private: + int processor() { + while (_alive) { + std::unique_lock plk(_p_mtx); + _p_cv.wait(plk, [&] { + return !_alive || _arrival_req_id.load() > _processed_req_id.load(); + }); + + while (_arrival_req_id.load() > _processed_req_id.load()) { + std::this_thread::sleep_for(std::chrono::milliseconds(kMissPenalty)); + _processed_req_id.fetch_add(1); + } + } + return 0; + } + std::mutex _p_mtx; + std::condition_variable _p_cv; + + std::atomic _arrival_req_id; + std::atomic _processed_req_id; + + bool _alive; + std::vector processor_thds; +}; + +static inline FakeBackend *global_fake_backend() { + static std::mutex mtx_; + static std::unique_ptr fake_be; + if (fake_be) + return fake_be.get(); + std::unique_lock ul(mtx_); + if (fake_be) + return fake_be.get(); + fake_be = std::make_unique(); + return fake_be.get(); +} +} // namespace FeatExt \ No newline at end of file diff --git a/apps/FeatureExtraction/inc/feat_extractor.hpp b/apps/FeatureExtraction/inc/feat_extractor.hpp new file mode 100644 index 0000000..f70d834 --- /dev/null +++ b/apps/FeatureExtraction/inc/feat_extractor.hpp @@ -0,0 +1,36 @@ +#pragma once + +#include +#include + +#include "constants.hpp" +#include "redis_utils.hpp" +#include "utils.hpp" + +namespace FeatExt { +struct FeatReq { + int tid; + std::string filename; + Feature *feat; +}; + +class FeatExtractor { +public: + FeatExtractor(); + ~FeatExtractor(); + int warmup_cache(float cache_ratio = 1.0); + void perf(); + +private: + int load_imgs(const std::string &img_file_name); + int load_feats(const std::string &feat_file_name); + + void gen_load(); + bool serve_req(FeatReq img_req); + + std::vector reqs[kNrThd]; + std::vector imgs; + char *raw_feats; + std::vector feats; +}; +} // namespace FeatExt \ No newline at end of file diff --git a/apps/FeatureExtraction/inc/midas_utils.hpp b/apps/FeatureExtraction/inc/midas_utils.hpp new file mode 100644 index 0000000..573b33e --- /dev/null +++ b/apps/FeatureExtraction/inc/midas_utils.hpp @@ -0,0 +1,44 @@ +#pragma once + +#include + +#include "constants.hpp" +#include "utils.hpp" + +#include +#include +#include + +namespace std { +template <> struct hash { + size_t operator()(const FeatExt::MD5Key &k) const { + return std::hash()( + std::string_view(k.data, FeatExt::kMD5Len)); + } +}; + +template <> struct equal_to { + size_t operator()(const FeatExt::MD5Key &k1, + const FeatExt::MD5Key &k2) const { + return std::memcmp(k1.data, k2.data, FeatExt::kMD5Len) == 0; + } +}; +} // namespace std + +namespace FeatExt { +constexpr static int kNumBuckets = 1 << 20; +using SyncHashMap = midas::SyncHashMap; + +SyncHashMap *global_hashmap() { + static std::mutex mtx_; + static std::unique_ptr hash_map_; + + if (hash_map_) + return hash_map_.get(); + std::unique_lock ul(mtx_); + if (hash_map_) + return hash_map_.get(); + hash_map_ = std::make_unique(); + return hash_map_.get(); +} +} // namespace FeatExt \ No newline at end of file diff --git a/apps/FeatureExtraction/inc/redis_utils.hpp b/apps/FeatureExtraction/inc/redis_utils.hpp new file mode 100644 index 0000000..ffd215b --- /dev/null +++ b/apps/FeatureExtraction/inc/redis_utils.hpp @@ -0,0 +1,36 @@ +#pragma once + +#include +#include +#include +#include +#include + +using namespace sw::redis; + +static inline Redis *global_redis() { + static std::mutex mtx_; + static std::unique_ptr redis_; + + if (redis_) + return redis_.get(); + std::unique_lock ul(mtx_); + if (!redis_) { + std::string host = "localhost"; + int port = 6379; + int connections = 512; + int timeout_ms = 10000; + int keepalive_ms = 10000; + + ConnectionOptions connection_options; + connection_options.host = host; + connection_options.port = port; + + ConnectionPoolOptions pool_options; + pool_options.size = connections; + pool_options.wait_timeout = std::chrono::milliseconds(timeout_ms); + pool_options.connection_lifetime = std::chrono::milliseconds(keepalive_ms); + redis_ = std::make_unique(connection_options, pool_options); + } + return redis_.get(); +} diff --git a/apps/FeatureExtraction/inc/socket.hpp b/apps/FeatureExtraction/inc/socket.hpp new file mode 100644 index 0000000..53bb64b --- /dev/null +++ b/apps/FeatureExtraction/inc/socket.hpp @@ -0,0 +1,182 @@ +#pragma once + +#include +#include +#include + +#include "constants.hpp" + +extern "C" { +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#define MAXDATASIZE 1024 // max number of bytes we can get at once + +// get sockaddr, IPv4 or IPv6: +inline void *get_in_addr(struct sockaddr *sa) { + if (sa->sa_family == AF_INET) { + return &(((struct sockaddr_in *)sa)->sin_addr); + } + + return &(((struct sockaddr_in6 *)sa)->sin6_addr); +} + +inline int connect_socket(const char *host, const char *port) { + int sockfd = -1; + int numbytes; + struct addrinfo hints, *servinfo, *p; + int rv; + char s[INET6_ADDRSTRLEN]; + + memset(&hints, 0, sizeof hints); + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + + if ((rv = getaddrinfo(host, port, &hints, &servinfo)) != 0) { + fprintf(stderr, "getaddrinfo: %s\n", gai_strerror(rv)); + return -1; + } + + // loop through all the results and connect to the first we can + for (p = servinfo; p != NULL; p = p->ai_next) { + if ((sockfd = socket(p->ai_family, p->ai_socktype, p->ai_protocol)) == -1) { + perror("client: socket"); + continue; + } + + if (connect(sockfd, p->ai_addr, p->ai_addrlen) == -1) { + close(sockfd); + perror("client: connect"); + continue; + } + + break; + } + + if (p == NULL) { + fprintf(stderr, "client: failed to connect\n"); + return -2; + } + + inet_ntop(p->ai_family, get_in_addr((struct sockaddr *)p->ai_addr), s, + sizeof s); + // printf("client: connecting to %s\n", s); + + freeaddrinfo(servinfo); // all done with this structure + return sockfd; +} + +inline int close_socket(int sockfd) { + close(sockfd); + return 0; +} + +inline int send_socket(int sockfd, const char *data, size_t len) { + int numbytes; + char buf[MAXDATASIZE]; + if ((numbytes = send(sockfd, data, len, 0)) == -1) { + perror("send"); + exit(-1); + } + return numbytes; +} + +inline int recv_socket(int sockfd) { + int numbytes; + char buf[MAXDATASIZE]; + if ((numbytes = recv(sockfd, buf, MAXDATASIZE - 1, 0)) == -1) { + perror("recv"); + exit(1); + } + + buf[numbytes] = '\0'; + + return 0; +} + +static int socket_fd; +inline int initInfClient(const char *host, const char *port) { + socket_fd = connect_socket(host, port); + return socket_fd; +} + +inline int send_recv_socket(const char *data, size_t len) { + int numbytes; + char buf[MAXDATASIZE]; + if ((numbytes = send(socket_fd, data, len, 0)) == -1) { + perror("send"); + exit(-1); + } + + if ((numbytes = recv(socket_fd, buf, MAXDATASIZE - 1, 0)) == -1) { + perror("recv"); + exit(1); + } + + buf[numbytes] = '\0'; + return numbytes; +} +} + +namespace FeatExt { +class Socket { +public: + Socket(const std::string &host = "localhost", + const std::string &port = "10080") + : socketfd(0), _host(host), _port(port) { + init(); + } + ~Socket() { close(); } + + void send(const std::string &msg) { + std::unique_lock lk(mtx); + send_socket(socketfd, msg.c_str(), msg.length()); + } + const std::string recv() { + std::unique_lock lk(mtx); + recv_socket(socketfd); + return ""; + } + + const std::string send_recv(const std::string &msg) { + std::unique_lock lk(mtx); + send_socket(socketfd, msg.c_str(), msg.length()); + recv_socket(socketfd); + return ""; + } + +private: + void init() { + std::unique_lock lk(mtx); + if (kSimulate) + socketfd = 1; + if (socketfd) + return; + socketfd = connect_socket(_host.c_str(), _port.c_str()); + std::cout << "socketfd: " << socketfd << std::endl; + if (socketfd <= 0) { + std::cerr << "Init inference client failed! " << socketfd << std::endl; + exit(-1); + } + } + + void close() { + std::unique_lock lk(mtx); + close_socket(socketfd); + } + + std::mutex mtx; + int socketfd; + const std::string _host; + const std::string _port; +}; +} // namespace FeatExt \ No newline at end of file diff --git a/apps/FeatureExtraction/inc/utils.hpp b/apps/FeatureExtraction/inc/utils.hpp new file mode 100644 index 0000000..81b8dd0 --- /dev/null +++ b/apps/FeatureExtraction/inc/utils.hpp @@ -0,0 +1,76 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "constants.hpp" + +namespace FeatExt { +inline std::string ExecuteShellCommand(const std::string cmd) { + const int BUF_SIZE = 1024; + char buffer[BUF_SIZE]; + std::string result = ""; + std::shared_ptr pipe(popen(cmd.c_str(), "r"), pclose); + if (!pipe) + throw std::runtime_error("popen() failed!"); + while (!feof(pipe.get())) { + if (fgets(buffer, BUF_SIZE, pipe.get()) != NULL) + result += buffer; + } + return result; +} + +inline const std::string md5_from_file(const std::string &filename) { + // std::string fullname = std::string("../validation/") + filename; + std::string fullname = std::string("") + filename; + std::ifstream file(fullname, std::ifstream::binary); + MD5_CTX md5Context; + MD5_Init(&md5Context); + char buf[1024 * 16]; + while (file.good()) { + file.read(buf, sizeof(buf)); + MD5_Update(&md5Context, buf, file.gcount()); + } + unsigned char result[MD5_DIGEST_LENGTH]; + MD5_Final(result, &md5Context); + + std::stringstream md5str_stream; + md5str_stream << std::hex << std::uppercase << std::setfill('0'); + for (const auto &byte : result) + md5str_stream << std::setw(2) << (int)byte; + return md5str_stream.str(); +} + +struct MD5Key { + char data[kMD5Len]; + + void from_string(const std::string &str) { + std::memcpy(data, str.c_str(), kMD5Len); + } + const std::string to_string() { + char str[kMD5Len + 1]; + std::memcpy(str, data, kMD5Len); + str[kMD5Len] = '\0'; + return str; + } +}; +static_assert(sizeof(MD5Key) == kMD5Len, "MD5Key size incorrect"); + +struct Feature { + float data[kFeatDim]; + + sw::redis::StringView to_string_view() { + return sw::redis::StringView(reinterpret_cast(data), + sizeof(Feature)); + } +}; +static_assert(sizeof(Feature) == sizeof(float) * kFeatDim, + "Feature struct size incorrect"); +} // namespace FeatExt \ No newline at end of file diff --git a/apps/FeatureExtraction/inc/zipf.hpp b/apps/FeatureExtraction/inc/zipf.hpp new file mode 100644 index 0000000..6777361 --- /dev/null +++ b/apps/FeatureExtraction/inc/zipf.hpp @@ -0,0 +1,97 @@ +/** Borrowed from Nu. */ + +#pragma once + +#include +#include +#include + +namespace FeatExt { +/** + * Example usage: + * + * std::random_device rd; + * std::mt19937 gen(rd()); + * zipf_table_distribution<> zipf(300); + * + * for (int i = 0; i < 100; i++) + * printf("draw %d %d\n", i, zipf(gen)); + */ +template +class zipf_table_distribution { +public: + typedef IntType result_type; + + static_assert(std::numeric_limits::is_integer, ""); + static_assert(!std::numeric_limits::is_integer, ""); + + /// zipf_table_distribution(N, s) + /// Zipf distribution for `N` items, in the range `[1,N]` inclusive. + /// The distribution follows the power-law 1/n^s with exponent `s`. + /// This uses a table-lookup, and thus provides values more + /// quickly than zipf_distribution. However, the table can take + /// up a considerable amount of RAM, and initializing this table + /// can consume significant time. + zipf_table_distribution(const IntType n, const RealType q = 1.0); + void reset(); + IntType operator()(std::mt19937 &rng); + /// Returns the parameter the distribution was constructed with. + RealType s() const; + /// Returns the minimum value potentially generated by the distribution. + result_type min() const; + /// Returns the maximum value potentially generated by the distribution. + result_type max() const; + +private: + std::vector pdf_; ///< Prob. distribution + IntType n_; ///< Number of elements + RealType q_; ///< Exponent + std::discrete_distribution dist_; ///< Draw generator + + /** Initialize the probability mass function */ + IntType init(const IntType n, const RealType q); +}; + +template +inline zipf_table_distribution::zipf_table_distribution( + const IntType n, const RealType q) + : n_(init(n, q)), q_(q), dist_(pdf_.begin(), pdf_.end()) {} + +template +inline void zipf_table_distribution::reset() {} + +template +inline IntType +zipf_table_distribution::operator()(std::mt19937 &rng) { + return dist_(rng) - 1; +} + +template +inline RealType zipf_table_distribution::s() const { + return q_; +} + +template +inline IntType +zipf_table_distribution::min() const { + return 0; +} + +template +inline IntType +zipf_table_distribution::max() const { + return n_ - 1; +} + +template +inline IntType +zipf_table_distribution::init(const IntType n, + const RealType q) { + pdf_.reserve(n + 1); + pdf_.emplace_back(0.0); + for (IntType i = 1; i <= n; i++) { + pdf_.emplace_back(std::pow((double)i, -q)); + } + return n; +} +} // namespace FeatExt \ No newline at end of file diff --git a/apps/FeatureExtraction/run.sh b/apps/FeatureExtraction/run.sh new file mode 100755 index 0000000..4989b32 --- /dev/null +++ b/apps/FeatureExtraction/run.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +pushd server +python3 server.py & +popd + +redis-server --save "" --appendonly no --maxmemory-policy allkeys-lru & + +sleep 5 + +./build/img2vec 1 + +pkill -9 redis-server +pkill -9 python3 \ No newline at end of file diff --git a/apps/FeatureExtraction/server/server.py b/apps/FeatureExtraction/server/server.py new file mode 100644 index 0000000..05da8b8 --- /dev/null +++ b/apps/FeatureExtraction/server/server.py @@ -0,0 +1,82 @@ +from PIL import Image +import socketserver +import threading + +from vision_model_loader import Img2Vec + +# model = Img2Vec(model='efficientnet_b5', cuda=True) +model = Img2Vec(model='efficientnet_b0', cuda=True) + + +def extractFeat(img_name): + img = Image.open(img_name).convert('RGB') + feat = model.get_vec(img) + + return feat + + +class TCPSocketHandler(socketserver.BaseRequestHandler): + """ + The RequestHandler class for our server. + + It is instantiated once per connection to the server, and must + override the handle() method to implement communication to the + client. + """ + + def handle(self): + # self.request is the TCP socket connected to the client + self.data = "" + remainder = "" + while str(self.data) != "end": + self.data = self.request.recv(1024).strip() + # print("{} wrote:".format(self.client_address[0])) + # print(self.data) + + raw_data = remainder + self.data.decode('ascii') + pos = raw_data.find('.jpg') + while pos != -1: + filename = raw_data[:pos+len('.jpg')] + # print(filename) + extractFeat(filename) + # just send back the same data, but upper-cased + self.request.send(b'feature_vec') + raw_data = raw_data[pos+len('.jpg'):] + pos = raw_data.find('.jpg') + remainder = raw_data + # def handle(self): + # # self.request is the TCP socket connected to the client + # self.data = self.request.recv(1024).strip() + # print("{} wrote:".format(self.client_address[0])) + # print(self.data) + + # filename = self.data + # print(filename) + # extractFeat(filename) + # # just send back the same data, but upper-cased + # self.request.sendall(self.data[:1]) + + +def SimpleServer(host="localhost", port=10080): + server = socketserver.TCPServer((host, port), TCPSocketHandler) + print("Server is listening at {}:{}".format(host, port)) + server.serve_forever() + + +def ThreadedServer(host="localhost", port=10080): + server = socketserver.ThreadingTCPServer((host, port), TCPSocketHandler) + print("Server is listening at {}:{}".format(host, port)) + with server: + server.serve_forever() + # server_thread = threading.Thread(target=server.serve_forever) + # # Exit the server thread when the main thread terminates + # # server_thread.daemon = True + # server_thread.start() + # print("Server loop running in thread:", server_thread.name) + + # server_thread.join() + # server.shutdown() + + +if __name__ == '__main__': + ThreadedServer() diff --git a/apps/FeatureExtraction/server/vision_model_loader.py b/apps/FeatureExtraction/server/vision_model_loader.py new file mode 100644 index 0000000..7e8c1cc --- /dev/null +++ b/apps/FeatureExtraction/server/vision_model_loader.py @@ -0,0 +1,195 @@ +import torch +import torchvision.models as models +import torchvision.transforms as transforms + +class Img2Vec(): + RESNET_OUTPUT_SIZES = { + 'resnet18': 512, + 'resnet34': 512, + 'resnet50': 2048, + 'resnet101': 2048, + 'resnet152': 2048 + } + + EFFICIENTNET_OUTPUT_SIZES = { + 'efficientnet_b0': 1280, + 'efficientnet_b1': 1280, + 'efficientnet_b2': 1408, + 'efficientnet_b3': 1536, + 'efficientnet_b4': 1792, + 'efficientnet_b5': 2048, + 'efficientnet_b6': 2304, + 'efficientnet_b7': 2560 + } + + def __init__(self, cuda=False, model='efficientnet_b5', layer='default', layer_output_size=512, gpu=0): + """ Img2Vec + :param cuda: If set to True, will run forward pass on GPU + :param model: String name of requested model + :param layer: String or Int depending on model. See more docs: https://github.com/christiansafka/img2vec.git + :param layer_output_size: Int depicting the output size of the requested layer + """ + self.device = torch.device(f"cuda:{gpu}" if cuda else "cpu") + self.layer_output_size = layer_output_size + self.model_name = model + + self.model, self.extraction_layer = self._get_model_and_layer(model, layer) + + self.model = self.model.to(self.device) + + self.model.eval() + + self.scaler = transforms.Resize((224, 224)) + self.normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], + std=[0.229, 0.224, 0.225]) + self.to_tensor = transforms.ToTensor() + + def get_vec(self, img, tensor=False): + """ Get vector embedding from PIL image + :param img: PIL Image or list of PIL Images + :param tensor: If True, get_vec will return a FloatTensor instead of Numpy array + :returns: Numpy ndarray + """ + if type(img) == list: + a = [self.normalize(self.to_tensor(self.scaler(im))) for im in img] + images = torch.stack(a).to(self.device) + if self.model_name in ['alexnet', 'vgg']: + my_embedding = torch.zeros(len(img), self.layer_output_size) + elif self.model_name == 'densenet' or 'efficientnet' in self.model_name: + my_embedding = torch.zeros(len(img), self.layer_output_size, 7, 7) + else: + my_embedding = torch.zeros(len(img), self.layer_output_size, 1, 1) + + def copy_data(m, i, o): + my_embedding.copy_(o.data) + + h = self.extraction_layer.register_forward_hook(copy_data) + with torch.no_grad(): + h_x = self.model(images) + h.remove() + + if tensor: + return my_embedding + else: + if self.model_name in ['alexnet', 'vgg']: + return my_embedding.numpy()[:, :] + elif self.model_name == 'densenet' or 'efficientnet' in self.model_name: + return torch.mean(my_embedding, (2, 3), True).numpy()[:, :, 0, 0] + else: + return my_embedding.numpy()[:, :, 0, 0] + else: + image = self.normalize(self.to_tensor(self.scaler(img))).unsqueeze(0).to(self.device) + + if self.model_name in ['alexnet', 'vgg']: + my_embedding = torch.zeros(1, self.layer_output_size) + elif self.model_name == 'densenet' or 'efficientnet' in self.model_name: + my_embedding = torch.zeros(1, self.layer_output_size, 7, 7) + else: + my_embedding = torch.zeros(1, self.layer_output_size, 1, 1) + + def copy_data(m, i, o): + my_embedding.copy_(o.data) + + h = self.extraction_layer.register_forward_hook(copy_data) + with torch.no_grad(): + h_x = self.model(image) + h.remove() + + if tensor: + return my_embedding + else: + if self.model_name in ['alexnet', 'vgg']: + return my_embedding.numpy()[0, :] + elif self.model_name == 'densenet': + return torch.mean(my_embedding, (2, 3), True).numpy()[0, :, 0, 0] + else: + return my_embedding.numpy()[0, :, 0, 0] + + def _get_model_and_layer(self, model_name, layer): + """ Internal method for getting layer from model + :param model_name: model name such as 'resnet-18' + :param layer: layer as a string for resnet-18 or int for alexnet + :returns: pytorch model, selected layer + """ + + if model_name.startswith('resnet') and not model_name.startswith('resnet-'): + model = getattr(models, model_name)(pretrained=True) + if layer == 'default': + layer = model._modules.get('avgpool') + self.layer_output_size = self.RESNET_OUTPUT_SIZES[model_name] + else: + layer = model._modules.get(layer) + return model, layer + elif model_name == 'resnet-18': + model = models.resnet18(pretrained=True) + if layer == 'default': + layer = model._modules.get('avgpool') + self.layer_output_size = 512 + else: + layer = model._modules.get(layer) + + return model, layer + + elif model_name == 'alexnet': + model = models.alexnet(pretrained=True) + if layer == 'default': + layer = model.classifier[-2] + self.layer_output_size = 4096 + else: + layer = model.classifier[-layer] + + return model, layer + + elif model_name == 'vgg': + # VGG-11 + model = models.vgg11_bn(pretrained=True) + if layer == 'default': + layer = model.classifier[-2] + self.layer_output_size = model.classifier[-1].in_features # should be 4096 + else: + layer = model.classifier[-layer] + + return model, layer + + elif model_name == 'densenet': + # Densenet-121 + model = models.densenet121(pretrained=True) + if layer == 'default': + layer = model.features[-1] + self.layer_output_size = model.classifier.in_features # should be 1024 + else: + raise KeyError('Un support %s for layer parameters' % model_name) + + return model, layer + + elif "efficientnet" in model_name: + # efficientnet-b0 ~ efficientnet-b7 + if model_name == "efficientnet_b0": + model = models.efficientnet_b0(pretrained=True) + elif model_name == "efficientnet_b1": + model = models.efficientnet_b1(pretrained=True) + elif model_name == "efficientnet_b2": + model = models.efficientnet_b2(pretrained=True) + elif model_name == "efficientnet_b3": + model = models.efficientnet_b3(pretrained=True) + elif model_name == "efficientnet_b4": + model = models.efficientnet_b4(pretrained=True) + elif model_name == "efficientnet_b5": + model = models.efficientnet_b5(pretrained=True) + elif model_name == "efficientnet_b6": + model = models.efficientnet_b6(pretrained=True) + elif model_name == "efficientnet_b7": + model = models.efficientnet_b7(pretrained=True) + else: + raise KeyError('Un support %s.' % model_name) + + if layer == 'default': + layer = model.features + self.layer_output_size = self.EFFICIENTNET_OUTPUT_SIZES[model_name] + else: + raise KeyError('Un support %s for layer parameters' % model_name) + + return model, layer + + else: + raise KeyError('Model %s was not found' % model_name) diff --git a/apps/FeatureExtraction/src/client.cpp b/apps/FeatureExtraction/src/client.cpp new file mode 100644 index 0000000..a2acf9d --- /dev/null +++ b/apps/FeatureExtraction/src/client.cpp @@ -0,0 +1,29 @@ +#include + +#include "constants.hpp" +#include "feat_extractor.hpp" +#include "redis_utils.hpp" + +float cache_ratio = 1.0; +constexpr size_t cache_size = 420 * 1024 * 1024; + +int main(int argc, char *argv[]) { + if (argc <= 1) { + std::cerr << "Usage: ./" << argv[0] << " " << std::endl; + exit(-1); + } + cache_ratio = std::atof(argv[1]); + + if (FeatExt::kUseRedis) { + auto redis = global_redis(); + // std::cout << redis.ping() << std::endl; + redis->command("config", "set", "maxmemory", + static_cast(cache_size * cache_ratio)); + } + + FeatExt::FeatExtractor client; + client.warmup_cache(cache_ratio); + client.perf(); + + return 0; +} \ No newline at end of file diff --git a/apps/FeatureExtraction/src/feat_extractor.cpp b/apps/FeatureExtraction/src/feat_extractor.cpp new file mode 100644 index 0000000..f068934 --- /dev/null +++ b/apps/FeatureExtraction/src/feat_extractor.cpp @@ -0,0 +1,193 @@ +#include +#include +#include + +#include "constants.hpp" +#include "fake_backend.hpp" +#include "feat_extractor.hpp" +#include "redis_utils.hpp" +#include "socket.hpp" +#include "utils.hpp" +#include "zipf.hpp" +#include "midas_utils.hpp" + +namespace FeatExt { +Socket sockets[kNrThd]; + +const std::string getFeatVector(struct FeatReq req) { + auto &md5 = md5_from_file(req.filename); + // Cache hit + if (kUseRedis) { + auto redis = global_redis(); + auto feat_opt = redis->get(md5); + if (feat_opt) + return *feat_opt; + } else { + auto hashmap = global_hashmap(); + MD5Key key; + key.from_string(md5); + auto feat_opt = hashmap->get(key); + if (feat_opt) + return ""; + } + + // Cache miss + if (kSimulate) { + global_fake_backend()->serve_req(); + } else { + sockets[req.tid].send_recv(req.filename); + } + // // Update cache + // if (kUseRedis) { + // auto redis = global_redis(); + // redis->set(md5, req.feat->to_string_view()); + // } else { + // auto hashmap = global_hashmap(); + // MD5Key key; + // key.from_string(md5); + // hashmap->set(key, *req.feat); + // } + + return ""; +} + +/** initialization & utils */ +FeatExtractor::FeatExtractor() + : raw_feats(nullptr) { + std::string img_file_name = data_dir + "val_img_names.txt"; + std::string feat_file_name = data_dir + "enb5_feat_vec.data"; + load_imgs(img_file_name); + load_feats(feat_file_name); + gen_load(); +} + +FeatExtractor::~FeatExtractor() { + if (raw_feats) + delete[] raw_feats; +} + +void FeatExtractor::gen_load() { + auto nr_imgs = imgs.size(); + std::random_device rd; + std::mt19937 gen(rd()); + zipf_table_distribution<> zipf_dist(nr_imgs, kSkewness); + std::uniform_int_distribution<> uni_dist(0, nr_imgs - 1); + + for (int tid = 0; tid < kNrThd; tid++) { + const auto nr_imgs = imgs.size(); + reqs[tid].clear(); + for (int o = 0; o < KPerThdLoad; o++) { + auto id = kSkewedDist ? zipf_dist(gen) : uni_dist(gen); + FeatReq req{ + .tid = tid, .filename = imgs.at(id), .feat = feats.at(id)}; + reqs[tid].push_back(req); + } + } + std::cout << "Finish load generation." << std::endl; +} + +bool FeatExtractor::serve_req(FeatReq req) { + auto &feat = getFeatVector(req); + return true; +} + +int FeatExtractor::load_imgs(const std::string &img_file_name) { + std::ifstream img_file(img_file_name, std::ifstream::in); + if (!img_file.good()) { + std::cerr << "cannot open img_file " << img_file_name << std::endl; + return -1; + } + + while (img_file.good()) { + std::string name; + std::getline(img_file, name); + if (!name.empty()) + imgs.push_back(name); + // std::cout << name << " " << imgs.at(imgs.size() - 1) << std::endl; + } + + auto nr_imgs = imgs.size(); + std::cout << nr_imgs << " " << imgs[0] << " " << md5_from_file(imgs[0]) + << std::endl; + return nr_imgs; +} + +int FeatExtractor::load_feats(const std::string &feat_file_name) { + size_t nr_imgs = imgs.size(); + raw_feats = new char[nr_imgs * kFeatDim * sizeof(float)]; + size_t nr_feat_vecs = 0; + + std::ifstream feat_file(feat_file_name, std::ifstream::binary); + if (feat_file.good()) { + feat_file.read(raw_feats, nr_imgs * sizeof(float) * kFeatDim); + } + + char *ptr = raw_feats; + for (int i = 0; i < nr_imgs; i++) { + feats.push_back(reinterpret_cast(ptr)); + ptr += sizeof(float) * kFeatDim; + } + + return feats.size(); +} + +int FeatExtractor::warmup_cache(float cache_ratio) { + const auto filename = data_dir + "md5.txt"; + std::ifstream md5_file(filename); + if (!md5_file.is_open()) { + std::cerr << "cannot open file " << filename << std::endl; + return -1; + } + + if (kUseRedis) { + size_t nr_imgs = imgs.size(); + std::cout << nr_imgs << " " << feats.size() << std::endl; + auto pipe = global_redis()->pipeline(false); + for (int i = 0; i < nr_imgs * cache_ratio; i++) { + std::string md5; + md5_file >> md5; + // std::cout << imgs.at(i) << " " << md5 << std::endl; + pipe.set(md5, feats.at(i)->to_string_view()); + } + pipe.exec(); + std::cout << "Done warm up redis" << std::endl; + } else { // midas + size_t nr_imgs = imgs.size(); + std::cout << nr_imgs << " " << feats.size() << std::endl; + auto hashmap = global_hashmap(); + for (int i = 0; i < nr_imgs * cache_ratio; i++) { + std::string md5; + md5_file >> md5; + MD5Key key; + key.from_string(md5); + hashmap->set(key, *feats.at(i)); + // std::cout << imgs.at(i) << " " << md5 << std::endl; + } + std::cout << "Done warm up midas" << std::endl; + } + return 0; +} + +void FeatExtractor::perf() { + auto stt = std::chrono::high_resolution_clock::now(); + std::vector worker_thds; + for (int tid = 0; tid < kNrThd; tid++) { + worker_thds.push_back(std::thread([&, tid = tid]() { + for (int i = 0; i < KPerThdLoad; i++) { + serve_req(reqs[tid][i]); + } + })); + } + + for (auto &thd : worker_thds) { + thd.join(); + } + + auto end = std::chrono::high_resolution_clock::now(); + auto duration = + std::chrono::duration_cast(end - stt).count(); + auto tput = static_cast(kNrThd * KPerThdLoad) / duration; + std::cout << "Perf done. Duration: " << duration + << " ms, Throughput: " << tput << " Kops" << std::endl; +} +} // namespace FeatExt diff --git a/apps/FeatureExtraction/tools/gen_feats.cpp b/apps/FeatureExtraction/tools/gen_feats.cpp new file mode 100644 index 0000000..ac42968 --- /dev/null +++ b/apps/FeatureExtraction/tools/gen_feats.cpp @@ -0,0 +1,33 @@ +#include +#include +#include + +#include "../inc/constants.hpp" + +namespace FeatExt { +int gen_fake_feats(int nr_imgs) { + std::ofstream feat_file("fake_feat_vec.data", std::ofstream::binary); + float *feats = new float[nr_imgs * kFeatDim]; + + for (int i = 0; i < nr_imgs; i++) + for (int j = 0; j < kFeatDim; j++) + feats[i * kFeatDim + j] = static_cast(j); + + if (feat_file.good()) { + feat_file.write(reinterpret_cast(feats), + sizeof(float) * kFeatDim * nr_imgs); + } + + return 0; +} +} // namespace FeatExt + +int main(int argc, char *argv[]) { + if (argc < 2) { + std::cerr << "Usage: ./gen_feats " << std::endl; + } + + int nr_imgs = std::stoi(argv[1]); + FeatExt::gen_fake_feats(nr_imgs); + return 0; +} \ No newline at end of file diff --git a/apps/HDSearch/.gitignore b/apps/HDSearch/.gitignore new file mode 100644 index 0000000..46dbc92 --- /dev/null +++ b/apps/HDSearch/.gitignore @@ -0,0 +1,4 @@ +bin/ + +logs/ +logs.bak diff --git a/apps/HDSearch/Makefile b/apps/HDSearch/Makefile new file mode 100644 index 0000000..4656ebd --- /dev/null +++ b/apps/HDSearch/Makefile @@ -0,0 +1,47 @@ +CXX = g++ +LDXX = g++ +CXXFLAGS += -std=c++1z -O2 +CXXFLAGS += -march=native # required for avx-enhanced rmemcpy + +BUILD_DIR = bin + +# INC += -Ithird_party/json/build/ +# INC += -I/usr/include/mysql-cppconn-8/ +# LIBS += -lmysqlcppconn8 + +INC += -I../../inc +libmidas_lib := ../../lib/libmidas++.a -lrt -ldl +LIBS += $(libmidas_lib) + +INC += -Iinc +LIBS += -lcrypto -pthread +# For stacktrace logging +LIBS += -rdynamic -ggdb -no-pie -fno-pie +LIBS += -ldl + +SRC = $(wildcard src/*.cpp) + +APP_BIN = hdsearch +APP_SRC = $(SRC) +APP_OBJ = $(APP_SRC:%.cpp=$(BUILD_DIR)/%.o) +APP_DEP = $(APP_OBJ:.o=.d) + +all: $(APP_BIN) + +$(APP_BIN) : $(BUILD_DIR)/$(APP_BIN) + +$(BUILD_DIR)/$(APP_BIN) : $(APP_OBJ) + @mkdir -p $(@D) + $(LDXX) $(CXXFLAGS) $^ -o $@ $(LIBS) + +$(BUILD_DIR)/%.o : %.cpp Makefile + @mkdir -p $(@D) + $(CXX) $(CXXFLAGS) $(INC) -MMD -MP -c $< -o $@ + +ifneq ($(MAKECMDGOALS),clean) +-include $(APP_DEP) +endif + +.PHONY : clean +clean : + -rm -rf $(BUILD_DIR)/$(APP_BIN) $(APP_OBJ) $(APP_DEP) \ No newline at end of file diff --git a/apps/HDSearch/inc/constants.hpp b/apps/HDSearch/inc/constants.hpp new file mode 100644 index 0000000..8027fbc --- /dev/null +++ b/apps/HDSearch/inc/constants.hpp @@ -0,0 +1,28 @@ +#pragma once + +#include + +namespace hdsearch { +constexpr static int kFeatDim = 2048; +constexpr static int kMD5Len = 32; + +constexpr static int kNrThd = 12; +constexpr static int kNumBuckets = 1 << 20; + +constexpr static bool kSkewedDist = true; // false for uniform distribution +constexpr static double kSkewness = 0.9; // zipf + +constexpr static bool kSimulate = true; +constexpr static int kSimuNumImgs = 1900 * 1000; // 1.9M images + +constexpr static float kMissPenalty = 4; // ms +constexpr static int kNrGPUs = 4; + +const static std::string data_dir = + "/mnt/ssd/yifan/code/cachebank/apps/FeatureExtraction/data/"; +const static std::string md5_filename = data_dir + "md5.txt"; +const static std::string img_filename = data_dir + "val_img_names.txt"; +const static std::string feat_filename = data_dir + "enb5_feat_vec.data"; + +const static std::string cachepool_name = "feats"; +} // namespace hdsearch \ No newline at end of file diff --git a/apps/HDSearch/inc/fake_backend.hpp b/apps/HDSearch/inc/fake_backend.hpp new file mode 100644 index 0000000..2fb48b8 --- /dev/null +++ b/apps/HDSearch/inc/fake_backend.hpp @@ -0,0 +1,78 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "constants.hpp" +#include "utils.hpp" + +namespace hdsearch { +class FakeBackend { +public: + FakeBackend(int nr_processors = 2) + : _nr_processors(nr_processors), _arrival_req_id(-1), + _processed_req_id(-1), _alive(true) { + for (int i = 0; i < _nr_processors; i++) { + processor_thds.push_back(std::thread([&]() { processor(); })); + } + } + ~FakeBackend() { + _alive = false; + { + std::unique_lock lk(_p_mtx); + _p_cv.notify_all(); + } + for (auto &thd : processor_thds) + thd.join(); + } + Feature *serve_req() { + int req_id = 0; + { + std::unique_lock plk(_p_mtx); + req_id = _arrival_req_id.fetch_add(1) + 1; + } + _p_cv.notify_all(); + while (_processed_req_id.load() < req_id) + std::this_thread::sleep_for(std::chrono::microseconds(100)); + auto ret = new Feature(); + return ret; + } + +private: + int processor() { + while (_alive) { + { + std::unique_lock plk(_p_mtx); + _p_cv.wait(plk, [&] { + return !_alive || _arrival_req_id.load() > _processed_req_id.load(); + }); + } + + while (_arrival_req_id.load() > _processed_req_id.load()) { + std::this_thread::sleep_for( + std::chrono::microseconds(std::lround(kMissPenalty * 1000))); + _processed_req_id.fetch_add(1); + } + } + return 0; + } + + int _nr_processors; + + std::mutex _p_mtx; + std::condition_variable _p_cv; + // std::mutex _c_mtx; + // std::condition_variable _c_cv; + + std::atomic _arrival_req_id; + std::atomic _processed_req_id; + + bool _alive; + std::vector processor_thds; +}; +} // namespace hdsearch \ No newline at end of file diff --git a/apps/HDSearch/inc/feat_extractor.hpp b/apps/HDSearch/inc/feat_extractor.hpp new file mode 100644 index 0000000..5867922 --- /dev/null +++ b/apps/HDSearch/inc/feat_extractor.hpp @@ -0,0 +1,62 @@ +#pragma once + +#include +#include +#include + +#include "constants.hpp" +#include "fake_backend.hpp" +#include "utils.hpp" + +// [midas] +#include "cache_manager.hpp" +#include "perf.hpp" +#include "sync_hashmap.hpp" +#include "zipf.hpp" + +namespace hdsearch { + +struct FeatReq : midas::PerfRequest { + int tid; + int rid; + std::string filename; + Feature *feat; +}; + +class FeatExtractor : public midas::PerfAdapter { +public: + FeatExtractor(); + ~FeatExtractor(); + std::unique_ptr gen_req(int tid) override; + bool serve_req(int tid, const midas::PerfRequest *img_req) override; + void warmup(); + +private: + size_t load_imgs(); + size_t load_feats(); + + // To re-construct cache-miss objects + int construct_callback(void *arg); + + FakeBackend fakeGPUBackend; + + size_t nr_imgs; + std::vector imgs; + char *raw_feats; + std::vector> feats; + + std::unique_ptr> zipf_dist; + std::unique_ptr> uni_dist; + std::unique_ptr gens[kNrThd]; + + struct { + int nr_hit = 0; + int nr_miss = 0; + } perthd_cnts[kNrThd]; + void report_hit_rate(); + + // midas cache + midas::CachePool *cpool; + std::shared_ptr> feat_map; +}; +} // namespace hdsearch \ No newline at end of file diff --git a/apps/HDSearch/inc/utils.hpp b/apps/HDSearch/inc/utils.hpp new file mode 100644 index 0000000..7ade420 --- /dev/null +++ b/apps/HDSearch/inc/utils.hpp @@ -0,0 +1,75 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "constants.hpp" + +namespace hdsearch { + +struct MD5Key { + char data[kMD5Len]; + + const std::string to_string() { + char str[kMD5Len + 1]; + std::memcpy(str, data, kMD5Len); + str[kMD5Len] = '\0'; + return str; + } +}; + +struct Feature { + float data[kFeatDim]; +}; +static_assert(sizeof(Feature) == sizeof(float) * kFeatDim, + "Feature struct size incorrect"); + +struct Trace { + uint64_t absl_start_us; + uint64_t start_us; + uint64_t duration; +}; + +static inline void md5_from_file(MD5Key &md5_result, + const std::string &filename) { + std::ifstream file(filename, std::ifstream::binary); + MD5_CTX md5Context; + MD5_Init(&md5Context); + char buf[1024 * 16]; + while (file.good()) { + file.read(buf, sizeof(buf)); + MD5_Update(&md5Context, buf, file.gcount()); + } + unsigned char result[MD5_DIGEST_LENGTH]; + MD5_Final(result, &md5Context); + + std::stringstream md5str_stream; + md5str_stream << std::hex << std::uppercase << std::setfill('0'); + for (const auto &byte : result) + md5str_stream << std::setw(2) << (int)byte; + md5str_stream << "\0"; + std::memcpy(md5_result.data, md5str_stream.str().c_str(), kMD5Len); +} + +} // namespace hdsearch + +namespace std { +template <> struct hash { + size_t operator()(const hdsearch::MD5Key &k) const { + return std::hash()( + std::string_view(k.data, hdsearch::kMD5Len)); + } +}; + +template <> struct equal_to { + size_t operator()(const hdsearch::MD5Key &k1, + const hdsearch::MD5Key &k2) const { + return std::memcmp(k1.data, k2.data, hdsearch::kMD5Len) == 0; + } +}; +} // namespace std diff --git a/apps/HDSearch/run.sh b/apps/HDSearch/run.sh new file mode 100755 index 0000000..a112491 --- /dev/null +++ b/apps/HDSearch/run.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +memratio=0.5 + +# numactl --cpunodebind=0 --membind=0 \ +taskset -c 1,3,5,7,9,11,13,15,17,19,21,23 \ + ./bin/hdsearch $memratio 2>&1 | tee midas.log diff --git a/apps/HDSearch/src/feat_extractor.cpp b/apps/HDSearch/src/feat_extractor.cpp new file mode 100644 index 0000000..7ee4acb --- /dev/null +++ b/apps/HDSearch/src/feat_extractor.cpp @@ -0,0 +1,189 @@ +#include +#include + +#include "constants.hpp" +#include "fake_backend.hpp" +#include "feat_extractor.hpp" + +// [midas] +#include "perf.hpp" +#include "utils.hpp" +#include "zipf.hpp" + +namespace hdsearch { +FeatExtractor::FeatExtractor() + : raw_feats(nullptr), nr_imgs(0), fakeGPUBackend(kNrGPUs) { + cpool = midas::CacheManager::global_cache_manager()->get_pool(cachepool_name); + if (!cpool) { + std::cerr << "Failed to get cache pool!" << std::endl; + exit(-1); + } + // set up midas + cpool->set_construct_func( + [&](void *args) { return construct_callback(args); }); + feat_map = + std::make_unique>(cpool); + + // load images + load_imgs(); + load_feats(); + nr_imgs = kSimulate ? kSimuNumImgs : imgs.size(); + + // init random number generator + zipf_dist = + std::make_unique>(nr_imgs, kSkewness); + uni_dist = std::make_unique>(0, nr_imgs - 1); + for (int i = 0; i < kNrThd; i++) { + std::random_device rd; + gens[i] = std::make_unique(rd()); + } +} + +FeatExtractor::~FeatExtractor() { + if (raw_feats) + delete[] raw_feats; +} + +int FeatExtractor::construct_callback(void *arg) { + auto args_ = reinterpret_cast(arg); + auto md5 = reinterpret_cast(args_->key); + auto feat_buf = reinterpret_cast(args_->value); + assert(args_->key_len == sizeof(MD5Key)); + + auto feat = fakeGPUBackend.serve_req(); + if (feat_buf) { + assert(args_->value_len == sizeof(Feature)); + std::memcpy(feat_buf, feat, sizeof(Feature)); + delete feat; + } else { + args_->value = feat; + args_->value_len = sizeof(Feature); + } + + return 0; +} + +std::unique_ptr FeatExtractor::gen_req(int tid) { + int id = kSkewedDist ? (*zipf_dist)(*gens[tid]) : (*uni_dist)(*gens[tid]); + id = nr_imgs - 1 - id; + + auto req = std::make_unique(); + req->tid = tid; + req->rid = id; + req->filename = imgs.at(id % imgs.size()); + req->feat = feats.at(id % feats.size()).get(); + + return req; +} + +bool FeatExtractor::serve_req(int tid, const midas::PerfRequest *req_) { + auto *req = dynamic_cast(req_); + MD5Key md5; + md5_from_file(md5, req->filename); + if (kSimulate) { + std::ostringstream oss; + oss << std::setw(32) << std::setfill('0') << req->rid; + std::string md5_str = oss.str(); + md5_str.copy(md5.data, kMD5Len); + } + auto feat_opt = feat_map->get(md5); + if (feat_opt) { + perthd_cnts[req->tid].nr_hit++; + return true; + } + + // Cache miss + auto stt = midas::Time::get_cycles_stt(); + perthd_cnts[req->tid].nr_miss++; + midas::ConstructArgs args{.key = &md5, + .key_len = sizeof(md5), + .value = req->feat, + .value_len = sizeof(Feature)}; + assert(cpool->construct(&args) == 0); + feat_map->set(md5, req->feat); + auto end = midas::Time::get_cycles_end(); + + cpool->record_miss_penalty(end - stt, sizeof(*req->feat)); + return true; +} + +void FeatExtractor::warmup() { + std::vector thds; + for (int i = 0; i < kNrThd; i++) { + thds.emplace_back([&, tid = i] { + for (int j = tid; j < nr_imgs; j += kNrThd) { + auto value = feats.at(j % feats.size()).get(); + MD5Key md5; + if (kSimulate) { + std::ostringstream oss; + oss << std::setw(32) << std::setfill('0') << j; + std::string md5_str = oss.str(); + md5_str.copy(md5.data, kMD5Len); + } else { + md5_from_file(md5, imgs.at(j)); + } + feat_map->set(md5, *value); + } + }); + } + for (auto &thd : thds) + thd.join(); + std::cout << "Done warm up!" << std::endl; +} + +size_t FeatExtractor::load_imgs() { + std::ifstream img_file(img_filename, std::ifstream::in); + if (!img_file.good()) { + std::cerr << "cannot open img_file " << img_filename << std::endl; + return -1; + } + + while (img_file.good()) { + std::string name; + std::getline(img_file, name); + if (!name.empty()) + imgs.push_back(name); + // std::cout << name << " " << imgs.at(imgs.size() - 1) << std::endl; + } + + size_t nr_imgs = imgs.size(); + MD5Key md5; + md5_from_file(md5, imgs[0]); + std::cout << "Load " << nr_imgs << " images, MD5 of " << imgs[0] << ": " + << md5.to_string() << std::endl; + + return nr_imgs; +} + +size_t FeatExtractor::load_feats() { + size_t nr_imgs = imgs.size(); + raw_feats = new char[nr_imgs * kFeatDim * sizeof(float)]; + size_t nr_feat_vecs = 0; + + std::ifstream feat_file(feat_filename, std::ifstream::binary); + if (feat_file.good()) { + feat_file.read(raw_feats, nr_imgs * sizeof(float) * kFeatDim); + } + + char *ptr = raw_feats; + for (int i = 0; i < nr_imgs; i++) { + auto new_feat = std::make_shared(); + feats.emplace_back(new_feat); + std::memcpy(new_feat->data, ptr, sizeof(float) * kFeatDim); + ptr += sizeof(float) * kFeatDim; + } + + return feats.size(); +} + +void FeatExtractor::report_hit_rate() { + int nr_hit = 0; + int nr_miss = 0; + for (int i = 0; i < kNrThd; i++) { + nr_hit += perthd_cnts[i].nr_hit; + nr_miss += perthd_cnts[i].nr_miss; + } + std::cout << "Cache hit ratio = " << nr_hit << "/" << nr_hit + nr_miss + << " = " << 1.0 * nr_hit / (nr_hit + nr_miss) << std::endl; +} +} // namespace hdsearch \ No newline at end of file diff --git a/apps/HDSearch/src/main.cpp b/apps/HDSearch/src/main.cpp new file mode 100644 index 0000000..ba02b0d --- /dev/null +++ b/apps/HDSearch/src/main.cpp @@ -0,0 +1,138 @@ +#include +#include +#include +#include +#include + +#include "constants.hpp" +#include "feat_extractor.hpp" +#include "utils.hpp" + +// [midas] +#include "perf.hpp" +#include "cache_manager.hpp" + +namespace hdsearch { +float cache_ratio = 1.0; +size_t cache_size = (kSimulate ? kSimuNumImgs : 41620ull) * (80 + 8192); +} // namespace hdsearch + +using namespace hdsearch; + +void get_timeseries_nth_lats(midas::Perf &perf, double nth) { + const auto slice_us = 5 * midas::to_us; // 5s + std::string filename = "p"; + if (nth == 99.9) + filename += "99.9"; + else + filename += std::to_string(int(nth)); + filename += ".txt"; + std::ofstream nth_file(filename); + assert(nth_file.good()); + auto timeseries = perf.get_timeseries_nth_lats(slice_us, nth); + std::cout << "P" << nth << " time series: "; + for (auto &ts : timeseries) { + std::cout << ts.duration_us << " "; + nth_file << ts.duration_us << "\n"; + } + std::cout << std::endl; + nth_file << std::endl; + nth_file.close(); +} + +int const_workload(int argc, char *argv[]) { + FeatExtractor client; + client.warmup(); + + for (int i = 0; i < 10; i++) { + midas::Perf perf(client); + auto target_kops = 20; + auto duration_us = 10000 * midas::to_us; + auto warmup_us = 10 * midas::to_us; + auto miss_ddl = 10 * midas::to_us; + perf.run(kNrThd, target_kops, duration_us, warmup_us, miss_ddl); + auto real_kops = perf.get_real_kops(); + std::cout << real_kops << std::endl; + std::cout << perf.get_nth_lat(50) << " " << perf.get_nth_lat(99) << " " + << perf.get_nth_lat(99.9) << std::endl; + get_timeseries_nth_lats(perf, 99); + get_timeseries_nth_lats(perf, 99.9); + } + + return 0; +} + +int phase_workload(int argc, char *argv[]) { + FeatExtractor client; + client.warmup(); + + midas::Perf perf(client); + std::vector target_kops_vec; + std::vector duration_us_vec; + std::vector transition_us_vec; + for (int i = 0; i < 20; i++) { + target_kops_vec.emplace_back((i + 1)); + duration_us_vec.emplace_back(600 * midas::to_us); + transition_us_vec.emplace_back(100 * midas::to_us); + } + double warmup_kops = 10; + uint64_t warmup_us = 10 * midas::to_us; + perf.run_phased(kNrThd, target_kops_vec, duration_us_vec, transition_us_vec, + warmup_kops, warmup_us); + // perf.run(kNrThd, 1.0, 10 * 1000 * 1000); + auto real_kops = perf.get_real_kops(); + std::cout << real_kops << std::endl; + std::cout << perf.get_nth_lat(50) << " " << perf.get_nth_lat(99) << " " + << perf.get_nth_lat(99.9) << std::endl; + get_timeseries_nth_lats(perf, 99); + get_timeseries_nth_lats(perf, 99.9); + + return 0; +} + +int stepped_workload(int argc, char *argv[]) { + FeatExtractor client; + client.warmup(); + + midas::Perf perf(client); + // warm up + auto warmup_kops = 100; + auto warmup_us = 5 * midas::to_us; + perf.run(kNrThd, warmup_kops, 0, warmup_us); + perf.reset(); + + // perf + std::vector target_kops_vec{10, 11, 12, 14, 16, 18, 20}; + for (auto target_kops : target_kops_vec) { + auto duration_us = 20 * midas::to_us; + perf.reset(); + perf.run(kNrThd, target_kops, duration_us); + + auto real_kops = perf.get_real_kops(); + std::cout << "Target Kops: " << target_kops << ", Real Kops:" << real_kops << std::endl; + std::cout << "Latency (us): " << perf.get_nth_lat(50) << " " + << perf.get_nth_lat(99) << " " << perf.get_nth_lat(99.9) + << std::endl; + get_timeseries_nth_lats(perf, 99); + get_timeseries_nth_lats(perf, 99.9); + } + + return 0; +} + +int main(int argc, char *argv[]) { + if (argc <= 1) { + std::cout << "Usage: ./" << argv[0] << " " << std::endl; + exit(-1); + } + cache_ratio = std::stof(argv[1]); + midas::CacheManager::global_cache_manager()->create_pool(cachepool_name); + auto pool = + midas::CacheManager::global_cache_manager()->get_pool(cachepool_name); + // pool->update_limit(cache_size * cache_ratio); + pool->update_limit(5ll * 1024 * 1024 * 1024); // GB + + return const_workload(argc, argv); + // return phase_workload(argc, argv); + // return stepped_workload(argc, argv); +} \ No newline at end of file diff --git a/apps/OneRF/.gitignore b/apps/OneRF/.gitignore new file mode 100644 index 0000000..63f9c2e --- /dev/null +++ b/apps/OneRF/.gitignore @@ -0,0 +1,2 @@ +build/ +tracegen/ \ No newline at end of file diff --git a/apps/OneRF/Makefile b/apps/OneRF/Makefile new file mode 100644 index 0000000..5c2c85d --- /dev/null +++ b/apps/OneRF/Makefile @@ -0,0 +1,44 @@ +CXX = g++ +LDXX = g++ +CXXFLAGS += -std=c++1z -O2 +CXXFLAGS += -march=native # required for avx-enhanced rmemcpy + +BUILD_DIR = build + +INC += -Ithird_party/json/build/ +# INC += -I/usr/include/mysql-cppconn-8/ +# LIBS += -lmysqlcppconn8 + +INC += -I../../inc +libmidas_lib := ../../lib/libmidas++.a -lrt -ldl +LIBS += $(libmidas_lib) + +INC += -Iinc +LIBS += -pthread + +SRC = $(wildcard src/*.cpp) + +RH_APP_BIN = rh_app +RH_APP_SRC = $(SRC) $(wildcard rh_server/*.cpp) +RH_APP_OBJ = $(RH_APP_SRC:%.cpp=$(BUILD_DIR)/%.o) +RH_APP_DEP = $(RH_APP_OBJ:.o=.d) + +all: $(RH_APP_BIN) + +$(RH_APP_BIN) : $(BUILD_DIR)/$(RH_APP_BIN) + +$(BUILD_DIR)/$(RH_APP_BIN) : $(RH_APP_OBJ) + @mkdir -p $(@D) + $(CXX) $(CXXFLAGS) $^ -o $@ $(LIBS) + +$(BUILD_DIR)/%.o : %.cpp Makefile + @mkdir -p $(@D) + $(CXX) $(CXXFLAGS) $(INC) -MMD -MP -c $< -o $@ + +ifneq ($(MAKECMDGOALS),clean) +-include $(RH_APP_DEP) +endif + +.PHONY : clean +clean : + -rm -rf $(BUILD_DIR)/$(RH_APP_BIN) $(RH_APP_OBJ) $(RH_APP_DEP) \ No newline at end of file diff --git a/apps/OneRF/inc/backend.hpp b/apps/OneRF/inc/backend.hpp new file mode 100644 index 0000000..778fdd7 --- /dev/null +++ b/apps/OneRF/inc/backend.hpp @@ -0,0 +1,17 @@ +#pragma once + +#include +#include +#include + +namespace onerf { +using Item = std::basic_string; + +class Backend { +public: + virtual ~Backend() = default; + + virtual bool Request(const std::vector &ids, + std::map &res_map) = 0; +}; +} // namespace onerf \ No newline at end of file diff --git a/apps/OneRF/inc/fback.hpp b/apps/OneRF/inc/fback.hpp new file mode 100644 index 0000000..2adba05 --- /dev/null +++ b/apps/OneRF/inc/fback.hpp @@ -0,0 +1,59 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "backend.hpp" +#include "semaphore.hpp" + +namespace onerf { +class FBack : public Backend { +public: + struct Param { + int32_t Hardness{0}; + int32_t IdCount{0}; + int32_t SizeLower{0}; + int32_t SizeUpper{0}; + }; + struct Matrix { + float **data; + int rows; + int cols; + + Matrix(); + ~Matrix(); + void init(int rows_, int cols_); + void free(); + }; + + FBack(const struct Param ¶m, int max_conns); + ~FBack() override; + bool Request(const std::vector &ids, + std::map &res_map) override; + bool GetIDs(const Param ¶m, std::vector &data); + void Report(); + +private: + void calculate(int hardness); + bool dot(const Matrix &x, const Matrix &y, Matrix &z); + + Param param_; + Semaphore sem_; + + std::atomic_uint64_t sent_q_; + std::atomic_uint64_t hardness_; + std::atomic_uint64_t width_s_; + std::atomic_int64_t conc_reqs_; + + bool terminated_; + std::unique_ptr reporter_; + std::random_device rd; + std::mt19937 gen; +}; +} // namespace onerf diff --git a/apps/OneRF/inc/mysql.hpp b/apps/OneRF/inc/mysql.hpp new file mode 100644 index 0000000..28bb48d --- /dev/null +++ b/apps/OneRF/inc/mysql.hpp @@ -0,0 +1,35 @@ +#pragma once + +#define DISABLE_MYSQL_BACK +#ifndef DISABLE_MYSQL_BACK + +#include +#include +#include +#include + +#include "backend.hpp" +#include + +namespace onerf { +class MySqlBack : public Backend { +public: + MySqlBack(int port = 33000); + ~MySqlBack() override; + bool Request(const std::vector &ids, + std::map &ret_map) override; + void connect(); + void disconnect(); + +private: + std::unique_ptr session_; + // std::unique_ptr db_; + int port_; + + const std::string kDBUser = "onerf"; + const std::string kDBPasswd = "onerf"; + const std::string kDBName = "onerfdb"; +}; +} // namespace onerf + +#endif // DISABLE_MYSQL_BACK \ No newline at end of file diff --git a/apps/OneRF/inc/nuapp.hpp b/apps/OneRF/inc/nuapp.hpp new file mode 100644 index 0000000..98e7e3d --- /dev/null +++ b/apps/OneRF/inc/nuapp.hpp @@ -0,0 +1,11 @@ +#pragma once + +#include "query_types.hpp" + +namespace onerf { +class AppServer { +public: + virtual ~AppServer() = default; + virtual bool ParseRequest(const QueryLayerSlice &layers) = 0; +}; +} \ No newline at end of file diff --git a/apps/OneRF/inc/query_types.hpp b/apps/OneRF/inc/query_types.hpp new file mode 100644 index 0000000..0ab2461 --- /dev/null +++ b/apps/OneRF/inc/query_types.hpp @@ -0,0 +1,59 @@ +#pragma once + +#include +#include +#include +#include + +namespace onerf { +// util classes +struct DepQuery { + std::vector size_vec; // S + std::vector cacheable_vec; // C + std::vector url_vec; // U +}; + +using QueryLayer = std::map; +using QueryLayerSlice = std::vector; + +struct MemLimit { + std::vector limits; + std::vector mallocs; +}; +using MemLimits = std::map; +using PerControllerMemLimit = std::map; + +using HitStat = std::map; +struct HitStatEntry { + std::string raddr; + HitStat hit_stat; +}; + +static inline std::string request_str(const QueryLayerSlice &request) { + std::string str; + str += std::to_string(request.size()); + str += ": [\n"; + for (auto &query : request) { + str += " {\n"; + for (auto &[k, v] : query) { + str += " " + k + " " + std::to_string(v.size_vec.size()); + str += "\n "; + for (auto s : v.size_vec) { + str += std::to_string(s) + " "; + } + str += "\n "; + for (auto s : v.url_vec) { + str += s + " "; + } + str += "\n "; + for (auto s : v.cacheable_vec) { + str += std::to_string(s) + " "; + } + str += "\n"; + } + str += " }\n"; + } + str += "]"; + return str; +} +} // namespace onerf \ No newline at end of file diff --git a/apps/OneRF/inc/requestor.hpp b/apps/OneRF/inc/requestor.hpp new file mode 100644 index 0000000..d832603 --- /dev/null +++ b/apps/OneRF/inc/requestor.hpp @@ -0,0 +1,58 @@ +#pragma once + +#include +#include +#include +#include + +#include "query_types.hpp" +#include "nuapp.hpp" + +namespace onerf { +constexpr static int kNumPerfThds = 20; + +struct PerfReqWithTime { + QueryLayerSlice *req; + uint64_t start_us; +}; + +struct Trace { + uint64_t absl_start_us; + uint64_t start_us; + uint64_t duration_us; +}; + +class Requestor { +public: + Requestor(AppServer *nuapp, + const std::string &trace_file_name = kTraceFileName); + void InitTrace(); + void Perf(double target_kops); + +private: + bool init_timer(); + uint64_t microtime(); + + void prepare_reqs(double target_kops); + std::vector + benchmark(std::vector all_reqs[kNumPerfThds]); + std::vector all_warmup_reqs_[kNumPerfThds]; + std::vector all_perf_reqs_[kNumPerfThds]; + std::vector traces_; + + std::vector requests_; + uint64_t CPU_FREQ; + + AppServer *nuapp_; + std::string trace_file_; + + constexpr static int64_t kMissDDL = 10 * 1000 * 1000; // us + constexpr static char kTraceFileName[] = "tracegen/trace.json"; + // constexpr static int64_t kWarmupDur = 10l * 1000 * 1000; // 10s + // constexpr static int64_t kPerfDur = 300l * 1000 * 1000; // 300s + constexpr static float kWarmupLambda = 200; + constexpr static int64_t kWarmupReqs = 10000; + constexpr static float kExperimentLambda = 400; + constexpr static int64_t kExperimentReqs = 90000; +}; +} // namespace onerf \ No newline at end of file diff --git a/apps/OneRF/inc/semaphore.hpp b/apps/OneRF/inc/semaphore.hpp new file mode 100644 index 0000000..87ec7cd --- /dev/null +++ b/apps/OneRF/inc/semaphore.hpp @@ -0,0 +1,20 @@ +#pragma once + +#include +#include + +namespace onerf { +class Semaphore { +public: + Semaphore(int capacity); + void get(); + void put(); + +private: + int size_; + int capacity_; + + std::mutex mtx_; + std::condition_variable cv_; +}; +} // namespace onerf diff --git a/apps/OneRF/rh_server/main.cpp b/apps/OneRF/rh_server/main.cpp new file mode 100644 index 0000000..52c34db --- /dev/null +++ b/apps/OneRF/rh_server/main.cpp @@ -0,0 +1,12 @@ +#include "rh_nuapp.hpp" +#include "requestor.hpp" + +int main(int argc, char *argv[]) { + onerf::RHAppServer app_server; + onerf::Requestor requestor(&app_server); + + for (int i = 1; i <= 10; i++) + requestor.Perf(i); + + return 0; +} \ No newline at end of file diff --git a/apps/OneRF/rh_server/rh_nuapp.cpp b/apps/OneRF/rh_server/rh_nuapp.cpp new file mode 100644 index 0000000..df32838 --- /dev/null +++ b/apps/OneRF/rh_server/rh_nuapp.cpp @@ -0,0 +1,265 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "fback.hpp" +#include "mysql.hpp" +#include "rh_nuapp.hpp" + +// [midas cache] +#include "cache_manager.hpp" +#include "sync_kv.hpp" + +namespace onerf { + +constexpr static int64_t kTimeout = 10ll * 1000 * 1000 * 1000; // 10s +constexpr static FBack::Param kDefaultFBParam{ + .Hardness = 100, .IdCount = 1, .SizeLower = 8, .SizeUpper = 17}; +constexpr static bool kBypassCache = false; + +SizeGen::SizeGen() : gen(rd()) {} +SizeGen::SizeGen(const std::map &buckets) : gen(rd()) { + init(buckets); +} + +int64_t SizeGen::size() { + assert(!ordered.empty()); + auto val = real_dist(gen); + for (auto k : ordered) { + if (val < k) + return cutoffs[k]; + } + return cutoffs[ordered.back()]; +} + +void SizeGen::init(const std::map &buckets) { + float cutoff = 0; + for (auto &[k, v] : buckets) { + cutoff += v; + cutoffs[cutoff] = std::stoll(k); + ordered.emplace_back(cutoff); + } +} + +SubSystem::SubSystem(std::string name_, std::string type, int max_cache_conns, + int max_back_conns) + : name(name_), cache_sem(max_cache_conns), back_sem(max_back_conns) { + auto cmanager = midas::CacheManager::global_cache_manager(); + if (!cmanager->create_pool(name)) { + std::cerr << "Failed to create cache pool " << name << std::endl; + exit(-1); + } + pool_ = cmanager->get_pool(name); + pool_->update_limit(1ll * 1024 * 1024 * 1024); // 1GB + cache_client = std::make_unique>(pool_); + if (type == "fb") { + FBack::Param kFBParam{ + .Hardness = 400, .IdCount = 1, .SizeLower = 7, .SizeUpper = 16}; + back_client = std::make_unique(kFBParam, max_back_conns); + } else if (type == "mysql") { + constexpr static FBack::Param kMySqlParam{ + .Hardness = 425, .IdCount = 1, .SizeLower = 10, .SizeUpper = 19}; + back_client = std::make_unique(kMySqlParam, max_back_conns); + // NOTE: RobinHood didn't provide mysql config so we use FB to simulate it. + // back_client = std::make_unique(33000); + } +} + +RHAppServer::RHAppServer() { init_subsystems(); } + +bool RHAppServer::ExecSubQuery(std::mutex &lats_mtx, + std::vector &ret_lats, + SubSystem *subsys, const std::string dep, + const std::vector &urls, + const std::vector cacheables) { + auto start_time = std::chrono::high_resolution_clock::now(); + bool has_error = false; + if (!subsys) + return false; + std::map real_size; + for (auto &url : urls) { + // TODO: supposely we should convert url to lower case + real_size[url] = -1; + } + std::vector cache_queries; + std::map cache_hits; + std::vector back_queries; + for (auto &[k, _] : real_size) { + cache_queries.emplace_back(dep + ":" + k); + } + auto fulfilled = 0; + if (!kBypassCache && subsys->cache_client) { // cache query + auto cache_client = subsys->cache_client.get(); + midas::kv_types::BatchPlug plug; + cache_client->batch_stt(plug); + for (auto &k : cache_queries) { + std::string real_key = k.substr((dep + ":").length()); + auto cache_key = midas::kv_utils::make_key(k.c_str(), k.length()); + auto [value, vlen] = cache_client->bget_single(cache_key, plug); + if (value) { + cache_hits[real_key] = true; + real_size[real_key] = vlen; + fulfilled++; + free(value); + } else { + back_queries.emplace_back(real_key); + } + } + cache_client->batch_end(plug); + } else { + for (auto &k : cache_queries) { + std::string real_key = k.substr((dep + ":").length()); + back_queries.emplace_back(real_key); + } + } + auto hit_time = std::chrono::high_resolution_clock::now(); + auto miss_time = hit_time; + // cache miss path + if (!has_error && !back_queries.empty()) { + std::map vals; + subsys->back_sem.get(); + auto succ = subsys->back_client->Request(back_queries, vals); + miss_time = std::chrono::high_resolution_clock::now(); + if (succ) { + for (auto &[key, item] : vals) { + real_size[key] = item.length(); + fulfilled++; + // set into cache + std::string cache_key = dep + ":" + key; + auto cache_client = subsys->cache_client.get(); + cache_client->set(cache_key.c_str(), cache_key.length(), item.c_str(), + item.length()); + } + } else { + std::cerr << "Backend error " << dep << std::endl; + has_error = true; + } + subsys->back_sem.put(); + } + + if (fulfilled != real_size.size()) + has_error = true; + + // calculate latency. RobinHood only + int64_t hit_lat, miss_lat; + Latency::Status status = Latency::Status::ERROR; + if (has_error) { + hit_lat = miss_lat = kTimeout; + } else { + hit_lat = std::chrono::duration_cast(hit_time - + start_time) + .count(); // ns + miss_lat = std::chrono::duration_cast(miss_time - + start_time) + .count(); + if (fulfilled == real_size.size()) + status = Latency::Status::HIT; + } + Latency lat{ + .latency = miss_lat, .status = status, .type = dep, .critical_path = ""}; + { + std::unique_lock ul(lats_mtx); + ret_lats.emplace_back(lat); + } + + for (auto &[key, item_size] : real_size) { + auto cur_lat = miss_lat; + auto cur_status = status; + if (cache_hits[key]) { + cur_lat = hit_lat; + cur_status = Latency::Status::HIT; + } + Latency lat{.latency = cur_lat, + .status = cur_status, + .type = dep, + .critical_path = ""}; + { + std::unique_lock ul(rh_ctrl_.measure_mtx); + rh_ctrl_.lat_measurements.emplace_back(lat); + } + // TODO: add to query sequence + } + return true; +} + +bool RHAppServer::ParseRequest(const QueryLayerSlice &layers) { + auto start_time = std::chrono::high_resolution_clock::now(); + Latency::Status req_state = Latency::Status::ERROR; + std::string slowest_depname; + int64_t slowest_latency = 0; + for (auto &layer : layers) { + std::vector> futures; + std::mutex lats_mtx; + std::vector sq_lats; + for (auto &[dep, queries] : layer) { + auto iter = subs_.find(dep); + if (iter == subs_.cend()) + continue; // skip + auto subsys = iter->second.get(); + auto future = + std::async(std::launch::async, [&, &dep = dep, &queries = queries] { + return ExecSubQuery(lats_mtx, sq_lats, subsys, dep, queries.url_vec, + queries.cacheable_vec); + }); + futures.emplace_back(std::move(future)); + } + + for (auto &future : futures) { + future.get(); + } + for (auto &query_result : sq_lats) { + if (query_result.latency > slowest_latency) { + slowest_latency = query_result.latency; + slowest_depname = query_result.type; + } + if (query_result.status == Latency::Status::MISS) + req_state = Latency::Status::MISS; + else if (query_result.status == Latency::Status::HIT && + req_state != Latency::Status::MISS) + req_state = Latency::Status::HIT; + } + } + int64_t response_time = kTimeout; + if (req_state != Latency::Status::MISS) { + auto end_time = std::chrono::high_resolution_clock::now(); + response_time = std::chrono::duration_cast( + end_time - start_time) + .count(); + } + { + std::unique_lock ul(rh_ctrl_.measure_mtx); + Latency lat{.latency = response_time, + .status = req_state, + .type = "req", + .critical_path = slowest_depname}; + rh_ctrl_.lat_measurements.emplace_back(lat); + } + + return true; +} + +void RHAppServer::init_subsystems() { + int max_cache_conns = 100; + int max_back_conns = 100; + std::vector fb_instances{ + "63956c27", "5b63fdf5", "812126d3", "64c1ce15", "df1794e4", "ac59f41b", + "4607349c", "6a2ef110", "c1042784", "e8fc6018", "b02bdd0d"}; + std::vector mysql_instances{"d6018659", "b4fbebd8", "7385c12d", + "b293d37d", "9ee74b0b", "39f00c48", + "e5fffc73", "1289b3bb", "30eaf8be"}; + + for (auto &instance : fb_instances) { + subs_[instance] = std::make_unique( + instance, "fb", max_cache_conns, max_back_conns); + } + for (auto &instance : mysql_instances) { + subs_[instance] = std::make_unique( + instance, "mysql", max_cache_conns, max_back_conns); + } +} +} // namespace onerf \ No newline at end of file diff --git a/apps/OneRF/rh_server/rh_nuapp.hpp b/apps/OneRF/rh_server/rh_nuapp.hpp new file mode 100644 index 0000000..de177e1 --- /dev/null +++ b/apps/OneRF/rh_server/rh_nuapp.hpp @@ -0,0 +1,95 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "backend.hpp" +#include "fback.hpp" +#include "mysql.hpp" +#include "nuapp.hpp" +#include "query_types.hpp" +#include "semaphore.hpp" + +// [midas cache] +#include "cache_manager.hpp" +#include "sync_kv.hpp" +constexpr static int kNumBuckets = 1 << 20; + +namespace onerf { +struct SubSystem { + std::string name; + Semaphore cache_sem; + std::unique_ptr> cache_client; + + Semaphore back_sem; + std::unique_ptr back_client; + void *shadow_cache; + + SubSystem(std::string name, std::string type, int max_cache_conns, + int max_back_conns); + +private: + midas::CachePool *pool_; +}; + +struct SizeGen { + std::random_device rd; + std::mt19937 gen; + std::uniform_real_distribution real_dist; + std::map cutoffs; + std::vector ordered; + + SizeGen(); + SizeGen(const std::map &buckets); + + int64_t size(); + +private: + void init(const std::map &buckets); +}; + +struct Latency { + int64_t latency; + enum class Status { MISS, HIT, ERROR } status; + std::string type; // request type + std::string critical_path; // critical path: only set iff type == "req" then + // slowest request type +}; +using Results = std::vector; + +class RHController { +public: + std::mutex measure_mtx; + std::list lat_measurements; + // size_t max_nr_measures; + + // private: + // constexpr static size_t kMaxNrMeasures = 100 * 1000; +}; + +// App server with RobinHood +class RHAppServer : public AppServer { +public: + RHAppServer(); + + bool ParseRequest(const QueryLayerSlice &layers) override; + + bool ExecSubQuery(std::mutex &ret_mtx, std::vector &ret_lats, + SubSystem *subsys, const std::string dep, + const std::vector &urls, + const std::vector cacheables); + +private: + void init_subsystems(); + void report_latency(); + void process_queries(); + void report_hit_ratio(); + + std::map> subs_; + RHController rh_ctrl_; +}; +} // namespace onerf \ No newline at end of file diff --git a/apps/OneRF/src/fback.cpp b/apps/OneRF/src/fback.cpp new file mode 100644 index 0000000..af7733a --- /dev/null +++ b/apps/OneRF/src/fback.cpp @@ -0,0 +1,131 @@ +#include +#include +#include +#include + +#include "fback.hpp" + +namespace onerf { +FBack::FBack(const Param ¶m, int max_conn) + : param_(param), sem_(max_conn), sent_q_(0), hardness_(0), width_s_(0), + conc_reqs_(0), terminated_(false), gen(rd()) { + reporter_ = std::make_unique([&] { Report(); }); +} + +FBack::~FBack() { + terminated_ = true; + reporter_->join(); +} + +bool FBack::Request(const std::vector &ids, + std::map &res_map) { + if (ids.empty()) { + std::cerr << "Empty fback ids" << std::endl; + return false; + } + + sem_.get(); + auto param = param_; + param.IdCount = ids.size(); + std::vector reply; + auto succ = GetIDs(param, reply); + sem_.put(); + if (succ) { + if (ids.size() == reply.size()) { + for (int i = 0; i < ids.size(); i++) { + res_map[ids[i]].reserve(reply[i]); + } + } else { + std::cerr << "Incorrect reply count" << std::endl; + return false; + } + } + return true; +} + +bool FBack::GetIDs(const Param ¶m, std::vector &data) { + conc_reqs_++; + calculate(param.Hardness); + auto rlen = param.IdCount; + for (int i = 0; i < rlen; i++) { + std::uniform_int_distribution dist(0, param.SizeUpper - + param.SizeLower); + uint32_t blocksize = + 1 << reinterpret_cast(param.SizeLower + dist(gen)); + data.push_back(blocksize + 4); + } + + sent_q_++; + hardness_ += param.Hardness; + width_s_ += rlen; + conc_reqs_--; + return true; +} + +void FBack::Report() { + while (!terminated_) { + auto sentq = sent_q_.load(); + // if (sentq > 0) { + // std::cout << sentq << " " << hardness_ << " " << width_s_ / sentq << " " + // << conc_reqs_ << std::endl; + // } + sent_q_ = 0; + hardness_ = 0; + width_s_ = 0; + std::this_thread::sleep_for(std::chrono::milliseconds(1000)); + } +} + +void FBack::calculate(int hardness) { + Matrix x; + Matrix y; + x.init(hardness, hardness); + y.init(hardness, hardness); + Matrix z; + dot(x, y, z); + x.free(); + y.free(); + z.free(); +} + +bool FBack::dot(const Matrix &x, const Matrix &y, Matrix &z) { + // this essentially restricts x and y to be square matrices + if (x.cols != y.rows || x.rows != y.cols) { + std::cerr << "Wrong matrix format!" << std::endl; + return false; + } + z.init(x.rows, y.rows); + for (int i = 0; i < x.rows; i++) { + for (int j = 0; j < y.rows; j++) { + z.data[i][j] = x.data[i][j] * y.data[j][i]; + } + } + + return true; +} + +FBack::Matrix::Matrix() : data(nullptr), rows(0), cols(0) {} +FBack::Matrix::~Matrix() { free(); } + +void FBack::Matrix::init(int rows_, int cols_) { + rows = rows_; + cols = cols_; + data = new float *[rows]; + for (int i = 0; i < rows; i++) { + data[i] = new float[cols]; + for (int j = 0; j < cols; j++) { + data[i][j] = i + j; // random fill + } + } +} + +void FBack::Matrix::free() { + if (!data) + return; + for (int i = 0; i < rows; i++) + delete[] data[i]; + delete[] data; + data = nullptr; + rows = cols = 0; +} +} // namespace onerf diff --git a/apps/OneRF/src/mysql.cpp b/apps/OneRF/src/mysql.cpp new file mode 100644 index 0000000..8778e24 --- /dev/null +++ b/apps/OneRF/src/mysql.cpp @@ -0,0 +1,66 @@ +#include "mysql.hpp" +#include +#include + +#ifndef DISABLE_MYSQL_BACK + +namespace onerf { +MySqlBack::MySqlBack(int port) : port_(port) { + connect(); +} + +MySqlBack::~MySqlBack() { disconnect(); } + +bool MySqlBack::Request(const std::vector &ids, + std::map &ret_map) { + if (ids.empty()) { + std::cerr << "Empty mysql ids" << std::endl; + return false; + } + + // Execute a query + std::string query; + if (!ids.empty()) { + query = "SELECT oid, value FROM blobs WHERE oid in (?"; + for (int i = 0; i < ids.size(); i++) + query += ",?"; + query += ") LIMIT " + std::to_string(ids.size()); + } else { + query = "SELECT oid, value FROM blobs WHERE oid=? LIMIT 1"; + } + mysqlx::SqlResult result = session_->sql(query).execute(); + + // Fetch the results + auto &columns = result.getColumns(); + std::cout << "Columns: "; + for (auto & col : columns) { + std::cout << col.getColumnName() << " "; + } + std::cout << std::endl; + std::cout << "Results:" << std::endl; + while (mysqlx::Row row = result.fetchOne()) { + std::cout << row[0].get() << " " << row[1].get() + << std::endl; + } + return true; +} + +void MySqlBack::connect() { + // Connect to the database + // std::string uri = "mysqlx://" + kDBUser + ":" + kDBPasswd + + // "@localhost:" + std::to_string(port_) + "/" + kDBName; + // + "?timeout=10s&writeTimeout=10s&readTimeout=10s"; + std::string uri = "mysqlx://" + kDBUser + ":" + kDBPasswd + + "@localhost:" + "/" + kDBName; + std::cout << "Connect to " << uri << std::endl; + session_ = std::make_unique(uri); + // db_ = std::unique_ptr(session_->getSchema("mydb")); +} + +void MySqlBack::disconnect() { + if (session_) + session_->close(); +} +} // namespace onerf + +#endif // DISABLE_MYSQL_BACK \ No newline at end of file diff --git a/apps/OneRF/src/requestor.cpp b/apps/OneRF/src/requestor.cpp new file mode 100644 index 0000000..1ba4df2 --- /dev/null +++ b/apps/OneRF/src/requestor.cpp @@ -0,0 +1,213 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "query_types.hpp" +#include "requestor.hpp" +#include + +namespace onerf { +using json = nlohmann::json; + +static inline uint64_t rdtscp() { + uint32_t a, d, c; + asm volatile("rdtscp" : "=a"(a), "=d"(d), "=c"(c)); + return ((uint64_t)a) | (((uint64_t)d) << 32); +} + +static inline void json_to_request(json &json_data, QueryLayerSlice &request) { + auto &json_qarray = json_data["d"]; + for (const json &query : json_qarray) { + QueryLayer qlayer; + for (const auto &[instance, vecs] : query.items()) { + DepQuery dep_query; + dep_query.size_vec = vecs["S"].get>(); + dep_query.cacheable_vec = vecs["C"].get>(); + dep_query.url_vec = vecs["U"].get>(); + qlayer[instance] = dep_query; + } + request.emplace_back(qlayer); + } +} + +Requestor::Requestor(AppServer *nuapp, const std::string &trace_file_name) + : nuapp_(nuapp), trace_file_(trace_file_name) { + init_timer(); + InitTrace(); + // InitReqTimes(); +} + +void Requestor::InitTrace() { + std::ifstream input_file(trace_file_); + if (!input_file) { + std::cerr << "Failed to open input file!" << std::endl; + exit(1); + } + + int64_t trace_len = 0; + std::string line; + while (std::getline(input_file, line)) { + json json_data; + try { + json_data = nlohmann::json::parse(line); + QueryLayerSlice request; + json_to_request(json_data, request); + requests_.emplace_back(request); + // std::cout << json_data.dump() << std::endl; + } catch (const std::exception &e) { + std::cerr << "Failed to parse input JSON data: " << e.what() << std::endl; + } + trace_len++; + } + + std::cout << "Finish load trace! In total " << requests_.size() << " " + << trace_len << std::endl; +} + +void Requestor::Perf(double target_kops) { + prepare_reqs(target_kops); + std::cout << "Start warm up..." << std::endl; + benchmark(all_warmup_reqs_); + std::cout << "Done warm up" << std::endl; + traces_ = std::move(benchmark(all_perf_reqs_)); + auto real_duration_us = std::accumulate( + traces_.begin(), traces_.end(), 0ul, [](uint64_t ret, Trace t) { + return std::max(ret, t.start_us + t.duration_us); + }); + auto real_kops = + static_cast(traces_.size()) * 1000 / real_duration_us; + std::cout << traces_.size() << std::endl; + std::cout << "Target Tput: " + << static_cast(kExperimentReqs) * 1000 / real_duration_us + << " KOps" << std::endl; + std::cout << "Real Tput: " << real_kops << " KOps" << std::endl; +} + +bool Requestor::init_timer() { + auto stt_time = std::chrono::high_resolution_clock::now(); + uint64_t stt_cycles = rdtscp(); + volatile int64_t sum = 0; + for (int i = 0; i < 10000000; i++) { + sum += i; + } + uint64_t end_cycles = rdtscp(); + auto end_time = std::chrono::high_resolution_clock::now(); + uint64_t dur_ns = + std::chrono::duration_cast(end_time - stt_time) + .count(); + CPU_FREQ = (end_cycles - stt_cycles) * 1000 / dur_ns; + std::cout << "Timer initialized, CPU Freq: " << CPU_FREQ << " MHz" + << std::endl; + return true; +} + +inline uint64_t Requestor::microtime() { return rdtscp() / CPU_FREQ; } + +std::vector +Requestor::benchmark(std::vector all_reqs[kNumPerfThds]) { + std::vector thds; + std::vector all_traces[kNumPerfThds]; + + for (int i = 0; i < kNumPerfThds; i++) + all_traces[i].reserve(all_reqs[i].size()); + + for (int i = 0; i < kNumPerfThds; i++) { + thds.emplace_back([&, tid = i] { + auto &reqs = all_reqs[tid]; + auto &traces = all_traces[tid]; + // std::cout << tid << " reqs.size " << reqs.size() << std::endl; + + auto start_us = microtime(); + for (const auto &req : reqs) { + auto relative_us = microtime() - start_us; + if (req.start_us > relative_us) { + std::this_thread::sleep_for( + std::chrono::microseconds(req.start_us - relative_us)); + } else if (req.start_us + kMissDDL < relative_us) { + continue; + } + Trace trace; + trace.absl_start_us = microtime(); + trace.start_us = trace.absl_start_us - start_us; + bool succ = nuapp_->ParseRequest(*req.req); + trace.duration_us = microtime() - start_us - trace.start_us; + if (succ) + traces.emplace_back(trace); + } + }); + } + + for (auto &thd : thds) + thd.join(); + + std::vector gathered_traces; + for (int i = 0; i < kNumPerfThds; i++) { + gathered_traces.insert(gathered_traces.end(), all_traces[i].begin(), + all_traces[i].end()); + } + return gathered_traces; +} + +void Requestor::prepare_reqs(double target_kops) { + assert(kWarmupReqs + kExperimentReqs <= requests_.size()); + for (int i = 0; i < kNumPerfThds; i++) { + all_warmup_reqs_[i].clear(); + all_perf_reqs_[i].clear(); + } + + std::vector thds; + for (int i = 0; i < kNumPerfThds; i++) { + thds.emplace_back([&, tid = i] { + std::random_device rd; + std::mt19937 gen(rd()); + + { // warmup reqs + std::exponential_distribution d(target_kops / 1000 / + kNumPerfThds); // ms -> us + const auto chunk_size = (kWarmupReqs + kNumPerfThds - 1) / kNumPerfThds; + const auto stt_req_idx = chunk_size * tid; + const auto end_req_idx = + std::min(stt_req_idx + chunk_size, kWarmupReqs); + + uint64_t cur_us = 0; + for (int j = stt_req_idx; j < end_req_idx; j++) { + PerfReqWithTime prwt; + prwt.start_us = cur_us; + prwt.req = &requests_[j]; + auto interval = std::max(1l, std::lround(d(gen))); + cur_us += interval; + all_warmup_reqs_[tid].emplace_back(prwt); + } + } + { // perf reqs + std::exponential_distribution d(target_kops / 1000 / + kNumPerfThds); // ms -> us + const auto chunk_size = + (kExperimentReqs + kNumPerfThds - 1) / kNumPerfThds; + const auto stt_req_idx = kWarmupReqs + chunk_size * tid; + const auto end_req_idx = + std::min(stt_req_idx + chunk_size, kWarmupReqs + kExperimentReqs); + + uint64_t cur_us = 0; + for (int j = stt_req_idx; j < end_req_idx; j++) { + PerfReqWithTime prwt; + prwt.start_us = cur_us; + prwt.req = &requests_[j]; + auto interval = std::max(1l, std::lround(d(gen))); + cur_us += interval; + all_perf_reqs_[tid].emplace_back(prwt); + } + } + }); + } + for (auto &thd : thds) + thd.join(); +} + +} // namespace onerf \ No newline at end of file diff --git a/apps/OneRF/src/semaphore.cpp b/apps/OneRF/src/semaphore.cpp new file mode 100644 index 0000000..1311449 --- /dev/null +++ b/apps/OneRF/src/semaphore.cpp @@ -0,0 +1,26 @@ +#include +#include + +#include "semaphore.hpp" + +namespace onerf { +Semaphore::Semaphore(int capacity) : size_(0), capacity_(capacity) {} + +void Semaphore::get() { + std::unique_lock ul(mtx_); + while (size_ >= capacity_) + cv_.wait(ul, [&] { return size_ < capacity_; }); + size_++; +} + +void Semaphore::put() { + std::unique_lock ul(mtx_); + bool full = size_ >= capacity_; + if (full) + assert(size_ == capacity_); + size_--; + ul.unlock(); + if (full) + cv_.notify_one(); +} +} // namespace onerf \ No newline at end of file diff --git a/apps/build_all.sh b/apps/build_all.sh new file mode 100755 index 0000000..8390805 --- /dev/null +++ b/apps/build_all.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +function build() { + pushd $SCRIPT_DIR/$1 + make -j$(nproc) + popd +} + +build HDSearch +build storage +build synthetic diff --git a/apps/storage/.gitignore b/apps/storage/.gitignore new file mode 100644 index 0000000..05c5c80 --- /dev/null +++ b/apps/storage/.gitignore @@ -0,0 +1,2 @@ +storage_server +*.bin diff --git a/apps/storage/Makefile b/apps/storage/Makefile new file mode 100644 index 0000000..76c6586 --- /dev/null +++ b/apps/storage/Makefile @@ -0,0 +1,36 @@ +CXX = ccache g++ +LDXX = ccache g++ +CXXFLAGS += -std=c++1z -O2 +CXXFLAGS += -march=native # required for avx-enhanced rmemcpy +# CXXFLAGS += -g -O0 +CXXFLAGS += -rdynamic + +INC += -I../../inc +libmidas_lib := ../../lib/libmidas++.a -lrt -ldl +LIBS += $(libmidas_lib) + +INC += -Iinc +LIBS += -pthread + +APP_BIN = storage_server +APP_SRC = server.cpp +APP_OBJ = $(APP_SRC:%.cpp=%.o) +APP_DEP = $(APP_OBJ:.o=.d) + +all: $(APP_BIN) + +$(APP_BIN) : $(APP_OBJ) + @mkdir -p $(@D) + $(CXX) $(CXXFLAGS) $^ -o $@ $(LIBS) + +%.o : %.cpp Makefile + @mkdir -p $(@D) + $(CXX) $(CXXFLAGS) $(INC) -MMD -MP -c $< -o $@ + +ifneq ($(MAKECMDGOALS),clean) +-include $(APP_DEP) +endif + +.PHONY : clean +clean : + -rm -rf $(APP_BIN) $(APP_OBJ) $(APP_DEP) \ No newline at end of file diff --git a/apps/storage/run.sh b/apps/storage/run.sh new file mode 100755 index 0000000..dca3a63 --- /dev/null +++ b/apps/storage/run.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +taskset -c 24,26,28,30 \ + ./storage_server \ + 2>&1 | tee midas.log diff --git a/apps/storage/server.cpp b/apps/storage/server.cpp new file mode 100644 index 0000000..290ffc1 --- /dev/null +++ b/apps/storage/server.cpp @@ -0,0 +1,136 @@ +#include +#include +#include +#include +#include +#include +#include + +#include "server.hpp" +// [Midas] +#include "perf.hpp" +#include "zipf.hpp" + +namespace storage { +Server::Server() + : disk_file_(kDiskPath, std::ios::in | std::ios::out | std::ios::binary) { + if (!disk_file_.is_open()) { + std::cout << "Error opening file!" << std::endl; + exit(-1); + } else { + disk_file_.seekg(0, std::ios::end); + std::streampos size = disk_file_.tellg(); + disk_file_.seekg(0); + std::cout << "File size: " << size << " bytes, " << size / kPageSize + << " pages" << std::endl; + } + auto cmanager = midas::CacheManager::global_cache_manager(); + assert(cmanager->create_pool("page-cache")); + pool_ = cmanager->get_pool("page-cache"); + pool_->update_limit(5ll * 1024 * 1024 * 1024); + page_cache_ = std::make_unique>(pool_, kNumPages); + + for (int i = 0; i < kNumThds; i++) { + std::random_device rd; + gens[i] = std::make_unique(rd()); + zipf_dist[i] = std::make_unique>( + kNumPages, kSkewness); + op_dist[i] = std::make_unique>(0.0, 1.0); + } +} + +Server::~Server() { disk_file_.close(); } + +bool Server::construct(size_t page_idx) { + Page page; + { + std::unique_lock ul(disk_mtx_); + disk_file_.seekg(page_idx * kPageSize, std::ios::beg); + disk_file_.read(page, sizeof(Page)); + int num_read = disk_file_.gcount(); + assert(num_read == sizeof(Page)); + } + bool succ = page_cache_->set(page_idx, page); + return true; +} + +bool Server::read(size_t page_idx) { + if (page_idx >= kNumPages) + return false; + auto cached = page_cache_->get(page_idx); + if (!cached) { + auto stt = midas::Time::get_cycles_stt(); + construct(page_idx); + auto end = midas::Time::get_cycles_end(); + pool_->record_miss_penalty(end - stt, sizeof(Page)); + } + return true; +} + +bool Server::write(size_t page_idx) { + Page page; + + if (page_idx >= kNumPages) + return false; + else { + { + std::unique_lock ul(disk_mtx_); + // disk_file_.seekg(0); // reset the pointer + disk_file_.seekg(page_idx * kPageSize, std::ios::beg); + disk_file_.write(page, sizeof(Page)); + } + bool succ = page_cache_->set(page_idx, page); + return true; + } +} + +void Server::warmup() { + for (int64_t i = 0; i < kNumPages; i++) { + read(i); + } + std::cout << "Done warmup!" << std::endl; +} + +std::unique_ptr Server::gen_req(int tid) { + auto req = std::make_unique(); + req->tid = tid; + req->op = + (*op_dist[tid])(*gens[tid]) < kReadRatio ? PgReq::READ : PgReq::WRITE; + req->pg_idx = (*zipf_dist[tid])(*gens[tid]); + + return req; +} + +bool Server::serve_req(int tid, const midas::PerfRequest *req_) { + auto *req = dynamic_cast(req_); + if (req->op == PgReq::READ) + read(req->pg_idx); + else if (req->op == PgReq::WRITE) + write(req->pg_idx); + else + std::cerr << "Unknown OP code: " << req->op << std::endl; + return true; +} +} // namespace storage + +int run() { + storage::Server server; + server.warmup(); + for (int i = 0; i < 100; i++) { + midas::Perf perf(server); + auto target_kops = 800; + auto duration_us = 1500ull * midas::to_us; + auto warmup_us = 10ull * midas::to_us; + auto miss_ddl = 10ull * midas::to_us; + perf.run(storage::kNumThds, target_kops, duration_us, warmup_us, miss_ddl); + std::cout << "Real Tput: " << perf.get_real_kops() << " Kops" << std::endl; + std::cout << "P99 Latency: " << perf.get_nth_lat(99) << " us" << std::endl; + } + + return 0; +} + +int main(int argc, char *argv[]) { + run(); + return 0; +} diff --git a/apps/storage/server.hpp b/apps/storage/server.hpp new file mode 100644 index 0000000..0450145 --- /dev/null +++ b/apps/storage/server.hpp @@ -0,0 +1,79 @@ +#pragma once + +#include +#include +#include +#include +#include + +// [midas] +#include "array.hpp" +#include "cache_manager.hpp" +#include "time.hpp" +#include "perf.hpp" +#include "zipf.hpp" + +namespace storage { +constexpr static double kSkewness = 0.99; +constexpr static uint32_t kNumThds = 4; +constexpr static float kReadRatio = 0.7; +constexpr static float kWriteRatio = 1.0 - kReadRatio; + +constexpr static uint64_t kPageSize = 4096; // 4KB +constexpr static uint64_t kNumPages = 4 * 1024 * 1024; // 4M pages = 16GB +constexpr static char kDiskPath[] = "disk.bin"; + +using Page = char[kPageSize]; + +struct PgReq : public midas::PerfRequest { + int tid; + enum { READ, WRITE } op; + int pg_idx; +}; + +template class SyncArray { +public: + SyncArray(int n) : array_(n) {} + SyncArray(midas::CachePool *pool, int n) : array_(pool, n) {} + std::unique_ptr get(int idx) { + auto &lock = locks_[idx % kNumChunks]; + std::unique_lock ul(lock); + return array_.get(idx); + } + bool set(int idx, const T &t) { + auto &lock = locks_[idx % kNumChunks]; + std::unique_lock ul(lock); + return array_.set(idx, t); + } + +private: + constexpr static int kNumChunks = 1 << 10; + std::mutex locks_[kNumChunks]; + midas::Array array_; +}; + +class Server : public midas::PerfAdapter { +public: + Server(); + ~Server(); + + bool read(size_t pg_idx); + bool write(size_t pg_idx); + + bool construct(size_t pg_idx); + + void warmup(); + std::unique_ptr gen_req(int tid) override; + bool serve_req(int tid, const midas::PerfRequest *req) override; + +private: + midas::CachePool *pool_; + std::unique_ptr> page_cache_; + std::mutex disk_mtx_; + std::fstream disk_file_; + + std::unique_ptr gens[kNumThds]; + std::unique_ptr> zipf_dist[kNumThds]; + std::unique_ptr> op_dist[kNumThds]; +}; +} \ No newline at end of file diff --git a/apps/storage/setup_disk.sh b/apps/storage/setup_disk.sh new file mode 100755 index 0000000..916d33c --- /dev/null +++ b/apps/storage/setup_disk.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +DISK_NAME=disk.bin +DISK_SIZE=16G + +fallocate -l ${DISK_SIZE} ${DISK_NAME} \ No newline at end of file diff --git a/apps/synthetic/.gitignore b/apps/synthetic/.gitignore new file mode 100644 index 0000000..d12d687 --- /dev/null +++ b/apps/synthetic/.gitignore @@ -0,0 +1 @@ +synthetic \ No newline at end of file diff --git a/apps/synthetic/Makefile b/apps/synthetic/Makefile new file mode 100644 index 0000000..2baab88 --- /dev/null +++ b/apps/synthetic/Makefile @@ -0,0 +1,36 @@ +CXX = ccache g++ +LDXX = ccache g++ +CXXFLAGS += -std=c++1z -O2 +CXXFLAGS += -march=native # required for avx-enhanced rmemcpy +# CXXFLAGS += -g -O0 +CXXFLAGS += -rdynamic + +INC += -I../../inc +libmidas_lib := ../../lib/libmidas++.a -lrt -ldl +LIBS += $(libmidas_lib) + +INC += -Iinc +LIBS += -pthread + +APP_BIN = synthetic +APP_SRC = main.cpp +APP_OBJ = $(APP_SRC:%.cpp=%.o) +APP_DEP = $(APP_OBJ:.o=.d) + +all: $(APP_BIN) + +$(APP_BIN) : $(APP_OBJ) + @mkdir -p $(@D) + $(CXX) $(CXXFLAGS) $^ -o $@ $(LIBS) + +%.o : %.cpp Makefile + @mkdir -p $(@D) + $(CXX) $(CXXFLAGS) $(INC) -MMD -MP -c $< -o $@ + +ifneq ($(MAKECMDGOALS),clean) +-include $(APP_DEP) +endif + +.PHONY : clean +clean : + -rm -rf $(APP_BIN) $(APP_OBJ) $(APP_DEP) \ No newline at end of file diff --git a/apps/synthetic/main.cpp b/apps/synthetic/main.cpp new file mode 100644 index 0000000..c8a393d --- /dev/null +++ b/apps/synthetic/main.cpp @@ -0,0 +1,131 @@ +#include +#include +#include +#include +#include +#include + +// [Midas] +#include "cache_manager.hpp" +#include "perf.hpp" +#include "time.hpp" +#include "utils.hpp" +#include "zipf.hpp" +#include "sync_kv.hpp" + +namespace synthetic { + +constexpr static int kNumBuckets = 1 << 20; +constexpr static float kSkewness = 0.99; +constexpr static int kNumThds = 12; +constexpr static int kComputeCost = 30; +constexpr static int64_t kVSize = 4000; +constexpr static int64_t kNumValues = 2 * 1024 * 1024; +constexpr static int64_t kCacheSize = (sizeof(int) + kVSize) * kNumValues * 1.1; +constexpr static float kCacheRatio = 0.2; + +struct Value { + char data[kVSize]; +}; + +struct Req : public midas::PerfRequest { + int tid; + int key; +}; + +class App : public midas::PerfAdapter { +public: + App(int recon_cost_ = 16) : recon_cost(recon_cost_) { + auto cmanager = midas::CacheManager::global_cache_manager(); + cmanager->create_pool("synthetic"); + pool = cmanager->get_pool("synthetic"); + assert(pool); + pool->update_limit(kCacheRatio * kCacheSize); + cache = std::make_unique>(pool); + + for (int i = 0; i < kNumThds; i++) { + std::random_device rd; + gens[i] = std::make_unique(rd()); + zipf_dists[i] = std::make_unique>( + kNumValues, kSkewness); + } + } + + std::unique_ptr gen_req(int tid) override { + int key = (*zipf_dists[tid])(*gens[tid]); + + auto req = std::make_unique(); + req->tid = tid; + req->key = key; + + return req; + } + + bool serve_req(int tid, const midas::PerfRequest *req_) override { + auto *req = dynamic_cast(req_); + compute(req->key); + return true; + } + + void compute(int key) { + auto v = cache->get(key); + if (!v) { + reconstruct(key); + } + std::this_thread::sleep_for(std::chrono::microseconds(kComputeCost)); + } + + void reconstruct(int key) { + auto stt = midas::Time::get_cycles_stt(); + // std::this_thread::sleep_for(std::chrono::microseconds(recon_cost)); + // usleep(recon_cost); + while (true) { + volatile int tmp = 0; + for (int i = 0; i < 50 * recon_cost; i++) + tmp++; + auto ts = midas::Time::get_cycles_end(); + if ((ts - stt) >= recon_cost * midas::kCPUFreq) + break; + } + Value v; + cache->set(key, v); + auto end = midas::Time::get_cycles_end(); + pool->record_miss_penalty(end - stt, midas::kPageSize); + } + + void warmup() { + std::cout << "Start warm up..." << std::endl; + for (int i = 0; i < kNumValues; i++) { + Value v; + cache->set(i, v); + } + std::cout << "Warm up done!" << std::endl; + } + + int recon_cost; + midas::CachePool *pool; + std::unique_ptr> cache; + + std::unique_ptr gens[kNumThds]; + std::unique_ptr> zipf_dists[kNumThds]; +}; +} // namespace synthetic + + +int main() { + std::vector recon_costs = {0, 1, 2, 4, 8, 16, 32, + 64, 128, 256, 512, 1024, 2048, 4096}; + synthetic::App app; + app.warmup(); + for (auto recon_cost : recon_costs) { + app.recon_cost = recon_cost; + midas::Perf perf(app); + auto target_kops = 200; + auto duration_us = 20ull * midas::to_us; + auto warmup_us = 0ull * midas::to_us; + auto miss_ddl = 1ull * midas::to_us; + perf.run(synthetic::kNumThds, target_kops, duration_us, warmup_us, miss_ddl); + std::cout << "Real Tput: " << perf.get_real_kops() << " Kops" << std::endl; + // std::cout << "P99 Latency: " << perf.get_nth_lat(99) << " us" << std::endl; + } +} \ No newline at end of file diff --git a/bin/.gitkeep b/bin/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/bindings/c/.gitignore b/bindings/c/.gitignore new file mode 100644 index 0000000..3312abb --- /dev/null +++ b/bindings/c/.gitignore @@ -0,0 +1,45 @@ +**/libmidas.a +**/libmidas.so +test + +# Prerequisites +*.d + +# Compiled Object files +*.slo +*.lo +*.o +*.obj + +# Precompiled Headers +*.gch +*.pch + +# Compiled Dynamic libraries +*.so +*.dylib +*.dll + +# Fortran module files +*.mod +*.smod + +# Compiled Static libraries +*.lai +*.la +*.a +*.lib + +# Executables +bin/* +*.exe +*.out +*.app + +# Clangd index +**/.cache/* +**/compile_commands.json +**/.clangd/* + +# vscode +**/.vscode/* \ No newline at end of file diff --git a/bindings/c/Makefile b/bindings/c/Makefile new file mode 100644 index 0000000..7598d19 --- /dev/null +++ b/bindings/c/Makefile @@ -0,0 +1,67 @@ +CXXFLAGS += -std=c++1z -O2 +CXXFLAGS += -march=native # required for avx-enhanced rmemcpy +CXXFLAGS += -fPIC +# CXXFLAGS += -Wall -Wextra +# CXXFLAGS += -g -O0 +CXX = g++ +LDXX = g++ + +CFLAGS += -O2 -fPIC +# CFLAGS += -march=native +CC = gcc +LD = gcc + +INC += -I../../inc -I./include +INC += -I/mnt/ssd/yifan/tools/boost_1_79_0 + +# override LDFLAGS += -static -static-libstdc++ -static-libgcc +override LDFLAGS += -lrt -pthread +# For stacktrace logging +override LDFLAGS += -ldl + +root_dir = ../.. +lib_src = $(wildcard $(root_dir)/src/*.cpp) +lib_src := $(filter-out $(wildcard src/*main.cpp),$(lib_src)) +lib_obj = $(lib_src:.cpp=.o) + +bindings_src = $(wildcard *.cpp) +bindings_obj = $(bindings_src:.cpp=.o) + +test_src = test.c +test_obj = $(test_src:.c=.o) + +src = $(bindings_src) $(test_src) +obj = $(bindings_obj) $(test_obj) +dep = $(obj:.o=.d) + +.PHONY: all clean install + +target = lib/libmidas.a \ + test + +# libmidas.so \ + +all: $(target) + +test: $(test_obj) lib/libmidas.a + $(LD) -o $@ $^ $(LDFLAGS) -lstdc++ + +lib/libmidas.a: $(bindings_obj) $(lib_obj) + mkdir -p lib + $(AR) rcs $@ $^ + +libmidas.so: $(bindings_obj) $(lib_obj) + $(LD) -o $@ $^ -shared $(LDFLAGS) + +%.o: %.cpp Makefile + $(CXX) $(CXXFLAGS) $(INC) -MMD -MP -c $< -o $@ + +%.o: %.c Makefile + $(CC) $(CFLAGS) $(INC) -MMD -MP -c $< -o $@ + +ifneq ($(MAKECMDGOALS),clean) +-include $(dep) +endif + +clean: + $(RM) $(dep) $(obj) $(target) diff --git a/bindings/c/cache_manager_ffi.cpp b/bindings/c/cache_manager_ffi.cpp new file mode 100644 index 0000000..73adb95 --- /dev/null +++ b/bindings/c/cache_manager_ffi.cpp @@ -0,0 +1,85 @@ +#include "cache_manager.h" + +#include "../../inc/cache_manager.hpp" + +bool midas_create_cache_pool(const char name[]) { + auto c_mgr = midas::CacheManager::global_cache_manager(); + return c_mgr->create_pool(name); +} + +bool midas_delete_cache_pool(const char name[]) { + auto c_mgr = midas::CacheManager::global_cache_manager(); + return c_mgr->delete_pool(name); +} + +cache_pool_t midas_get_cache_pool(const char name[]) { + auto c_mgr = midas::CacheManager::global_cache_manager(); + return c_mgr->get_pool(name); +} + +cache_pool_t midas_get_global_cache_pool(void) { + return midas::CachePool::global_cache_pool(); +} + +void midas_pool_set_construct_func(cache_pool_t pool, + midas_construct_func_t callback) { + auto pool_ = reinterpret_cast(pool); + if (pool_) + pool_->set_construct_func(callback); +} + +bool midas_pool_has_construct_func(cache_pool_t pool) { + auto pool_ = reinterpret_cast(pool); + if (!pool_) + return false; + return (bool)pool_->get_construct_func(); +} + +int midas_pool_construct(cache_pool_t pool, void *arg) { + auto pool_ = reinterpret_cast(pool); + if (!pool_) + return -1; + return pool_->construct(arg); +} + +int midas_pool_update_limit(cache_pool_t pool, uint64_t limit_in_bytes) { + auto pool_ = reinterpret_cast(pool); + if (!pool_) + return -1; + pool_->update_limit(limit_in_bytes); + return 0; +} + +int midas_pool_set_weight(cache_pool_t pool, float weight) { + auto pool_ = reinterpret_cast(pool); + if (!pool_) + return -1; + pool_->set_weight(weight); + return 0; +} + +void midas_inc_cache_hit(cache_pool_t pool) { + auto pool_ = reinterpret_cast(pool); + if (pool_) + pool_->inc_cache_hit(); +} + +void midas_inc_cache_miss(cache_pool_t pool) { + auto pool_ = reinterpret_cast(pool); + if (pool_) + pool_->inc_cache_miss(); +} + +void midas_inc_cache_victim_hit(cache_pool_t pool) { + auto pool_ = reinterpret_cast(pool); + if (pool_) + pool_->inc_cache_victim_hit(); +} + +void midas_record_miss_penalty(cache_pool_t pool, uint64_t cycles, + uint64_t bytes) { + auto pool_ = reinterpret_cast(pool); + assert(pool_); + if (pool_) + pool_->record_miss_penalty(cycles, bytes); +} diff --git a/bindings/c/include/cache_manager.h b/bindings/c/include/cache_manager.h new file mode 100644 index 0000000..72fa090 --- /dev/null +++ b/bindings/c/include/cache_manager.h @@ -0,0 +1,38 @@ +#ifndef __MIDAS_CACHE_MANAGER_H__ +#define __MIDAS_CACHE_MANAGER_H__ + +#include + +#include "midas_utils.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef void *cache_pool_t; + +bool midas_create_cache_pool(const char name[]); +bool midas_delete_cache_pool(const char name[]); +cache_pool_t midas_get_cache_pool(const char name[]); +cache_pool_t midas_get_global_cache_pool(void); +int midas_pool_update_limit(cache_pool_t pool, uint64_t limit_in_bytes); +int midas_pool_set_weight(cache_pool_t pool, float weight); + +typedef int (*midas_construct_func_t)(void *arg); + +void midas_pool_set_construct_func(cache_pool_t pool, + midas_construct_func_t callback); +bool midas_pool_get_construct_func(cache_pool_t pool); +int midas_pool_construct(cache_pool_t pool, void *arg); + +void midas_inc_cache_hit(cache_pool_t pool); +void midas_inc_cache_miss(cache_pool_t pool); +void midas_inc_cache_victim_hit(cache_pool_t pool); +void midas_record_miss_penalty(cache_pool_t pool, uint64_t cycles, + uint64_t bytes); + +#ifdef __cplusplus +} +#endif + +#endif // __MIDAS_CACHE_MANAGER_H__ \ No newline at end of file diff --git a/bindings/c/include/midas.h b/bindings/c/include/midas.h new file mode 100644 index 0000000..9755cd3 --- /dev/null +++ b/bindings/c/include/midas.h @@ -0,0 +1,17 @@ +#ifndef __MIDAS_H__ +#define __MIDAS_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "cache_manager.h" +#include "midas_utils.h" +#include "resource_manager.h" +#include "softmem.h" + +#ifdef __cplusplus +} +#endif + +#endif // __MIDAS_H__ \ No newline at end of file diff --git a/bindings/c/include/midas_utils.h b/bindings/c/include/midas_utils.h new file mode 100644 index 0000000..8e5e90c --- /dev/null +++ b/bindings/c/include/midas_utils.h @@ -0,0 +1,27 @@ +#ifndef __MIDAS_UTILS_H__ +#define __MIDAS_UTILS_H__ + +#include + +// high resolution timer functions +static inline uint64_t rdtsc(void) { + uint32_t a, d; + __asm__ volatile("rdtsc" : "=a"(a), "=d"(d)); + return ((uint64_t)a) | (((uint64_t)d) << 32); +} + +static inline uint64_t rdtscp(void) { + uint32_t a, d, c; + __asm__ volatile("rdtscp" : "=a"(a), "=d"(d), "=c"(c)); + return ((uint64_t)a) | (((uint64_t)d) << 32); +} + +static inline uint64_t get_cycles_stt(void) { + return rdtscp(); +} + +static inline uint64_t get_cycles_end(void) { + return rdtscp(); +} + +#endif // __MIDAS_UTILS_H__ \ No newline at end of file diff --git a/bindings/c/include/resource_manager.h b/bindings/c/include/resource_manager.h new file mode 100644 index 0000000..20afa79 --- /dev/null +++ b/bindings/c/include/resource_manager.h @@ -0,0 +1,16 @@ +#ifndef __MIDAS_RESOURCE_MANAGER_H +#define __MIDAS_RESOURCE_MANAGER_H + +#ifdef __cplusplus +extern "C" { +#endif + +typedef void * resource_manager_t; + +resource_manager_t midas_get_global_manager(void); + +#ifdef __cplusplus +} +#endif + +#endif // __MIDAS_RESOURCE_MANAGER_H \ No newline at end of file diff --git a/bindings/c/include/softmem.h b/bindings/c/include/softmem.h new file mode 100644 index 0000000..9f318df --- /dev/null +++ b/bindings/c/include/softmem.h @@ -0,0 +1,32 @@ +#ifndef __MIDAS_SOFTMEM_H +#define __MIDAS_SOFTMEM_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include + +#include "cache_manager.h" + +typedef uint64_t object_ptr_t; + +extern object_ptr_t midas_alloc_soft(const cache_pool_t pool, size_t size); +extern bool midas_free_soft(const cache_pool_t pool, object_ptr_t optr); + +extern bool midas_copy_from_soft(const cache_pool_t pool, void *dest, + const object_ptr_t src, size_t len, + int64_t offset); +extern bool midas_copy_to_soft(const cache_pool_t pool, object_ptr_t dest, + const void *src, size_t len, int64_t offset); + +extern bool midas_soft_ptr_null(const object_ptr_t optr); +extern bool midas_soft_ptr_is_victim(const object_ptr_t optr); +extern bool midas_soft_contains(const object_ptr_t optr, const uint64_t addr); + +#ifdef __cplusplus +} +#endif + +#endif // __MIDAS_SOFTMEM_H \ No newline at end of file diff --git a/bindings/c/resource_manager_ffi.cpp b/bindings/c/resource_manager_ffi.cpp new file mode 100644 index 0000000..c4205a6 --- /dev/null +++ b/bindings/c/resource_manager_ffi.cpp @@ -0,0 +1,6 @@ +#include "resource_manager.h" +#include "../../inc/resource_manager.hpp" + +resource_manager_t midas_get_global_manager(void) { + return midas::ResourceManager::global_manager(); +}; diff --git a/bindings/c/softmem_ffi.cpp b/bindings/c/softmem_ffi.cpp new file mode 100644 index 0000000..c76243a --- /dev/null +++ b/bindings/c/softmem_ffi.cpp @@ -0,0 +1,123 @@ +#include "softmem.h" + +#include "../../inc/object.hpp" +#include "../../inc/cache_manager.hpp" +#include "../../inc/log.hpp" + +#include +#include + +namespace midas { +namespace ffi { +constexpr static uint64_t kFreeListSize = 100000; +// static thread_local std::list local_free_list; +static std::list global_free_list; +static std::mutex fl_mtx; + +static inline ObjectPtr *alloc_optr() { + std::unique_lock ul(fl_mtx); + if (global_free_list.empty()) { + for (int i = 0; i < kFreeListSize; i++) { + global_free_list.emplace_back(new ObjectPtr()); + } + } + auto optr = global_free_list.front(); + global_free_list.pop_front(); + return optr; +} + +static inline void free_optr(ObjectPtr *optr) { + std::unique_lock ul(fl_mtx); + global_free_list.emplace_back(optr); +} + +// static inline ObjectPtr *alloc_optr() { +// if (local_free_list.empty()) { +// std::unique_lock ul(fl_mtx); +// if (global_free_list.empty()) { +// for (int i = 0; i < kFreeListSize; i++) { +// local_free_list.emplace_back(new ObjectPtr()); +// } +// } else { +// auto it = global_free_list.begin(); +// uint64_t len = std::min(global_free_list.size(), kFreeListSize); +// std::advance(it, len); // move all elements +// local_free_list.splice(local_free_list.begin(), global_free_list, +// global_free_list.begin(), it); +// } +// assert(!local_free_list.empty()); +// } +// auto optr = local_free_list.front(); +// local_free_list.pop_front(); +// return optr; +// } + +// static inline void free_optr(ObjectPtr *optr) { +// local_free_list.emplace_back(optr); +// if (local_free_list.size() > kFreeListSize) { +// auto it = local_free_list.begin(); +// uint64_t len = std::min(local_free_list.size(), kFreeListSize); +// std::advance(it, len); + +// std::unique_lock ul(fl_mtx); +// global_free_list.splice(global_free_list.begin(), local_free_list, +// local_free_list.begin(), it); +// } +// } +} // namespace ffi +} // namespace midas + +object_ptr_t midas_alloc_soft(const cache_pool_t pool, size_t size) { + auto pool_ = reinterpret_cast(pool); + if (!pool_) + return reinterpret_cast(nullptr); + // auto optr = new midas::ObjectPtr(); + auto optr = midas::ffi::alloc_optr(); + if (!optr) { + midas::MIDAS_LOG(midas::kError) << "Failed to allocate!"; + return reinterpret_cast(nullptr); + } + auto allocator = pool_->get_allocator(); + if (!allocator->alloc_to(size, optr)) { + // delete optr; + midas::ffi::free_optr(optr); + return reinterpret_cast(nullptr); + } + return reinterpret_cast(optr); +} + +bool midas_free_soft(const cache_pool_t pool, object_ptr_t optr) { + auto optr_ = reinterpret_cast(optr); + auto pool_ = reinterpret_cast(pool); + auto ret = pool_->free(*optr_); + // delete optr_; + midas::ffi::free_optr(optr_); + return ret; +} + +bool midas_copy_from_soft(const cache_pool_t pool, void *dest, + const object_ptr_t src, size_t len, int64_t offset) { + auto optr = reinterpret_cast(src); + return optr->copy_to(dest, len, offset); +} + +bool midas_copy_to_soft(const cache_pool_t pool, object_ptr_t dest, + const void *src, size_t len, int64_t offset) { + auto optr = reinterpret_cast(dest); + return optr->copy_from(src, len, offset); +} + +bool midas_soft_ptr_null(const object_ptr_t optr) { + auto optr_ = reinterpret_cast(optr); + return optr_->null(); +} + +bool midas_soft_ptr_is_victim(const object_ptr_t optr) { + auto optr_ = reinterpret_cast(optr); + return optr_->is_victim(); +} + +bool midas_soft_ptr_contains(const object_ptr_t optr, const uint64_t addr) { + auto optr_ = reinterpret_cast(optr); + return optr_->contains(addr); +} \ No newline at end of file diff --git a/bindings/c/test.c b/bindings/c/test.c new file mode 100644 index 0000000..b9cc221 --- /dev/null +++ b/bindings/c/test.c @@ -0,0 +1,16 @@ +#include +#include + +#include +#include + +int main(int argc, char *argv[]) { + resource_manager_t rmanager = midas_get_global_manager(); + printf("get midas resource manager @ %p\n", rmanager); + bool succ = midas_create_cache_pool("test"); + cache_pool_t pool; + pool = midas_get_cache_pool("test"); + object_ptr_t optr = midas_alloc_soft(pool, 10); + sleep(1); // leave some time for rmanager to establish connections. + return 0; +} \ No newline at end of file diff --git a/config/.gitkeep b/config/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/daemon/daemon_types.cpp b/daemon/daemon_types.cpp new file mode 100644 index 0000000..9367f10 --- /dev/null +++ b/daemon/daemon_types.cpp @@ -0,0 +1,1008 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "inc/daemon_types.hpp" +#include "logging.hpp" +#include "qpair.hpp" +#include "shm_types.hpp" +#include "utils.hpp" + +namespace midas { +/** Global control flags */ +/* by default set to "false" and Midas deamon will automatically detect system + * avail memory. Set it to "true" to simulate memory pressure by arbitrarily + * update the memory cfg file. */ +constexpr static bool kEnableMemPressureSimu = true; +constexpr static bool kEnableDynamicRebalance = true; +constexpr static Daemon::Policy kDefaultPolicy = Daemon::Policy::Midas; +// WARNING: two flags below should always be enabled to adapt clients' memory +// usage to the amount of server's idle memory +constexpr static bool kEnableProfiler = true; +constexpr static bool kEnableRebalancer = true; +/** Monitor related */ +constexpr static uint32_t kMonitorInteral = 1; // in seconds +/** Profiler related */ +constexpr static float kPerfZeroThresh = 1; +constexpr static uint32_t kProfInterval = 5; // in seconds +constexpr static float KProfWDecay = 0.3; +/** Rebalancer related */ +constexpr static float kExpandFactor = 0.5; +constexpr static uint32_t kMaxExpandThresh = 128; +constexpr static int32_t kWarmupRounds = 2; +/** Server related */ +constexpr static uint32_t kAliveTimeout = 3; // in seconds +constexpr static uint32_t kReclaimTimeout = 5; // in seconds +constexpr static uint32_t kServeTimeout = 1; // in seconds + +/** Client */ +Client::Client(Daemon *daemon, uint64_t id_, uint64_t region_limit) + : daemon_(daemon), status(ClientStatusCode::INIT), id(id_), region_cnt_(0), + region_limit_(region_limit), weight_(1), warmup_ttl_(0), + lat_critical_(false), cq(utils::get_ackq_name(kNameCtrlQ, id), false), + txqp(std::to_string(id), false) { + daemon_->charge(region_limit_); +} + +Client::~Client() { + daemon_->uncharge(region_limit_); + cq.destroy(); + txqp.destroy(); + destroy(); +} + +void Client::connect() { + CtrlMsg ack{.op = CtrlOpCode::CONNECT, + .ret = CtrlRetCode::CONN_SUCC, + .mmsg{.size = region_limit_ * kRegionSize}}; + cq.send(&ack, sizeof(ack)); + status = ClientStatusCode::CONNECTED; +} + +void Client::disconnect() { + status = ClientStatusCode::DISCONNECTED; + CtrlMsg ret_msg{.op = CtrlOpCode::DISCONNECT, .ret = CtrlRetCode::CONN_SUCC}; + cq.send(&ret_msg, sizeof(ret_msg)); +} + +/** @overcommit: evacuator might request several overcommitted regions for + * temporary usages. It will soon return more regions back. + */ +bool Client::alloc_region_(bool overcommit) { + CtrlRetCode ret = CtrlRetCode::MEM_FAIL; + MemMsg mm; + + if (overcommit || region_cnt_ < region_limit_) { + int64_t region_id = new_region_id_(); + const auto rwmode = boost::interprocess::read_write; + const std::string region_name = utils::get_region_name(id, region_id); + boost::interprocess::permissions perms; + perms.set_unrestricted(); + + if (regions.find(region_id) == regions.cend()) { + int64_t actual_size; + auto region = std::make_shared( + boost::interprocess::create_only, region_name.c_str(), rwmode, perms); + regions[region_id] = region; + region_cnt_++; + + region->truncate(kRegionSize); + region->get_size(actual_size); + + ret = CtrlRetCode::MEM_SUCC; + mm.region_id = region_id; + mm.size = actual_size; + } else { + /* region has already existed */ + MIDAS_LOG(kError) << "Client " << id << " has already allocated region " + << region_id; + + ret = CtrlRetCode::MEM_FAIL; + } + } + + CtrlMsg ret_msg{.op = CtrlOpCode::ALLOC, .ret = ret, .mmsg = mm}; + cq.send(&ret_msg, sizeof(ret_msg)); + return ret == CtrlRetCode::MEM_SUCC; +} + +bool Client::free_region(int64_t region_id) { + CtrlRetCode ret = CtrlRetCode::MEM_FAIL; + MemMsg mm; + int64_t actual_size; + + auto region_iter = regions.find(region_id); + if (region_iter != regions.end()) { + /* Successfully find the region to be freed */ + region_iter->second->get_size(actual_size); + regions.erase(region_id); + SharedMemObj::remove(utils::get_region_name(id, region_id).c_str()); + region_cnt_--; + + ret = CtrlRetCode::MEM_SUCC; + mm.region_id = region_id; + mm.size = actual_size; + } else { + /* Failed to find corresponding region */ + MIDAS_LOG(kError) << "Client " << id << " doesn't have region " + << region_id; + ret = CtrlRetCode::MEM_FAIL; + } + + CtrlMsg ack{.op = CtrlOpCode::FREE, .ret = ret, .mmsg = mm}; + cq.send(&ack, sizeof(ack)); + return ret == CtrlRetCode::MEM_SUCC; +} + +void Client::set_weight(float weight) { + weight_ = weight; + MIDAS_LOG(kInfo) << "Client " << id << " set weight to " << weight_; +} + +void Client::set_lat_critical(bool value) { + lat_critical_ = value; + MIDAS_LOG(kInfo) << "Client " << id << " set lat_critical to " + << lat_critical_; +} + +bool Client::update_limit(uint64_t new_limit) { + if (new_limit + kForceReclaimThresh < region_limit_) + return force_reclaim(new_limit); + if (new_limit == region_limit_) + return true; + daemon_->charge(new_limit - region_limit_); + region_limit_ = new_limit; + + std::unique_lock ul(tx_mtx); + CtrlMsg msg{.op = CtrlOpCode::UPDLIMIT, + .ret = CtrlRetCode::MEM_FAIL, + .mmsg{.size = region_limit_}}; + txqp.send(&msg, sizeof(msg)); + CtrlMsg ack; + if (txqp.timed_recv(&ack, sizeof(ack), kReclaimTimeout) != 0) { + MIDAS_LOG(kError) << "Client " << id << " timed out!"; + return false; + } + if (ack.mmsg.size != region_cnt_) { + MIDAS_LOG(kError) << ack.mmsg.size << " != " << region_cnt_; + } + // assert(ack.mmsg.size <= region_cnt_); + if (ack.ret != CtrlRetCode::MEM_SUCC) { + MIDAS_LOG(kError) << "Client " << id << " failed to reclaim " << region_cnt_ + << "/" << region_limit_; + } + return true; +} + +bool Client::force_reclaim(uint64_t new_limit) { + if (new_limit == region_limit_) + return true; + daemon_->charge(new_limit - region_limit_); + region_limit_ = new_limit; + + std::unique_lock ul(tx_mtx); + CtrlMsg msg{.op = CtrlOpCode::FORCE_RECLAIM, + .ret = CtrlRetCode::MEM_FAIL, + .mmsg{.size = region_limit_}}; + txqp.send(&msg, sizeof(msg)); + CtrlMsg ack; + if (txqp.timed_recv(&ack, sizeof(ack), kReclaimTimeout) != 0) { + MIDAS_LOG(kError) << "Client " << id << " timed out!"; + return false; + } + // assert(ack.mmsg.size <= region_cnt_); + if (ack.ret != CtrlRetCode::MEM_SUCC) { + MIDAS_LOG(kError) << "Client " << id << " failed to force reclaim " + << region_cnt_ << "/" << region_limit_; + } + return true; +} + +bool Client::profile_stats() { + std::unique_lock ul(tx_mtx); + CtrlMsg msg{.op = CtrlOpCode::PROF_STATS}; + txqp.send(&msg, sizeof(msg)); + StatsMsg statsmsg; + int ret = txqp.timed_recv(&statsmsg, sizeof(statsmsg), kAliveTimeout); + if (ret != 0) { + MIDAS_LOG(kWarning) << "Client " << id << " is dead!"; + return false; + } + if (statsmsg.hits || statsmsg.misses) { + MIDAS_LOG(kDebug) << "Client " << id << " " << statsmsg.hits << " " + << statsmsg.misses << " " << statsmsg.miss_penalty << " " + << statsmsg.vhits; + } + // update historical stats + stats.hits = stats.hits * KProfWDecay + statsmsg.hits * (1 - KProfWDecay); + stats.misses = + stats.misses * KProfWDecay + statsmsg.misses * (1 - KProfWDecay); + stats.vhits = stats.vhits * KProfWDecay + statsmsg.vhits * (1 - KProfWDecay); + stats.penalty = + stats.penalty * KProfWDecay + statsmsg.miss_penalty * (1 - KProfWDecay); + stats.perf_gain = weight_ * stats.penalty * stats.vhits; + stats.headroom = std::max(1, statsmsg.headroom); + return true; +} + +void Client::destroy() { + for (const auto &kv : regions) { + const std::string name = utils::get_region_name(id, kv.first); + SharedMemObj::remove(name.c_str()); + } +} + +bool Client::almost_full() noexcept { + auto threshold = std::max(kMaxExpandThresh, 4 * stats.headroom); + return region_cnt_ >= region_limit_ - threshold; +} + +/** Daemon */ +Daemon::Daemon(const std::string ctrlq_name) + : ctrlq_name_(utils::get_rq_name(ctrlq_name, true)), + ctrlq_(ctrlq_name_, true, kDaemonQDepth, kMaxMsgSize), region_cnt_(0), + region_limit_(kMaxRegions), terminated_(false), + status_(MemStatus::NORMAL), policy(kDefaultPolicy), monitor_(nullptr), + profiler_(nullptr), rebalancer_(nullptr) { + monitor_ = std::make_shared([&] { monitor(); }); + if (kEnableProfiler) + profiler_ = std::make_shared([&] { profiler(); }); + if (kEnableRebalancer) + rebalancer_ = std::make_shared([&] { rebalancer(); }); +} + +Daemon::~Daemon() { + terminated_ = true; + { + std::unique_lock ul(rbl_mtx_); + rbl_cv_.notify_all(); + } + if (monitor_) + monitor_->join(); + if (profiler_) + profiler_->join(); + if (rebalancer_) + rebalancer_->join(); + clients_.clear(); + MsgQueue::remove(ctrlq_name_.c_str()); +} + +int Daemon::do_connect(const CtrlMsg &msg) { + try { + std::unique_lock ul(mtx_); + if (clients_.find(msg.id) != clients_.cend()) { + MIDAS_LOG(kError) << "Client " << msg.id << " connected twice!"; + /* TODO: this might be some stale client. Probably we could try to + * send the ack back */ + return -1; + } + ul.unlock(); + auto client = std::make_shared(this, msg.id, kInitRegions); + client->connect(); + MIDAS_LOG(kInfo) << "Client " << msg.id << " connected."; + ul.lock(); + clients_.insert(std::make_pair(msg.id, std::move(client))); + ul.unlock(); + } catch (boost::interprocess::interprocess_exception &e) { + MIDAS_LOG(kError) << e.what(); + } + + return 0; +} + +int Daemon::do_disconnect(const CtrlMsg &msg) { + try { + std::unique_lock ul(mtx_); + auto client_iter = clients_.find(msg.id); + if (client_iter == clients_.cend()) { + /* TODO: this might be some unregistered client. Probably we could try to + * connect to it and send the ack back */ + MIDAS_LOG(kError) << "Client " << msg.id << " doesn't exist!"; + return -1; + } + auto client = std::move(client_iter->second); + clients_.erase(msg.id); + ul.unlock(); + client->disconnect(); + MIDAS_LOG(kInfo) << "Client " << msg.id << " disconnected!"; + } catch (boost::interprocess::interprocess_exception &e) { + MIDAS_LOG(kError) << e.what(); + } + + return 0; +} + +int Daemon::do_alloc(const CtrlMsg &msg) { + assert(msg.mmsg.size == kRegionSize); + std::unique_lock ul(mtx_); + auto client_iter = clients_.find(msg.id); + if (client_iter == clients_.cend()) { + /* TODO: same as in do_disconnect */ + MIDAS_LOG(kError) << "Client " << msg.id << " doesn't exist!"; + return -1; + } + ul.unlock(); + auto &client = client_iter->second; + assert(msg.id == client->id); + client->alloc_region(); + + return 0; +} + +int Daemon::do_overcommit(const CtrlMsg &msg) { + assert(msg.mmsg.size == kRegionSize); + std::unique_lock ul(mtx_); + auto client_iter = clients_.find(msg.id); + if (client_iter == clients_.cend()) { + /* TODO: same as in do_disconnect */ + MIDAS_LOG(kError) << "Client " << msg.id << " doesn't exist!"; + return -1; + } + ul.unlock(); + auto &client = client_iter->second; + assert(msg.id == client->id); + client->overcommit_region(); + + return 0; +} + +int Daemon::do_free(const CtrlMsg &msg) { + auto client_iter = clients_.find(msg.id); + std::unique_lock ul(mtx_); + if (client_iter == clients_.cend()) { + /* TODO: same as in do_disconnect */ + MIDAS_LOG(kError) << "Client " << msg.id << " doesn't exist!"; + return -1; + } + ul.unlock(); + uint64_t region_id = msg.mmsg.region_id; + size_t region_size = msg.mmsg.size; + auto &client = client_iter->second; + client->free_region(region_id); + + return 0; +} + +int Daemon::do_update_limit_req(const CtrlMsg &msg) { + std::unique_lock ul(mtx_); + auto client_iter = clients_.find(msg.id); + if (client_iter == clients_.cend()) { + /* TODO: same as in do_disconnect */ + MIDAS_LOG(kError) << "Client " << msg.id << " doesn't exist!"; + return -1; + } + ul.unlock(); + auto upd_region_lim = std::min(msg.mmsg.size / kRegionSize, region_limit_); + auto &client = client_iter->second; + if (upd_region_lim != client->region_limit_) { + bool succ = client->update_limit(upd_region_lim); + } + + return 0; +} + +int Daemon::do_set_weight(const CtrlMsg &msg) { + std::unique_lock ul(mtx_); + auto client_iter = clients_.find(msg.id); + if (client_iter == clients_.cend()) { + /* TODO: same as in do_disconnect */ + MIDAS_LOG(kError) << "Client " << msg.id << " doesn't exist!"; + return -1; + } + ul.unlock(); + float weight = msg.mmsg.weight; + auto &client = client_iter->second; + client->set_weight(weight); + return 0; +} + +int Daemon::do_set_lat_critical(const CtrlMsg &msg) { + std::unique_lock ul(mtx_); + auto client_iter = clients_.find(msg.id); + if (client_iter == clients_.cend()) { + /* TODO: same as in do_disconnect */ + MIDAS_LOG(kError) << "Client " << msg.id << " doesn't exist!"; + return -1; + } + ul.unlock(); + bool lat_critical = msg.mmsg.lat_critical; + auto &client = client_iter->second; + client->set_lat_critical(lat_critical); + return 0; +} + +void Daemon::charge(int64_t nr_regions) { + region_cnt_ += nr_regions; + if (region_cnt_ > region_limit_) { + std::unique_lock ul(rbl_mtx_); + status_ = MemStatus::NEED_SHRINK; + rbl_cv_.notify_one(); + } +} + +void Daemon::uncharge(int64_t nr_regions) { + region_cnt_ -= nr_regions; + if (region_cnt_ < region_limit_) { + std::unique_lock ul(rbl_mtx_); + status_ = MemStatus::NEED_EXPAND; + rbl_cv_.notify_one(); + } +} + +void Daemon::monitor() { + while (!terminated_) { + // monitor & update mem limit + auto upd_mem_limit = kEnableMemPressureSimu ? utils::check_file_avail_mem() + : utils::check_sys_avail_mem(); + uint64_t upd_region_limit = upd_mem_limit / kRegionSize; + if (region_limit_ != upd_region_limit) { + if (kEnableMemPressureSimu) + MIDAS_LOG(kDebug) << region_limit_ << " -> " << upd_region_limit; + region_limit_ = upd_region_limit; + if (region_cnt_ > region_limit_) { // invoke rebalancer + std::unique_lock ul(rbl_mtx_); + status_ = MemStatus::NEED_SHRINK; + rbl_cv_.notify_one(); + } + } + + // monitor & update policy + auto new_policy = utils::check_policy(); + if (new_policy >= Policy::Static && new_policy < Policy::NumPolicy && + new_policy != policy) { + MIDAS_LOG(kError) << "Policy changed " << policy << " -> " << new_policy; + policy = static_cast(new_policy); + } + + std::this_thread::sleep_for(std::chrono::seconds(kMonitorInteral)); + } +} + +void Daemon::profiler() { + while (!terminated_) { + std::this_thread::sleep_for(std::chrono::seconds(kProfInterval)); + std::atomic_int_fast32_t nr_active_clients{0}; + std::mutex dead_mtx; + std::vector dead_clients; + std::vector> futures; + std::unique_lock ul(mtx_); + for (auto &[_, client] : clients_) { + auto future = std::async(std::launch::async, [&, &client = client] { + bool alive = client->profile_stats(); + if (!alive) { + std::unique_lock ul(dead_mtx); + dead_clients.emplace_back(client->id); + return; + } + if (client->stats.perf_gain > kPerfZeroThresh) + nr_active_clients++; + }); + futures.emplace_back(std::move(future)); + } + ul.unlock(); + for (auto &f : futures) + f.get(); + ul.lock(); + for (const auto &cid : dead_clients) + clients_.erase(cid); + dead_clients.clear(); + ul.unlock(); + + // invoke rebalancer + if (region_cnt_ < region_limit_) { + if (nr_active_clients > 0) { + std::unique_lock ul(rbl_mtx_); + status_ = MemStatus::NEED_EXPAND; + rbl_cv_.notify_one(); + } + } else if (region_cnt_ == region_limit_) { + std::unique_lock ul(rbl_mtx_); + status_ = MemStatus::NEED_REBALANCE; + rbl_cv_.notify_one(); + } else if (region_cnt_ > region_limit_) { + std::unique_lock ul(rbl_mtx_); + status_ = MemStatus::NEED_SHRINK; + rbl_cv_.notify_one(); + } else { + MIDAS_ABORT("Impossible to reach here!"); + } + } +} + +void Daemon::rebalancer() { + while (!terminated_) { + { + std::unique_lock lk(rbl_mtx_); + rbl_cv_.wait( + lk, [this] { return terminated_ || status_ != MemStatus::NORMAL; }); + } + if (terminated_) + break; + switch (status_) { + case MemStatus::NEED_SHRINK: + on_mem_shrink(); + break; + case MemStatus::NEED_EXPAND: + on_mem_expand(); + break; + case MemStatus::NEED_REBALANCE: + if (policy == Policy::Static) { + /* do nothing */ + } else if (policy == Policy::Midas) + on_mem_rebalance(); + else if (policy == Policy::CliffHanger) + on_mem_rebalance_cliffhanger(); + else { + on_mem_rebalance(); + } + break; + default: + MIDAS_LOG(kError) << "Memory rebalancer is waken up for unknown reason"; + } + status_ = MemStatus::NORMAL; + } +} + +void Daemon::on_mem_shrink() { + while (region_cnt_ > region_limit_) { + int64_t nr_to_reclaim = region_cnt_ - region_limit_; + double nr_reclaim_ratio = static_cast(nr_to_reclaim) / region_cnt_; + + std::vector> inactive_clients; + std::vector> active_clients; + std::unique_lock ul(mtx_); + for (auto &[_, client] : clients_) { + if (client->stats.perf_gain < kPerfZeroThresh) + inactive_clients.emplace_back(client); + else + active_clients.emplace_back(client); + } + ul.unlock(); + for (auto client : inactive_clients) { + int64_t old_limit = client->region_limit_; + int64_t nr_reclaimed = std::ceil(nr_reclaim_ratio * old_limit); + int64_t new_limit = std::max(1l, old_limit - nr_reclaimed); + bool succ = client->update_limit(new_limit); + nr_to_reclaim -= nr_reclaimed; + MIDAS_LOG_PRINTF(kInfo, "Reclaimed client %lu %ld regions, %lu -> %lu\n", + client->id, nr_reclaimed, old_limit, new_limit); + } + if (region_cnt_ <= region_limit_) + break; + double total_reciprocal_gain = 0.0; + for (auto client : active_clients) + total_reciprocal_gain += 1.0 / client->stats.perf_gain; + for (auto client : active_clients) { + auto reciprocal_gain = 1.0 / client->stats.perf_gain; + int64_t old_limit = client->region_limit_; + int64_t nr_reclaimed = std::max( + std::ceil(reciprocal_gain / total_reciprocal_gain * nr_to_reclaim), + 1); + int64_t new_limit = std::max(1l, old_limit - nr_reclaimed); + bool succ = client->update_limit(new_limit); + nr_to_reclaim -= nr_reclaimed; + MIDAS_LOG_PRINTF(kInfo, "Reclaimed client %lu %ld regions, %lu -> %lu\n", + client->id, nr_reclaimed, old_limit, new_limit); + } + } + MIDAS_LOG(kInfo) << "Memory shrink done! Total regions: " << region_cnt_ + << "/" << region_limit_; + std::unique_lock ul(mtx_); + for (auto &[_, client] : clients_) { + MIDAS_LOG(kInfo) << "Client " << client->id + << " regions: " << client->region_cnt_ << "/" + << client->region_limit_; + } +} + +void Daemon::on_mem_expand() { + if (policy == Policy::Static) + return; + + bool expanded = false; + int64_t nr_to_grant = region_limit_ - region_cnt_; + if (nr_to_grant <= 0) + return; + + std::vector> active_clients; + std::vector> empty_clients; + { + std::unique_lock ul(mtx_); + for (auto &[_, client] : clients_) { + if (client->stats.perf_gain > kPerfZeroThresh && client->almost_full()) + active_clients.emplace_back(client); + else if (client->region_limit_ <= 10) + empty_clients.emplace_back(client); + } + } + + if (policy == Policy::ExpandOnly) { + if (!empty_clients.empty()) { // re-enable an empty pool + int64_t nr_per_client = nr_to_grant / empty_clients.size(); + nr_per_client = std::min(kMaxExpandThresh, nr_per_client); + for (auto client : empty_clients) { + nr_to_grant -= nr_per_client; + auto old_limit = client->region_limit_; + auto new_limit = old_limit + nr_per_client; + client->update_limit(new_limit); + expanded = true; + MIDAS_LOG_PRINTF(kInfo, "Grant client %lu %ld regions, %lu -> %lu\n", + client->id, nr_per_client, old_limit, new_limit); + } + } + + int64_t nr_per_client = nr_to_grant / active_clients.size(); + double total_gain = 0.0; + for (auto client : active_clients) + total_gain += std::log2(client->stats.perf_gain); + + for (auto client : active_clients) { + auto gain = std::log2(client->stats.perf_gain); + int64_t nr_per_client = gain / total_gain * nr_to_grant; + + auto old_limit = client->region_limit_; + auto new_limit = old_limit + nr_per_client; + client->update_limit(client->region_limit_ + nr_per_client); + expanded = true; + MIDAS_LOG_PRINTF(kInfo, "Grant client %lu %ld regions, %lu -> %lu\n", + client->id, nr_per_client, old_limit, new_limit); + } + } else { + if (active_clients.empty()) { + if (!empty_clients.empty()) { + // re-enable an empty pool + int64_t nr_per_client = nr_to_grant / empty_clients.size(); + nr_per_client = std::min(kMaxExpandThresh, nr_per_client); + for (auto client : empty_clients) { + client->update_limit(client->region_limit_ + nr_per_client); + expanded = true; + } + } + } else { // expand soft memory grant among active clients + double total_gain = 0.0; + for (auto client : active_clients) + total_gain += std::log2(client->stats.perf_gain); + + for (auto client : active_clients) { + auto gain = std::log2(client->stats.perf_gain); + auto old_limit = client->region_limit_; + int64_t nr_granted = std::min( + std::min(std::ceil(gain / total_gain * nr_to_grant), + std::ceil(old_limit * (1 - kExpandFactor))), + nr_to_grant); + nr_granted = std::min(nr_granted, kMaxExpandThresh); + auto new_limit = old_limit + nr_granted; + bool succ = client->update_limit(new_limit); + nr_to_grant -= nr_granted; + expanded = true; + MIDAS_LOG_PRINTF(kInfo, "Grant client %lu %ld regions, %lu -> %lu\n", + client->id, nr_granted, old_limit, new_limit); + } + } + } + + if (expanded) { + MIDAS_LOG(kInfo) << "Memory expansion done! Total regions: " << region_cnt_ + << "/" << region_limit_; + std::unique_lock ul(mtx_); + for (auto &[_, client] : clients_) { + MIDAS_LOG(kInfo) << "Client " << client->id + << " regions: " << client->region_cnt_ << "/" + << client->region_limit_; + } + } +} + +void Daemon::on_mem_rebalance() { + constexpr static int64_t kMaxReclaim = 64; + int64_t kMaxGrant = 128; + if (!kEnableDynamicRebalance) + return; + + std::vector> idles; + std::vector> victims; + std::vector> candidates; + { + std::unique_lock ul(mtx_); + for (auto &[_, client] : clients_) { + if (client->region_limit_ <= 1) + continue; + if (client->warmup_ttl_ > 0) { + client->warmup_ttl_--; + continue; + } + + if (client->stats.perf_gain <= kPerfZeroThresh) { + if (client->lat_critical_ && client->almost_full()) + continue; + idles.emplace_back(client); + } else + candidates.emplace_back(client); + } + } + if (candidates.empty()) + return; + if (idles.empty()) + kMaxGrant /= 4; // 32; + + // planning + std::sort(candidates.begin(), candidates.end(), + [](std::shared_ptr c1, std::shared_ptr c2) { + return c1->stats.perf_gain > c2->stats.perf_gain; + }); + float max_gain = std::log2(candidates.front()->stats.perf_gain); + float min_gain = std::log2(candidates.back()->stats.perf_gain); + std::vector plan_idle_reclaims(idles.size()); + std::vector plan_adjusts(candidates.size()); + + // planning + int64_t nr_plan_reclaimed = 0; + auto nr_idles = idles.size(); + auto nr_candidates = candidates.size(); + for (int i = 0; i < nr_idles; i++) { + auto idle = idles[i]; + int64_t max_reclaim = idle->lat_critical_ ? kMaxReclaim / 8 : kMaxReclaim; + int64_t nr_to_reclaim = + std::min(max_reclaim, idle->region_limit_ - 1); + plan_idle_reclaims[i] = -nr_to_reclaim; + nr_plan_reclaimed += nr_to_reclaim; + } + + int64_t nr_plan_granted = 0; + int gainer_idx = 0; + int victim_idx = candidates.size(); + while (gainer_idx < victim_idx) { + if (nr_plan_granted < nr_plan_reclaimed) { + auto candidate = candidates[gainer_idx]; + int64_t nr_to_grant = + candidate->almost_full() + ? kMaxGrant * std::log2(candidate->stats.perf_gain) / max_gain + : 0; + plan_adjusts[gainer_idx] = nr_to_grant; + nr_plan_granted += nr_to_grant; + gainer_idx++; + } else { + victim_idx--; + auto victim = candidates[victim_idx]; + int64_t max_reclaim = + victim->lat_critical_ ? kMaxReclaim / 8 : kMaxReclaim; + int64_t nr_to_reclaim = std::min( + max_reclaim * std::log2(victim->stats.perf_gain) / min_gain, + victim->region_limit_ - 1); + plan_adjusts[victim_idx] = -nr_to_reclaim; + nr_plan_reclaimed += nr_to_reclaim; + } + } + assert(gainer_idx == victim_idx); + if (gainer_idx == candidates.size()) { + if (nr_plan_granted < nr_plan_reclaimed) { + // All active clients get their portion but still too much reclaimed + // memory. So we return them back to idle clients evenly. + int64_t nr_overchaged = nr_plan_reclaimed - nr_plan_granted; + int64_t nr_remain = nr_overchaged; + for (int i = 0; i < nr_idles; i++) { + if (plan_idle_reclaims[i] == 0) + continue; + int64_t nr_to_return = std::min( + nr_remain, + (nr_overchaged * (-plan_idle_reclaims[i]) + nr_plan_reclaimed) / + nr_plan_reclaimed); // proportionally return over charged memory + plan_idle_reclaims[i] += nr_to_return; + nr_remain -= nr_to_return; + if (nr_remain <= 0) + break; + } + } else { + plan_adjusts[gainer_idx - 1] += nr_plan_reclaimed - nr_plan_granted; + } + } else { // adjust the last client's grant + // granted more than reclaimed. Try to reclaim more. + if (!candidates[gainer_idx]->lat_critical_) { + plan_adjusts[gainer_idx] += nr_plan_reclaimed - nr_plan_granted; + } else { // protect the lat-critical app + int64_t nr_overgranted = nr_plan_granted - nr_plan_reclaimed; + int64_t nr_remain = nr_overgranted; + int64_t nr_per_client = (nr_remain + gainer_idx - 1) / gainer_idx; + for (int i = gainer_idx - 1; i >= 0; i--) { + plan_adjusts[i] -= std::min(nr_remain, nr_per_client); + nr_remain -= nr_per_client; + if (nr_remain <= 0) + break; + } + } + } + + // applying + bool rebalanced = false; + for (int i = 0; i < nr_idles; i++) { + auto idle = idles[i]; + MIDAS_LOG(kError) << idle->stats.perf_gain << " " << plan_idle_reclaims[i]; + if (plan_idle_reclaims[i] == 0) + continue; + idle->update_limit(idle->region_limit_ + plan_idle_reclaims[i]); + rebalanced = true; + } + for (int i = 0; i < nr_candidates; i++) { + auto candidate = candidates[i]; + MIDAS_LOG(kError) << candidate->stats.perf_gain << " " << plan_adjusts[i]; + if (plan_adjusts[i] == 0) + continue; + candidate->update_limit(candidate->region_limit_ + plan_adjusts[i]); + rebalanced = true; + } + + if (rebalanced) { + MIDAS_LOG(kInfo) << "Memory rebalance done! Total regions: " << region_cnt_ + << "/" << region_limit_; + std::unique_lock ul(mtx_); + for (auto &[_, client] : clients_) { + MIDAS_LOG(kInfo) << "Client " << client->id + << " regions: " << client->region_cnt_ << "/" + << client->region_limit_; + } + } +} + +void Daemon::on_mem_rebalance_cliffhanger() { + if (!kEnableDynamicRebalance) + return; + static uint64_t kStepSize = 16ul; + + std::vector> clients; + { + std::unique_lock ul(mtx_); + for (auto &[_, client] : clients_) { + if (client->region_limit_ <= 1) + continue; + clients.emplace_back(client); + } + } + if (clients.size() <= 1) + return; + + std::sort(clients.begin(), clients.end(), + [](std::shared_ptr c1, std::shared_ptr c2) { + const auto &s1 = c1->stats; + const auto &s2 = c2->stats; + float h1 = -1., h2 = -1.; + if (s1.hits + s1.misses > 0) + h1 = static_cast(s1.hits + s1.vhits) / + (s1.hits + s1.vhits + s1.misses) - + static_cast(s1.hits) / (s1.hits + s1.misses); + if (s2.hits + s2.misses) + h2 = static_cast(s2.hits + s2.vhits) / + (s2.hits + s2.vhits + s2.misses) - + static_cast(s2.hits) / (s2.hits + s2.misses); + + return h1 < h2; + }); + + bool rebalanced = false; + int victim_idx = 0; + int winner_idx = clients.size() - 1; + while (victim_idx < winner_idx) { + // find a winner candidate + while (winner_idx > victim_idx) { + auto winner = clients[winner_idx]; + if (winner->almost_full()) + break; + winner_idx--; + } + if (winner_idx == victim_idx) + break; + + auto winner = clients[winner_idx]; + auto victim = clients[victim_idx]; + + auto nr_to_reclaim = std::min(victim->region_limit_ - 1, kStepSize); + victim->update_limit(victim->region_limit_ - nr_to_reclaim); + winner->update_limit(winner->region_limit_ + nr_to_reclaim); + if (nr_to_reclaim) + rebalanced = true; + + victim_idx++; + } + if (rebalanced) { + MIDAS_LOG(kInfo) << "Memory rebalance done! Total regions: " << region_cnt_ + << "/" << region_limit_; + std::unique_lock ul(mtx_); + for (auto &[_, client] : clients_) { + MIDAS_LOG(kInfo) << "Client " << client->id + << " regions: " << client->region_cnt_ << "/" + << client->region_limit_; + } + } +} + +void Daemon::serve() { + MIDAS_LOG(kInfo) << "Daemon starts listening..."; + + while (!terminated_) { + CtrlMsg msg; + if (ctrlq_.timed_recv(&msg, sizeof(CtrlMsg), kServeTimeout) != 0) + continue; + + MIDAS_LOG(kDebug) << "Daemon recved msg " << msg.op; + switch (msg.op) { + case CONNECT: + do_connect(msg); + break; + case DISCONNECT: + do_disconnect(msg); + break; + case ALLOC: + do_alloc(msg); + break; + case OVERCOMMIT: + do_overcommit(msg); + break; + case FREE: + do_free(msg); + break; + case UPDLIMIT_REQ: + do_update_limit_req(msg); + break; + case SET_WEIGHT: + do_set_weight(msg); + break; + case SET_LAT_CRITICAL: + do_set_lat_critical(msg); + break; + default: + MIDAS_LOG(kError) << "Recved unknown message: " << msg.op; + } + } + + MIDAS_LOG(kInfo) << "Daemon stopped to serve..."; +} + +namespace utils { +uint64_t check_sys_avail_mem() { + struct sysinfo info; + if (sysinfo(&info) != 0) { + MIDAS_LOG(kError) << "Error when calling sysinfo()"; + return -1; + } + + uint64_t avail_mem_bytes = info.freeram * info.mem_unit; + MIDAS_LOG(kDebug) << "Available Memory: " << avail_mem_bytes / (1024 * 1024) + << " MB"; + return avail_mem_bytes; +} + +uint64_t check_file_avail_mem() { + std::ifstream mem_cfg(kMemoryCfgFile); + if (!mem_cfg.is_open()) { + return check_sys_avail_mem(); + } + uint64_t upd_mem_limit; + mem_cfg >> upd_mem_limit; + mem_cfg.close(); + return upd_mem_limit; +} + +Daemon::Policy check_policy() { + std::ifstream policy_cfg(kPolicyCfgFile); + if (!policy_cfg.is_open()) { + return kDefaultPolicy; + } + + int new_policy; + policy_cfg >> new_policy; + policy_cfg.close(); + if (new_policy < Daemon::Policy::Static || + new_policy >= Daemon::Policy::NumPolicy) + return Daemon::Policy::Invalid; + return Daemon::Policy(new_policy); +} +} // namespace utils + +} // namespace midas diff --git a/daemon/inc/daemon_types.hpp b/daemon/inc/daemon_types.hpp new file mode 100644 index 0000000..70996ff --- /dev/null +++ b/daemon/inc/daemon_types.hpp @@ -0,0 +1,166 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "qpair.hpp" +#include "shm_types.hpp" +#include "utils.hpp" + +namespace midas { + +using SharedMemObj = boost::interprocess::shared_memory_object; +using MsgQueue = boost::interprocess::message_queue; + +enum class ClientStatusCode { + INIT, + CONNECTED, + DISCONNECTED, +}; + +struct CacheStats { + uint64_t hits{0}; + uint64_t misses{0}; + double penalty{0.}; + uint64_t vhits{0}; + uint64_t vcache_size{0}; + double perf_gain{0.}; + int32_t headroom{0}; +}; + +class Daemon; + +class Client { +public: + Client(Daemon *daemon, uint64_t id_, uint64_t region_limit); + ~Client(); + + uint64_t id; + ClientStatusCode status; + + void connect(); + void disconnect(); + bool alloc_region(); + bool overcommit_region(); + bool free_region(int64_t region_id); + bool update_limit(uint64_t new_limit); + void set_weight(float weight); + void set_lat_critical(bool value); + bool force_reclaim(uint64_t new_limit); + bool profile_stats(); + bool almost_full() noexcept; + +private: + inline int64_t new_region_id_() noexcept; + inline void destroy(); + + bool alloc_region_(bool overcommit); + + std::mutex tx_mtx; + QSingle cq; // per-client completion queue for the ctrl queue + QPair txqp; + std::unordered_map> regions; + + CacheStats stats; + + Daemon *daemon_; + uint64_t region_cnt_; + uint64_t region_limit_; + float weight_; + int32_t warmup_ttl_; + bool lat_critical_; + + friend class Daemon; +}; + +class Daemon { +public: + Daemon(const std::string ctrlq_name = kNameCtrlQ); + ~Daemon(); + void serve(); + + static Daemon *get_daemon(); + + enum Policy { + Invalid = -1, + Static = 0, + Midas, + CliffHanger, + RobinHood, + ExpandOnly, + NumPolicy + }; + +private: + int do_connect(const CtrlMsg &msg); + int do_disconnect(const CtrlMsg &msg); + int do_alloc(const CtrlMsg &msg); + int do_overcommit(const CtrlMsg &msg); + int do_free(const CtrlMsg &msg); + int do_update_limit_req(const CtrlMsg &msg); + int do_set_weight(const CtrlMsg &msg); + int do_set_lat_critical(const CtrlMsg &msg); + + void charge(int64_t nr_regions); + void uncharge(int64_t nr_regions); + + void monitor(); + void profiler(); + void rebalancer(); + + void on_mem_shrink(); + void on_mem_expand(); + void on_mem_rebalance(); + void on_mem_rebalance_cliffhanger(); + + enum class MemStatus { + NORMAL, + NEED_SHRINK, + NEED_EXPAND, + NEED_REBALANCE + } status_; + std::mutex rbl_mtx_; + std::condition_variable rbl_cv_; + + bool terminated_; + Policy policy; + std::shared_ptr monitor_; + std::shared_ptr profiler_; + std::shared_ptr rebalancer_; + + const std::string ctrlq_name_; + QSingle ctrlq_; + std::mutex mtx_; + std::unordered_map> clients_; + + std::atomic_int_fast64_t region_cnt_; + uint64_t region_limit_; + // For simluation + constexpr static uint64_t kInitRegions = (100ull << 20) / kRegionSize;// 100MB + constexpr static uint64_t kMaxRegions = (100ull << 30) / kRegionSize; // 100GB + + friend class Client; +}; + +namespace utils { +uint64_t check_sys_avail_mem(); +uint64_t check_file_avail_mem(); + +Daemon::Policy check_policy(); + +constexpr static char kMemoryCfgFile[] = "config/mem.config"; +constexpr static char kPolicyCfgFile[] = "config/policy.config"; +} + +} // namespace midas + +#include "impl/daemon_types.ipp" \ No newline at end of file diff --git a/daemon/inc/impl/daemon_types.ipp b/daemon/inc/impl/daemon_types.ipp new file mode 100644 index 0000000..fe7bb15 --- /dev/null +++ b/daemon/inc/impl/daemon_types.ipp @@ -0,0 +1,32 @@ +#pragma once + +namespace midas { + +inline int64_t Client::new_region_id_() noexcept { + static int64_t region_id = 0; + return region_id++; +} + +inline bool Client::alloc_region() { + return alloc_region_(false); +} + +/* for evacuator to temporarily overcommit memory during evacuation. It will + * return more regions afterwards. */ +inline bool Client::overcommit_region() { + return alloc_region_(true); +} + +inline Daemon *Daemon::get_daemon() { + static std::mutex mtx_; + static std::shared_ptr daemon_; + if (daemon_) + return daemon_.get(); + std::unique_lock ul(mtx_); + if (daemon_) + return daemon_.get(); + daemon_ = std::make_shared(); + return daemon_.get(); +} + +} // namespace midas \ No newline at end of file diff --git a/daemon/main.cpp b/daemon/main.cpp new file mode 100644 index 0000000..3f4e06e --- /dev/null +++ b/daemon/main.cpp @@ -0,0 +1,17 @@ +#include + +#include "inc/daemon_types.hpp" + +void signalHandler(int signum) { + // Let the process exit normally so that daemon_ can be naturally destroyed. + exit(signum); +} + +int main(int argc, char *argv[]) { + signal(SIGINT, signalHandler); + + auto daemon = midas::Daemon::get_daemon(); + daemon->serve(); + + return 0; +} \ No newline at end of file diff --git a/exp/harvest_additional_memory/antagonist.sh b/exp/harvest_additional_memory/antagonist.sh new file mode 100644 index 0000000..dc0a9cb --- /dev/null +++ b/exp/harvest_additional_memory/antagonist.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +stt_limit=5120 +end_limit=15360 + +nr_steps=2 +duration=300 +step_size=$((($end_limit - $stt_limit) / nr_steps)) + +mem_limit=$stt_limit +echo 0, $mem_limit +./set_memratio.sh $mem_limit +sleep 600 + +for i in $( seq 1 $nr_steps ) +do + mem_limit=$(($mem_limit + $step_size)) + echo $i, $mem_limit + ./set_memratio.sh $mem_limit + sleep $duration +done diff --git a/exp/harvest_additional_memory/set_memratio.sh b/exp/harvest_additional_memory/set_memratio.sh new file mode 100755 index 0000000..37e6c1a --- /dev/null +++ b/exp/harvest_additional_memory/set_memratio.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +mem_mb=$1 +mem_limit=$(($mem_mb * 1024 * 1024)) + +echo $mem_limit > ../../config/mem.config diff --git a/exp/intense_pressure/antagonist2.sh b/exp/intense_pressure/antagonist2.sh new file mode 100755 index 0000000..d35b0ad --- /dev/null +++ b/exp/intense_pressure/antagonist2.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +stt_limit=15360 +end_limit=5120 + +# stt_limit=10240 +# end_limit=2048 + +# stt_limit=12288 +# end_limit=2048 + +nr_steps=2 +step_size=$(($(($stt_limit - $end_limit)) / $nr_steps)) + +mem_limit=$stt_limit + +sleep 300 + +for i in $( seq 1 $nr_steps ) +do + mem_limit=$(($mem_limit - $step_size)) + echo $i, $mem_limit + ./set_memratio.sh $mem_limit + sleep 300 +done diff --git a/exp/intense_pressure/set_memratio.sh b/exp/intense_pressure/set_memratio.sh new file mode 100755 index 0000000..37e6c1a --- /dev/null +++ b/exp/intense_pressure/set_memratio.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +mem_mb=$1 +mem_limit=$(($mem_mb * 1024 * 1024)) + +echo $mem_limit > ../../config/mem.config diff --git a/exp/moderate_pressure/antagonist3.sh b/exp/moderate_pressure/antagonist3.sh new file mode 100755 index 0000000..a235706 --- /dev/null +++ b/exp/moderate_pressure/antagonist3.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +stt_limit=15360 +end_limit=5120 + +# stt_limit=10240 +# end_limit=2048 + +# stt_limit=12288 +# end_limit=2048 + +nr_steps=64 +step_size=$(($(($stt_limit - $end_limit)) / $nr_steps)) + +mem_limit=$stt_limit +./set_memratio.sh $mem_limit +sleep 300 + +for i in $( seq 1 $nr_steps ) +do + echo $i, $mem_limit + ./set_memratio.sh $mem_limit + sleep 10 + mem_limit=$(($mem_limit - $step_size)) +done + +./set_memratio.sh $mem_limit +sleep 300 diff --git a/exp/moderate_pressure/set_memratio.sh b/exp/moderate_pressure/set_memratio.sh new file mode 100755 index 0000000..37e6c1a --- /dev/null +++ b/exp/moderate_pressure/set_memratio.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +mem_mb=$1 +mem_limit=$(($mem_mb * 1024 * 1024)) + +echo $mem_limit > ../../config/mem.config diff --git a/exp/softptr_cost/.gitignore b/exp/softptr_cost/.gitignore new file mode 100644 index 0000000..e390b12 --- /dev/null +++ b/exp/softptr_cost/.gitignore @@ -0,0 +1 @@ +build/** \ No newline at end of file diff --git a/exp/softptr_cost/Makefile b/exp/softptr_cost/Makefile new file mode 100644 index 0000000..65ecda7 --- /dev/null +++ b/exp/softptr_cost/Makefile @@ -0,0 +1,38 @@ +CXX = ccache g++ +LDXX = ccache g++ + +# CXX = g++ +# LDXX = g++ +CXXFLAGS += -std=c++1z -O2 +# CXXFLAGS += -g -O0 + +INC += -I../../inc +libmidas_lib := ../../lib/libmidas++.a -lrt -ldl -lpthread +LIBS += $(libmidas_lib) + +# For stacktrace logging +LIBS += -rdynamic -ggdb -no-pie -fno-pie +LIBS += -ldl + +BUILD_DIR = build + +SRC = $(wildcard *.cpp) +DEP = $(SRC:%.cpp=$(BUILD_DIR)/%.d) +TARGET = $(SRC:%.cpp=$(BUILD_DIR)/%) + +# all: softptr_read_small softptr_read_large softptr_write_small softptr_write_large \ +# unique_ptr_read_small unique_ptr_read_large unique_ptr_write_small unique_ptr_write_large + +all: $(TARGET) + +$(BUILD_DIR)/% : %.cpp Makefile + @mkdir -p $(BUILD_DIR) + $(CXX) $(CXXFLAGS) $(INC) -MMD -MP $< $(LIBS) -o $@ + +ifneq ($(MAKECMDGOALS),clean) +-include $(DEP) +endif + +.PHONY : clean +clean : + -rm -rf $(TARGET) \ No newline at end of file diff --git a/exp/softptr_cost/cached_deref_cost.cpp b/exp/softptr_cost/cached_deref_cost.cpp new file mode 100644 index 0000000..d2ec924 --- /dev/null +++ b/exp/softptr_cost/cached_deref_cost.cpp @@ -0,0 +1,85 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cache_manager.hpp" +#include "object.hpp" +#include "time.hpp" + +#define ACCESS_ONCE(x) \ + (*static_cast::type volatile *>(&(x))) + +using data_t = uint64_t; + +constexpr static int kMeasureTimes = 1'000'000; // 1M times +constexpr static uint64_t kCachePoolSize = 1ull * 1024 * 1024 * 1024; + +void unique_ptr_read_small_cost() { + auto cmanager = midas::CacheManager::global_cache_manager(); + cmanager->create_pool("small"); + auto pool = cmanager->get_pool("small"); + pool->update_limit(kCachePoolSize); + auto allocator = pool->get_allocator(); + + auto obj = std::make_unique(); + + data_t dst; + uint64_t stt, end, avg_cycles; + std::vector durs; + for (int i = 0; i < kMeasureTimes; i++) { + stt = midas::Time::get_cycles_stt(); + { + dst = ACCESS_ONCE(*obj.get()); + } + end = midas::Time::get_cycles_end(); + auto dur_cycles = end - stt; + avg_cycles += dur_cycles; + } + avg_cycles /= kMeasureTimes; + printf("Access cached unique_ptr average latency (cycles): %lu\n", + avg_cycles); +} + +void softptr_read_small_cost() { + auto cmanager = midas::CacheManager::global_cache_manager(); + cmanager->create_pool("small"); + auto pool = cmanager->get_pool("small"); + pool->update_limit(kCachePoolSize); + auto allocator = pool->get_allocator(); + + midas::ObjectPtr obj; + auto succ = allocator->alloc_to(sizeof(data_t), &obj); + assert(succ); + + data_t dst; + uint64_t stt, end, avg_cycles; + std::vector durs; + for (int i = 0; i < kMeasureTimes; i++) { + stt = midas::Time::get_cycles_stt(); + { + obj.copy_to(&dst, sizeof(dst)); + } + end = midas::Time::get_cycles_end(); + auto dur_cycles = end - stt; + avg_cycles += dur_cycles; + } + avg_cycles /= kMeasureTimes; + printf("Access cached soft_ptr average latency (cycles): %lu\n", + avg_cycles); +} + +int main(int argc, char *argv[]) { + unique_ptr_read_small_cost(); + std::this_thread::sleep_for(std::chrono::seconds(1)); + softptr_read_small_cost(); + + return 0; +} \ No newline at end of file diff --git a/exp/softptr_cost/run-large.sh b/exp/softptr_cost/run-large.sh new file mode 100755 index 0000000..0e2d59b --- /dev/null +++ b/exp/softptr_cost/run-large.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +tests=( softptr_read_large softptr_write_large unique_ptr_read_large unique_ptr_write_large ) + +objsizes=( 64 128 256 512 1024 2048 4096 8192 12288 16384 ) + +partial="true" + +rm -rf log +mkdir -p log + +for size in "${objsizes[@]}" +do + echo "Object size: $size" + for test in "${tests[@]}" + do + bytes=$(($size * 1024)) + num_eles=$((40 * 1024 * 1024 / $size)) + sed "s/constexpr static bool kPartialAccess.*/constexpr static bool kPartialAccess = $partial;/g" -i $test.cpp + sed "s/constexpr static int kNumLargeObjs.*/constexpr static int kNumLargeObjs = $num_eles;/g" -i $test.cpp + sed "s/constexpr static int kLargeObjSize.*/constexpr static int kLargeObjSize = $bytes;/g" -i $test.cpp + done + make -j + + for test in "${tests[@]}" + do + log_file="log/$test.log" + echo "Running" build/$test + ./build/$test | tee -a $log_file + sleep 1 + done +done diff --git a/exp/softptr_cost/run.sh b/exp/softptr_cost/run.sh new file mode 100755 index 0000000..63a92c2 --- /dev/null +++ b/exp/softptr_cost/run.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +tests=( softptr_read_small softptr_read_large softptr_write_small softptr_write_large unique_ptr_read_small unique_ptr_read_large unique_ptr_write_small unique_ptr_write_large ) + +for test in "${tests[@]}" +do + echo "Running" build/$test + ./build/$test + sleep 1 +done \ No newline at end of file diff --git a/exp/softptr_cost/softptr_read_large.cpp b/exp/softptr_cost/softptr_read_large.cpp new file mode 100644 index 0000000..63c7451 --- /dev/null +++ b/exp/softptr_cost/softptr_read_large.cpp @@ -0,0 +1,115 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cache_manager.hpp" +#include "object.hpp" +#include "time.hpp" + +#define ACCESS_ONCE(x) \ + (*static_cast::type volatile *>(&(x))) + +using data_t = uint64_t; + +constexpr static int kMeasureTimes = 10'000; // 10K times +constexpr static bool kPartialAccess = false; +constexpr static uint64_t kRawMemAccessCycles = 170; +constexpr static uint64_t kCachePoolSize = 100ull * 1024 * 1024 * 1024; // 100GB + +constexpr static int kNumLargeObjs = 10'000; // 10K objs +constexpr static int kLargeObjSize = 4 * 1024 * 1024; // 4MB + +struct LargeObject { + char data[kLargeObjSize]; + + LargeObject() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution dist('A', 'z'); + + constexpr static int kFillStride = 100; + for (uint32_t i = 0; i < kLargeObjSize / kFillStride; i++) { + data[i * kFillStride] = dist(mt); + } + } +}; + +void print_lats(std::vector &durs) { + std::sort(durs.begin(), durs.end()); + uint64_t avg = std::reduce(durs.begin(), durs.end(), 0.0) / durs.size(); + auto median = durs[durs.size() / 2]; + auto p90 = durs[durs.size() * 9 / 10]; + auto p99 = durs[durs.size() * 99 / 100]; + + printf("avg\t med\t p90\t p99\n" + "%lu\t %lu\t %lu\t %lu\n", + avg, median, p90, p99); +} + +void softptr_read_large_cost() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution idx_dist(0, kNumLargeObjs - 1); + static std::uniform_int_distribution off_dist( + 0, kLargeObjSize / sizeof(data_t) - 1); + + auto cmanager = midas::CacheManager::global_cache_manager(); + cmanager->create_pool("large"); + auto pool = cmanager->get_pool("large"); + pool->update_limit(kCachePoolSize); + auto allocator = pool->get_allocator(); + + std::vector objs; + for (int i = 0; i < kNumLargeObjs; i++) { + objs.emplace_back(midas::ObjectPtr()); + } + for (int i = 0; i < kNumLargeObjs; i++) { + auto succ = allocator->alloc_to(kLargeObjSize, &objs[i]); + assert(succ); + } + for (int i = 0; i < kNumLargeObjs; i++) { + auto obj = std::make_unique(); + auto succ = objs[i].copy_from(obj->data, kLargeObjSize); + assert(succ); + } + + auto obj = std::make_unique(); + auto scratch = std::make_unique(); + uint64_t stt, end; + std::vector durs; + for (int i = 0; i < kMeasureTimes; i++) { + auto idx = idx_dist(mt); + auto off = off_dist(mt); + + volatile data_t dst; + stt = midas::Time::get_cycles_stt(); + if constexpr (kPartialAccess) { + data_t dst_; + objs[idx].copy_to(&dst_, sizeof(dst_), off); + dst = dst_; + } else { + objs[idx].copy_to(scratch->data, kLargeObjSize); + std::memcpy(obj->data, scratch->data, kLargeObjSize); + } + end = midas::Time::get_cycles_end(); + auto dur_cycles = end - stt; + if (dur_cycles > kRawMemAccessCycles) + durs.push_back(dur_cycles); + } + printf("Access soft_ptr latency distribution (cycles):\n"); + print_lats(durs); +} + +int main(int argc, char *argv[]) { + softptr_read_large_cost(); + + return 0; +} \ No newline at end of file diff --git a/exp/softptr_cost/softptr_read_small.cpp b/exp/softptr_cost/softptr_read_small.cpp new file mode 100644 index 0000000..35668fc --- /dev/null +++ b/exp/softptr_cost/softptr_read_small.cpp @@ -0,0 +1,106 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cache_manager.hpp" +#include "object.hpp" +#include "time.hpp" + +#define ACCESS_ONCE(x) \ + (*static_cast::type volatile *>(&(x))) + +using data_t = uint64_t; + +constexpr static int kMeasureTimes = 1'000'000; // 1M times +constexpr static uint64_t kRawMemAccessCycles = 170; +constexpr static uint64_t kCachePoolSize = 100ull * 1024 * 1024 * 1024; + +constexpr static int kNumSmallObjs = 1'000'000; // 1M objs +constexpr static int kSmallObjSize = 32; +constexpr static int kAlignedSize = + 64 - sizeof(midas::SmallObjectHdr); // cacheline aligned with header +static_assert(kSmallObjSize <= kAlignedSize, "AlignedSize is incorrect!"); + +struct SmallObject { + char data[kAlignedSize]; + + SmallObject() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution dist('A', 'z'); + + for (uint32_t i = 0; i < kAlignedSize; i++) { + data[i] = dist(mt); + } + } +}; + +void print_lats(std::vector &durs) { + std::sort(durs.begin(), durs.end()); + uint64_t avg = std::reduce(durs.begin(), durs.end(), 0.0) / durs.size(); + auto median = durs[durs.size() / 2]; + auto p90 = durs[durs.size() * 9 / 10]; + auto p99 = durs[durs.size() * 99 / 100]; + + printf("avg\t med\t p90\t p99\n" + "%lu\t %lu\t %lu\t %lu\n", + avg, median, p90, p99); +} + +void softptr_read_small_cost() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution dist(0, kNumSmallObjs - 1); + + auto cmanager = midas::CacheManager::global_cache_manager(); + cmanager->create_pool("small"); + auto pool = cmanager->get_pool("small"); + pool->update_limit(kCachePoolSize); + auto allocator = pool->get_allocator(); + + std::vector objs; + for (int i = 0; i < kNumSmallObjs; i++) { + objs.emplace_back(midas::ObjectPtr()); + } + for (int i = 0; i < kNumSmallObjs; i++) { + auto succ = allocator->alloc_to(kAlignedSize, &objs[i]); + assert(succ); + } + for (int i = 0; i < kNumSmallObjs; i++) { + SmallObject obj; + auto succ = objs[i].copy_from(obj.data, kSmallObjSize); + assert(succ); + } + + SmallObject obj; + uint64_t stt, end; + std::vector durs; + for (int i = 0; i < kMeasureTimes; i++) { + auto idx = dist(mt); + + stt = midas::Time::get_cycles_stt(); + { + objs[idx].copy_to(obj.data, kSmallObjSize); + } + end = midas::Time::get_cycles_end(); + auto dur_cycles = end - stt; + if (dur_cycles > kRawMemAccessCycles) + durs.push_back(dur_cycles); + } + printf("Access soft_ptr latency distribution (cycles):\n"); + print_lats(durs); +} + +int main(int argc, char *argv[]) { + softptr_read_small_cost(); + + return 0; +} \ No newline at end of file diff --git a/exp/softptr_cost/softptr_write_large.cpp b/exp/softptr_cost/softptr_write_large.cpp new file mode 100644 index 0000000..9f2a75a --- /dev/null +++ b/exp/softptr_cost/softptr_write_large.cpp @@ -0,0 +1,111 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cache_manager.hpp" +#include "object.hpp" +#include "time.hpp" + +#define ACCESS_ONCE(x) \ + (*static_cast::type volatile *>(&(x))) + +using data_t = uint64_t; +constexpr static data_t kWriteContent = 0x1f1f1f1f'1f1f1f1full; + +constexpr static int kMeasureTimes = 10'000; // 10K times +constexpr static bool kPartialAccess = false; +constexpr static uint64_t kRawMemAccessCycles = 170; +constexpr static uint64_t kCachePoolSize = 100ull * 1024 * 1024 * 1024; // 100GB + +constexpr static int kNumLargeObjs = 10'000; // 10K objs +constexpr static int kLargeObjSize = 4 * 1024 * 1024; // 4MB + +struct LargeObject { + char data[kLargeObjSize]; + + LargeObject() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution dist('A', 'z'); + + constexpr static int kFillStride = 100; + for (uint32_t i = 0; i < kLargeObjSize / kFillStride; i++) { + data[i * kFillStride] = dist(mt); + } + } +}; + +void print_lats(std::vector &durs) { + std::sort(durs.begin(), durs.end()); + uint64_t avg = std::reduce(durs.begin(), durs.end(), 0.0) / durs.size(); + auto median = durs[durs.size() / 2]; + auto p90 = durs[durs.size() * 9 / 10]; + auto p99 = durs[durs.size() * 99 / 100]; + + printf("avg\t med\t p90\t p99\n" + "%lu\t %lu\t %lu\t %lu\n", + avg, median, p90, p99); +} + +void softptr_write_large_cost() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution idx_dist(0, kNumLargeObjs - 1); + static std::uniform_int_distribution off_dist( + 0, kLargeObjSize / sizeof(data_t) - 1); + + auto cmanager = midas::CacheManager::global_cache_manager(); + cmanager->create_pool("large"); + auto pool = cmanager->get_pool("large"); + pool->update_limit(kCachePoolSize); + auto allocator = pool->get_allocator(); + + std::vector objs; + for (int i = 0; i < kNumLargeObjs; i++) { + objs.emplace_back(midas::ObjectPtr()); + } + for (int i = 0; i < kNumLargeObjs; i++) { + auto succ = allocator->alloc_to(kLargeObjSize, &objs[i]); + assert(succ); + } + for (int i = 0; i < kNumLargeObjs; i++) { + auto obj = std::make_unique(); + auto succ = objs[i].copy_from(obj->data, kLargeObjSize); + assert(succ); + } + + auto obj = std::make_unique(); + uint64_t stt, end; + std::vector durs; + for (int i = 0; i < kMeasureTimes; i++) { + auto idx = idx_dist(mt); + auto off = off_dist(mt); + + stt = midas::Time::get_cycles_stt(); + if constexpr (kPartialAccess) { + objs[idx].copy_from(&kWriteContent, sizeof(data_t), off); + } else { + objs[idx].copy_from(obj->data, kLargeObjSize); + } + end = midas::Time::get_cycles_end(); + auto dur_cycles = end - stt; + if (dur_cycles > kRawMemAccessCycles) + durs.push_back(dur_cycles); + } + printf("Access soft_ptr latency distribution (cycles):\n"); + print_lats(durs); +} + +int main(int argc, char *argv[]) { + softptr_write_large_cost(); + + return 0; +} \ No newline at end of file diff --git a/exp/softptr_cost/softptr_write_small.cpp b/exp/softptr_cost/softptr_write_small.cpp new file mode 100644 index 0000000..fe40798 --- /dev/null +++ b/exp/softptr_cost/softptr_write_small.cpp @@ -0,0 +1,107 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cache_manager.hpp" +#include "object.hpp" +#include "time.hpp" + +#define ACCESS_ONCE(x) \ + (*static_cast::type volatile *>(&(x))) + +using data_t = uint64_t; +constexpr static data_t kWriteContent = 0x1f1f1f1f'1f1f1f1full; + +constexpr static int kMeasureTimes = 1'000'000; // 1M times +constexpr static uint64_t kRawMemAccessCycles = 170; +constexpr static uint64_t kCachePoolSize = 100ull * 1024 * 1024 * 1024; + +constexpr static int kNumSmallObjs = 1'000'000; // 1M objs +constexpr static int kSmallObjSize = 32; +constexpr static int kAlignedSize = + 64 - sizeof(midas::SmallObjectHdr); // cacheline aligned with header +static_assert(kSmallObjSize <= kAlignedSize, "AlignedSize is incorrect!"); + +struct SmallObject { + char data[kAlignedSize]; + + SmallObject() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution dist('A', 'z'); + + for (uint32_t i = 0; i < kAlignedSize; i++) { + data[i] = dist(mt); + } + } +}; + +void print_lats(std::vector &durs) { + std::sort(durs.begin(), durs.end()); + uint64_t avg = std::reduce(durs.begin(), durs.end(), 0.0) / durs.size(); + auto median = durs[durs.size() / 2]; + auto p90 = durs[durs.size() * 9 / 10]; + auto p99 = durs[durs.size() * 99 / 100]; + + printf("avg\t med\t p90\t p99\n" + "%lu\t %lu\t %lu\t %lu\n", + avg, median, p90, p99); +} + +void softptr_write_small_cost() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution dist(0, kNumSmallObjs - 1); + + auto cmanager = midas::CacheManager::global_cache_manager(); + cmanager->create_pool("small"); + auto pool = cmanager->get_pool("small"); + pool->update_limit(kCachePoolSize); + auto allocator = pool->get_allocator(); + + std::vector objs; + for (int i = 0; i < kNumSmallObjs; i++) { + objs.emplace_back(midas::ObjectPtr()); + } + for (int i = 0; i < kNumSmallObjs; i++) { + auto succ = allocator->alloc_to(kAlignedSize, &objs[i]); + assert(succ); + } + for (int i = 0; i < kNumSmallObjs; i++) { + SmallObject obj; + auto succ = objs[i].copy_from(obj.data, kSmallObjSize); + assert(succ); + } + + SmallObject obj; + uint64_t stt, end; + std::vector durs; + for (int i = 0; i < kMeasureTimes; i++) { + auto idx = dist(mt); + + stt = midas::Time::get_cycles_stt(); + { + objs[idx].copy_from(obj.data, kSmallObjSize); + } + end = midas::Time::get_cycles_end(); + auto dur_cycles = end - stt; + if (dur_cycles > kRawMemAccessCycles) + durs.push_back(dur_cycles); + } + printf("Access soft_ptr latency distribution (cycles):\n"); + print_lats(durs); +} + +int main(int argc, char *argv[]) { + softptr_write_small_cost(); + + return 0; +} \ No newline at end of file diff --git a/exp/softptr_cost/unique_ptr_read_large.cpp b/exp/softptr_cost/unique_ptr_read_large.cpp new file mode 100644 index 0000000..c5a4121 --- /dev/null +++ b/exp/softptr_cost/unique_ptr_read_large.cpp @@ -0,0 +1,95 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cache_manager.hpp" +#include "object.hpp" +#include "time.hpp" + +#define ACCESS_ONCE(x) \ + (*static_cast::type volatile *>(&(x))) + +using data_t = uint64_t; + +constexpr static int kMeasureTimes = 10'000; // 10K times +constexpr static bool kPartialAccess = false; +constexpr static uint64_t kRawMemAccessCycles = 170; +constexpr static uint64_t kCachePoolSize = 100ull * 1024 * 1024 * 1024; // 100GB + +constexpr static int kNumLargeObjs = 10'000; // 10K objs +constexpr static int kLargeObjSize = 4 * 1024 * 1024; // 4MB + +struct LargeObject { + char data[kLargeObjSize]; + + LargeObject() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution dist('A', 'z'); + + constexpr static int kFillStride = 100; + for (uint32_t i = 0; i < kLargeObjSize / kFillStride; i++) { + data[i * kFillStride] = dist(mt); + } + } +}; + +void print_lats(std::vector &durs) { + std::sort(durs.begin(), durs.end()); + uint64_t avg = std::reduce(durs.begin(), durs.end(), 0.0) / durs.size(); + auto median = durs[durs.size() / 2]; + auto p90 = durs[durs.size() * 9 / 10]; + auto p99 = durs[durs.size() * 99 / 100]; + + printf("avg\t med\t p90\t p99\n" + "%lu\t %lu\t %lu\t %lu\n", + avg, median, p90, p99); +} + +void unique_ptr_read_large_cost() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution idx_dist(0, kNumLargeObjs - 1); + static std::uniform_int_distribution off_dist( + 0, kLargeObjSize / sizeof(data_t) - 1); + + std::vector> objs; + for (int i = 0; i < kNumLargeObjs; i++) { + objs.push_back(std::make_unique()); + } + + auto obj = std::make_unique(); + uint64_t stt, end; + std::vector durs; + for (int i = 0; i < kMeasureTimes; i++) { + auto idx = idx_dist(mt); + auto off = off_dist(mt); + + stt = midas::Time::get_cycles_stt(); + if constexpr (kPartialAccess) { + const data_t *ptr = reinterpret_cast(objs[idx].get()->data); + ACCESS_ONCE(ptr[off]); + } else { + std::memcpy(obj->data, objs[idx].get(), kLargeObjSize); + } + end = midas::Time::get_cycles_end(); + auto dur_cycles = end - stt; + durs.push_back(dur_cycles); + } + printf("Access unique_ptr latency distribution (cycles):\n"); + print_lats(durs); +} + +int main(int argc, char *argv[]) { + unique_ptr_read_large_cost(); + + return 0; +} \ No newline at end of file diff --git a/exp/softptr_cost/unique_ptr_read_small.cpp b/exp/softptr_cost/unique_ptr_read_small.cpp new file mode 100644 index 0000000..4431404 --- /dev/null +++ b/exp/softptr_cost/unique_ptr_read_small.cpp @@ -0,0 +1,88 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cache_manager.hpp" +#include "object.hpp" +#include "time.hpp" + +#define ACCESS_ONCE(x) \ + (*static_cast::type volatile *>(&(x))) + +using data_t = uint64_t; + +constexpr static int kMeasureTimes = 1'000'000; // 1M times +constexpr static uint64_t kRawMemAccessCycles = 170; +constexpr static uint64_t kCachePoolSize = 100ull * 1024 * 1024 * 1024; + +constexpr static int kNumSmallObjs = 1'000'000; // 1M objs +constexpr static int kSmallObjSize = 32; + +struct SmallObject { + char data[kSmallObjSize]; + + SmallObject() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution dist('A', 'z'); + + for (uint32_t i = 0; i < kSmallObjSize; i++) { + data[i] = dist(mt); + } + } +}; + +void print_lats(std::vector &durs) { + std::sort(durs.begin(), durs.end()); + uint64_t avg = std::reduce(durs.begin(), durs.end(), 0.0) / durs.size(); + auto median = durs[durs.size() / 2]; + auto p90 = durs[durs.size() * 9 / 10]; + auto p99 = durs[durs.size() * 99 / 100]; + + printf("avg\t med\t p90\t p99\n" + "%lu\t %lu\t %lu\t %lu\n", + avg, median, p90, p99); +} + +void unique_ptr_read_small_cost() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution dist(0, kNumSmallObjs - 1); + + std::vector> objs; + for (int i = 0; i < kNumSmallObjs; i++) { + objs.push_back(std::make_unique()); + } + + SmallObject obj; + uint64_t stt, end; + std::vector durs; + for (int i = 0; i < kMeasureTimes; i++) { + auto idx = dist(mt); + + stt = midas::Time::get_cycles_stt(); + { + obj = *objs[idx].get(); + } + end = midas::Time::get_cycles_end(); + auto dur_cycles = end - stt; + if (dur_cycles > kRawMemAccessCycles) + durs.push_back(dur_cycles); + } + printf("Access unique_ptr latency distribution (cycles):\n"); + print_lats(durs); +} + +int main(int argc, char *argv[]) { + unique_ptr_read_small_cost(); + + return 0; +} \ No newline at end of file diff --git a/exp/softptr_cost/unique_ptr_write_large.cpp b/exp/softptr_cost/unique_ptr_write_large.cpp new file mode 100644 index 0000000..2c239bc --- /dev/null +++ b/exp/softptr_cost/unique_ptr_write_large.cpp @@ -0,0 +1,96 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cache_manager.hpp" +#include "object.hpp" +#include "time.hpp" + +#define ACCESS_ONCE(x) \ + (*static_cast::type volatile *>(&(x))) + +using data_t = uint64_t; +constexpr static data_t kWriteContent = 0x1f1f1f1f'1f1f1f1full; + +constexpr static int kMeasureTimes = 10'000; // 10K times +constexpr static bool kPartialAccess = false; +constexpr static uint64_t kRawMemAccessCycles = 170; +constexpr static uint64_t kCachePoolSize = 100ull * 1024 * 1024 * 1024; // 100GB + +constexpr static int kNumLargeObjs = 10'000; // 10K objs +constexpr static int kLargeObjSize = 4 * 1024 * 1024; // 4MB + +struct LargeObject { + char data[kLargeObjSize]; + + LargeObject() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution dist('A', 'z'); + + constexpr static int kFillStride = 100; + for (uint32_t i = 0; i < kLargeObjSize / kFillStride; i++) { + data[i * kFillStride] = dist(mt); + } + } +}; + +void print_lats(std::vector &durs) { + std::sort(durs.begin(), durs.end()); + uint64_t avg = std::reduce(durs.begin(), durs.end(), 0.0) / durs.size(); + auto median = durs[durs.size() / 2]; + auto p90 = durs[durs.size() * 9 / 10]; + auto p99 = durs[durs.size() * 99 / 100]; + + printf("avg\t med\t p90\t p99\n" + "%lu\t %lu\t %lu\t %lu\n", + avg, median, p90, p99); +} + +void unique_ptr_write_large_cost() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution idx_dist(0, kNumLargeObjs - 1); + static std::uniform_int_distribution off_dist( + 0, kLargeObjSize / sizeof(data_t) - 1); + + std::vector> objs; + for (int i = 0; i < kNumLargeObjs; i++) { + objs.push_back(std::make_unique()); + } + + auto obj = std::make_unique(); + uint64_t stt, end; + std::vector durs; + for (int i = 0; i < kMeasureTimes; i++) { + auto idx = idx_dist(mt); + auto off = off_dist(mt); + + stt = midas::Time::get_cycles_stt(); + if constexpr (kPartialAccess) { + data_t *ptr = reinterpret_cast(objs[idx].get()->data); + ACCESS_ONCE(ptr[off]) = kWriteContent; + } else { + std::memcpy(objs[idx].get(), obj->data, kLargeObjSize); + } + end = midas::Time::get_cycles_end(); + auto dur_cycles = end - stt; + durs.push_back(dur_cycles); + } + printf("Access unique_ptr latency distribution (cycles):\n"); + print_lats(durs); +} + +int main(int argc, char *argv[]) { + unique_ptr_write_large_cost(); + + return 0; +} \ No newline at end of file diff --git a/exp/softptr_cost/unique_ptr_write_small.cpp b/exp/softptr_cost/unique_ptr_write_small.cpp new file mode 100644 index 0000000..079412a --- /dev/null +++ b/exp/softptr_cost/unique_ptr_write_small.cpp @@ -0,0 +1,90 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cache_manager.hpp" +#include "object.hpp" +#include "time.hpp" + +#define ACCESS_ONCE(x) \ + (*static_cast::type volatile *>(&(x))) + +using data_t = uint64_t; +constexpr static data_t kWriteContent = 0x1f1f1f1f'1f1f1f1full; + +constexpr static int kMeasureTimes = 1'000'000; // 1M times +constexpr static uint64_t kRawMemAccessCycles = 170; +constexpr static uint64_t kCachePoolSize = 100ull * 1024 * 1024 * 1024; + +constexpr static int kNumSmallObjs = 1'000'000; // 1M objs +constexpr static int kSmallObjSize = 32; + +struct SmallObject { + char data[kSmallObjSize]; + + SmallObject() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution dist('A', 'z'); + + for (uint32_t i = 0; i < kSmallObjSize; i++) { + data[i] = dist(mt); + } + } +}; + +void print_lats(std::vector &durs) { + std::sort(durs.begin(), durs.end()); + uint64_t avg = std::reduce(durs.begin(), durs.end(), 0.0) / durs.size(); + auto median = durs[durs.size() / 2]; + auto p90 = durs[durs.size() * 9 / 10]; + auto p99 = durs[durs.size() * 99 / 100]; + + printf("avg\t med\t p90\t p99\n" + "%lu\t %lu\t %lu\t %lu\n", + avg, median, p90, p99); +} + +void unique_ptr_write_small_cost() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution dist(0, kNumSmallObjs - 1); + + std::vector> objs; + for (int i = 0; i < kNumSmallObjs; i++) { + objs.push_back(std::make_unique()); + } + + SmallObject obj; + uint64_t stt, end; + std::vector durs; + for (int i = 0; i < kMeasureTimes; i++) { + auto idx = dist(mt); + + stt = midas::Time::get_cycles_stt(); + { + /* *objs[idx].get() = obj; */ + std::memcpy(objs[idx].get(), &obj, sizeof(obj)); + } + end = midas::Time::get_cycles_end(); + auto dur_cycles = end - stt; + if (dur_cycles > kRawMemAccessCycles) + durs.push_back(dur_cycles); + } + printf("Access unique_ptr latency distribution (cycles):\n"); + print_lats(durs); +} + +int main(int argc, char *argv[]) { + unique_ptr_write_small_cost(); + + return 0; +} diff --git a/inc/array.hpp b/inc/array.hpp new file mode 100644 index 0000000..01337c5 --- /dev/null +++ b/inc/array.hpp @@ -0,0 +1,25 @@ +#pragma once + +#include +#include + +#include "cache_manager.hpp" +#include "object.hpp" + +namespace midas { +template class Array { +public: + Array(int n); + Array(CachePool *pool, int n); + ~Array(); + std::unique_ptr get(int idx); + bool set(int idx, const T &t); + +private: + CachePool *pool_; + ObjectPtr *data_; + int len_; +}; +} // namespace midas + +#include "impl/array.ipp" \ No newline at end of file diff --git a/inc/cache_manager.hpp b/inc/cache_manager.hpp new file mode 100644 index 0000000..32732ab --- /dev/null +++ b/inc/cache_manager.hpp @@ -0,0 +1,138 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "evacuator.hpp" +#include "log.hpp" +#include "object.hpp" +#include "resource_manager.hpp" +#include "shm_types.hpp" +#include "time.hpp" +#include "victim_cache.hpp" + +namespace midas { + +struct ConstructPlug { + uint64_t stt_cycles{0}; + uint64_t end_cycles{0}; + int64_t bytes{0}; + + void reset() { + stt_cycles = 0; + end_cycles = 0; + bytes = 0; + } +}; + +class CachePool { +public: + CachePool(std::string name); + ~CachePool(); + + // Config + void update_limit(size_t limit_in_bytes); + void set_weight(float weight); + void set_lat_critical(bool value); + + // Callback Functions + using ConstructFunc = std::function; + void set_construct_func(ConstructFunc callback); + ConstructFunc get_construct_func() const noexcept; + int construct(void *arg); + using PreevictFunc = std::function; + int preevict(ObjectPtr *optr); + PreevictFunc get_preevict_func() const noexcept; + using DestructFunc = std::function; + int destruct(ObjectPtr *optr); + DestructFunc get_destruct_func() const noexcept; + + // Construct helper functions for recording penalty + void construct_stt(ConstructPlug &plug) noexcept; // mark stt/end of + void construct_end(ConstructPlug &plug) noexcept; // the construct path + void construct_add(uint64_t bytes, + ConstructPlug &plug) noexcept; // add up missed bytes + + // Allocator shortcuts + inline std::optional alloc(size_t size); + inline bool alloc_to(size_t size, ObjectPtr *dst); + inline bool free(ObjectPtr &ptr); + + // Profiling + void inc_cache_hit() noexcept; + void inc_cache_miss() noexcept; + void inc_cache_victim_hit(ObjectPtr *optr_addr = nullptr) noexcept; // TODO: check all callsites + void record_miss_penalty(uint64_t cycles, uint64_t bytes) noexcept; + void profile_stats(StatsMsg *msg = nullptr) noexcept; + + inline VictimCache *get_vcache() const noexcept; + inline ResourceManager *get_rmanager() const noexcept; + inline LogAllocator *get_allocator() const noexcept; + inline Evacuator *get_evacuator() const noexcept; + + static inline CachePool *global_cache_pool(); + +private: + std::string name_; + ConstructFunc construct_; + PreevictFunc preevict_; + DestructFunc destruct_; + + // Stats & Counters + struct CacheStats { + std::atomic_uint_fast64_t hits{0}; + std::atomic_uint_fast64_t misses{0}; + std::atomic_uint_fast64_t miss_cycles{0}; + std::atomic_uint_fast64_t miss_bytes{0}; + std::atomic_uint_fast64_t victim_hits{0}; + uint64_t timestamp{0}; + + void reset() noexcept; + } stats; + + std::unique_ptr vcache_; + std::shared_ptr rmanager_; + std::shared_ptr allocator_; + std::unique_ptr evacuator_; + + friend class CacheManager; + friend class ResourceManager; + + static constexpr uint64_t kVCacheSizeLimit = 64 * 1024 * 1024; // 64 MB + static constexpr uint64_t kVCacheCountLimit = 500000; +}; + +class CacheManager { +public: + CacheManager(); + ~CacheManager(); + + bool create_pool(std::string name); + bool delete_pool(std::string name); + CachePool *get_pool(std::string name); + + size_t num_pools() const noexcept; + + static CacheManager *global_cache_manager(); + + constexpr static char default_pool_name[] = "default"; + +private: + bool terminated_; + StatsMsg profile_pools(); + std::unique_ptr profiler_; + + std::mutex mtx_; + std::unordered_map> pools_; + + friend class ResourceManager; +}; + +} // namespace midas + +#include "impl/cache_manager.ipp" \ No newline at end of file diff --git a/inc/construct_args.hpp b/inc/construct_args.hpp new file mode 100644 index 0000000..c39c7c0 --- /dev/null +++ b/inc/construct_args.hpp @@ -0,0 +1,12 @@ +#pragma once + +#include + +namespace midas { +struct ConstructArgs { + const void *key; + const uint64_t key_len; + void *value; + uint64_t value_len; +}; +} // namespace midas \ No newline at end of file diff --git a/inc/evacuator.hpp b/inc/evacuator.hpp new file mode 100644 index 0000000..9b44dcc --- /dev/null +++ b/inc/evacuator.hpp @@ -0,0 +1,64 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "object.hpp" + +namespace midas { + +enum class EvacState { Succ, Fail, Fault, DelayRelease }; + +class LogSegment; // defined in log.hpp +class SegmentList; // defined in log.hpp +class LogAllocator; // defined in log.hpp + +class CachePool; // defined in cache_manager.hpp + +class ResourceManager; // defined in resource_manager.hpp + +class Evacuator { +public: + Evacuator(CachePool *pool, std::shared_ptr rmanager, + std::shared_ptr allocator); + ~Evacuator(); + void signal_gc(); + bool serial_gc(); + bool parallel_gc(int nr_workers); + int64_t force_reclaim(); + +private: + void init(); + + int64_t gc(SegmentList &stash_list); + + /** Segment opeartions */ + EvacState scan_segment(LogSegment *segment, bool deactivate); + EvacState evac_segment(LogSegment *segment); + EvacState free_segment(LogSegment *segment); + + /** Helper funcs */ + bool segment_ready(LogSegment *segment); + using RetCode = ObjectPtr::RetCode; + RetCode iterate_segment(LogSegment *segment, uint64_t &pos, ObjectPtr &optr); + + CachePool *pool_; + std::shared_ptr allocator_; + std::shared_ptr rmanager_; + + bool terminated_; + + std::shared_ptr gc_thd_; + std::condition_variable gc_cv_; + std::mutex gc_mtx_; + +}; + +} // namespace midas + +#include "impl/evacuator.ipp" \ No newline at end of file diff --git a/inc/fs_shim.hpp b/inc/fs_shim.hpp new file mode 100644 index 0000000..11dc250 --- /dev/null +++ b/inc/fs_shim.hpp @@ -0,0 +1,124 @@ +#pragma once + +// #define HIJACK_FS_SYSCALLS +#ifdef HIJACK_FS_SYSCALLS + +extern "C" { +#include +#include +#include + +/** NOTE: commented out below to avoid declaration conflicts with fcntl.h + * This is okay as long as fcntl.h is included, which is done by our + * shared-memory-based IPC implementation. + */ +/** File open/close related syscalls */ +// int open(const char *pathname, int flags, mode_t mode); +// int open64(const char *pathname, int flags, mode_t mode); +// int creat(const char *pathname, int flags, mode_t mode); +// int creat64(const char *pathname, int flags, mode_t mode); +// int openat(int dirfd, const char *pathname, int flags, mode_t mode); +// int openat64(int dirfd, const char *pathname, int flags, mode_t mode); +// int __openat_2(int dirfd, const char *pathname, int flags, mode_t mode) +// __attribute__((alias("openat"))); +// int dup(int oldfd); +// int dup2(int oldfd, int newfd); +// int close(int fd); +// FILE *fopen(const char *path, const char *mode); +// FILE *fopen64(const char *path, const char *mode); +// int fclose(FILE *fp); +// /** File read/write related syscalls */ +// ssize_t read(int fd, void *buf, size_t count); +// ssize_t write(int fd, const void *buf, size_t count); +// ssize_t pread(int fd, void *buf, size_t count, off_t offset); +// ssize_t pwrite(int fd, const void *buf, size_t count, off_t offset); +// size_t fread(void *ptr, size_t size, size_t nmemb, FILE *stream); +// size_t fwrite(const void *ptr, size_t size, size_t nmemb, FILE *stream); +// off_t lseek(int fd, off_t offset, int whence); +} + +#include +#include +#include +#include + +#include "logging.hpp" +#include "sync_hashmap.hpp" +#include "utils.hpp" + +namespace midas { +using pfn_t = int64_t; // page frame number +using Page = char[kPageSize]; +constexpr static int32_t kNumPageCacheBuckets = 1 << 4; +using PageMap = SyncHashMap; + +struct File { + File() = default; + File(CachePool *pool); + + off_t offset; + std::shared_ptr cache; +}; + +class FSShim { +public: + FSShim(); + + bool on_open(int fd); + bool on_close(int fd); + ssize_t on_read(int fd, void *buf, size_t count, bool upd_offset, + off_t offset = 0); + ssize_t on_write(int fd, const void *buf, size_t count, bool upd_offset, + off_t offset = 0); + bool on_lseek(int fd, off_t offset); + + // As we use shm files for IPC now, we need to exclude them from interposition + void exclude_interpose(int fd); + void reset_interpose(int fd); + bool is_excluded(int fd); + + static inline FSShim *global_shim(); + + // origial syscalls + int (*open)(const char *pathname, int flags, mode_t mode); + int (*open64)(const char *pathname, int flags, mode_t mode); + int (*creat)(const char *pathname, int flags, mode_t mode); + int (*creat64)(const char *pathname, int flags, mode_t mode); + int (*openat)(int dirfd, const char *pathname, int flags, mode_t mode); + int (*openat64)(int dirfd, const char *pathname, int flags, mode_t mode); + int (*dup)(int fd); + int (*dup2)(int newfd, int oldfd); + int (*close)(int fd); + FILE *(*fopen)(const char *path, const char *mode); + FILE *(*fopen64)(const char *path, const char *mode); + int (*fclose)(FILE *fp); + + ssize_t (*read)(int fd, void *buf, size_t count); + ssize_t (*write)(int fd, const void *buf, size_t count); + ssize_t (*pread)(int fd, void *buf, size_t count, off_t offset); + ssize_t (*pwrite)(int fd, const void *buf, size_t count, off_t offset); + size_t (*fread)(void *ptr, size_t size, size_t nmemb, FILE *stream); + size_t (*fwrite)(const void *ptr, size_t size, size_t nmemb, FILE *stream); + off_t (*lseek)(int fd, off_t offset, int whence); + +private: + void capture_syscalls(); + void init_cache(); + + std::mutex mtx_; + std::unordered_map files_; // file's inode number to its cache + std::mutex bl_mtx_; + std::set black_list_; // we need to exclude shm files from interposition + CachePool *pool_; + constexpr static char pool_name[] = "page-cache"; + constexpr static int64_t kPageCacheLimit = 100ul * 1024 * 1024; // 100 MB +}; + +static inline ino_t get_inode(int fd); +static inline bool invalid_inode(ino_t inode); +static inline bool is_shm_file(const char *pathname); +} // namespace midas + +#include "impl/fs_shim.ipp" + +#endif // HIJACK_FS_SYSCALLS \ No newline at end of file diff --git a/inc/impl/array.ipp b/inc/impl/array.ipp new file mode 100644 index 0000000..1af5030 --- /dev/null +++ b/inc/impl/array.ipp @@ -0,0 +1,64 @@ +#pragma once + +namespace midas { +template Array::Array(int n) : len_(n) { + pool_ = CachePool::global_cache_pool(); + data_ = new ObjectPtr[len_]; + assert(data_); +} + +template +Array::Array(CachePool *pool, int n) : pool_(pool), len_(n) { + data_ = new ObjectPtr[len_]; + assert(data_); +} + +template Array::~Array() { + if (data_) { + for (int i = 0; i < len_; i++) { + if (!data_[i].null()) + pool_->free(data_[i]); + } + delete[] data_; + data_ = nullptr; + } +} + +template std::unique_ptr Array::get(int idx) { + if (idx >= len_) + return nullptr; + T *t = nullptr; + auto &optr = data_[idx]; + if (optr.null()) + goto missed; + + t = reinterpret_cast(::operator new(sizeof(T))); + if (!optr.copy_to(t, sizeof(T))) + goto faulted; + pool_->inc_cache_hit(); + pool_->get_allocator()->count_access(); + return std::unique_ptr(t); + +faulted: + if (t) + ::operator delete(t); +missed: + if (optr.is_victim()) + pool_->inc_cache_victim_hit(&optr); + pool_->inc_cache_miss(); + return nullptr; +} + +template bool Array::set(int idx, const T &t) { + if (idx >= len_) + return false; + auto &optr = data_[idx]; + auto allocator = pool_->get_allocator(); + if (!optr.null()) + pool_->free(optr); + if (!allocator->alloc_to(sizeof(T), &optr) || !optr.copy_from(&t, sizeof(T))) + return false; + pool_->get_allocator()->count_access(); + return true; +} +} // namespace midas \ No newline at end of file diff --git a/inc/impl/cache_manager.ipp b/inc/impl/cache_manager.ipp new file mode 100644 index 0000000..5b892af --- /dev/null +++ b/inc/impl/cache_manager.ipp @@ -0,0 +1,155 @@ +#pragma once + +namespace midas { +inline CachePool::CachePool(std::string name) + : name_(name), construct_(nullptr) { + vcache_ = std::make_unique(kVCacheSizeLimit, kVCacheCountLimit); + allocator_ = std::make_shared(this); + rmanager_ = std::make_shared(this); + evacuator_ = std::make_unique(this, rmanager_, allocator_); +} + +inline CachePool::~CachePool() {} + +inline CachePool *CachePool::global_cache_pool() { + static std::mutex mtx_; + static CachePool *pool_ = nullptr; + if (pool_) + return pool_; + std::unique_lock ul(mtx_); + if (pool_) + return pool_; + ul.unlock(); + auto cache_mgr = CacheManager::global_cache_manager(); + if (!cache_mgr) + return nullptr; + auto pool = cache_mgr->get_pool(CacheManager::default_pool_name); + if (pool) + return pool; + else if (!cache_mgr->create_pool(CacheManager::default_pool_name)) + return nullptr; + ul.lock(); + pool_ = cache_mgr->get_pool(CacheManager::default_pool_name); + return pool_; +} + +inline void CachePool::update_limit(size_t limit_in_bytes) { + rmanager_->UpdateLimit(limit_in_bytes); +} + +inline void CachePool::set_weight(float weight) { + rmanager_->SetWeight(weight); +} + +inline void CachePool::set_lat_critical(bool value) { + rmanager_->SetLatCritical(value); +} + +inline void CachePool::set_construct_func(ConstructFunc callback) { + if (construct_) + MIDAS_LOG(kWarning) << "Cache pool " << name_ + << " has already set its construct callback"; + else + construct_ = callback; +} + +inline CachePool::ConstructFunc CachePool::get_construct_func() const noexcept { + return construct_; +} + +inline int CachePool::construct(void *arg) { return construct_(arg); }; + +inline std::optional CachePool::alloc(size_t size) { + return allocator_->alloc(size); +} + +inline void CachePool::construct_stt(ConstructPlug &plug) noexcept { + plug.reset(); + plug.stt_cycles = Time::get_cycles_stt(); +} + +inline void CachePool::construct_end(ConstructPlug &plug) noexcept { + plug.end_cycles = Time::get_cycles_end(); + auto cycles = plug.end_cycles - plug.stt_cycles; + if (plug.bytes && cycles) + record_miss_penalty(cycles, plug.bytes); +} + +inline void CachePool::construct_add(uint64_t bytes, + ConstructPlug &plug) noexcept { + plug.bytes += bytes; +} + +inline bool CachePool::alloc_to(size_t size, ObjectPtr *dst) { + return allocator_->alloc_to(size, dst); +} + +inline bool CachePool::free(ObjectPtr &ptr) { + if (ptr.is_victim()) + vcache_->remove(&ptr); + return allocator_->free(ptr); +} + +inline void CachePool::inc_cache_hit() noexcept { stats.hits++; } + +inline void CachePool::inc_cache_miss() noexcept { stats.misses++; } + +inline void CachePool::inc_cache_victim_hit(ObjectPtr *optr_addr) noexcept { + stats.victim_hits++; + if (optr_addr) + vcache_->get(optr_addr); +} + +inline void CachePool::record_miss_penalty(uint64_t cycles, + uint64_t bytes) noexcept { + stats.miss_cycles += cycles; + stats.miss_bytes += bytes; +} + +inline VictimCache *CachePool::get_vcache() const noexcept { + return vcache_.get(); +} + +inline LogAllocator *CachePool::get_allocator() const noexcept { + return allocator_.get(); +} + +inline Evacuator *CachePool::get_evacuator() const noexcept { + return evacuator_.get(); +} + +inline ResourceManager *CachePool::get_rmanager() const noexcept { + return rmanager_.get(); +} + +inline CacheManager::~CacheManager() { + terminated_ = true; + if (profiler_) + profiler_->join(); + pools_.clear(); +} + +inline CachePool *CacheManager::get_pool(std::string name) { + std::unique_lock ul(mtx_); + auto found = pools_.find(name); + if (found == pools_.cend()) + return nullptr; + return found->second.get(); +} + +inline size_t CacheManager::num_pools() const noexcept { return pools_.size(); } + +inline CacheManager *CacheManager::global_cache_manager() { + static std::mutex mtx_; + static std::unique_ptr manager_; + + if (manager_.get()) + return manager_.get(); + std::unique_lock ul(mtx_); + if (manager_.get()) + return manager_.get(); + manager_ = std::make_unique(); + return manager_.get(); +} + +} // namespace midas \ No newline at end of file diff --git a/inc/impl/evacuator.ipp b/inc/impl/evacuator.ipp new file mode 100644 index 0000000..7e2894e --- /dev/null +++ b/inc/impl/evacuator.ipp @@ -0,0 +1,32 @@ +#pragma once + +#include +#include +#include +#include + +namespace midas { + +inline Evacuator::Evacuator(CachePool *pool, + std::shared_ptr rmanager, + std::shared_ptr allocator) + : pool_(pool), rmanager_(rmanager), allocator_(allocator), + terminated_(false) { + init(); +} + +inline void Evacuator::signal_gc() { + std::unique_lock ul(gc_mtx_); + gc_cv_.notify_all(); +} + +inline Evacuator::~Evacuator() { + terminated_ = true; + signal_gc(); + if (gc_thd_) { + gc_thd_->join(); + gc_thd_.reset(); + } +} + +} // namespace midas \ No newline at end of file diff --git a/inc/impl/fs_shim.ipp b/inc/impl/fs_shim.ipp new file mode 100644 index 0000000..f8bcf33 --- /dev/null +++ b/inc/impl/fs_shim.ipp @@ -0,0 +1,102 @@ +#pragma once + +namespace midas { +inline File::File(CachePool *pool) + : offset(0), cache(std::make_shared(pool)) {} + +inline FSShim::FSShim() { capture_syscalls(); } + +inline void FSShim::init_cache() { + if (pool_) + return; + auto cmanager = CacheManager::global_cache_manager(); + assert(cmanager->create_pool(pool_name)); + pool_ = cmanager->get_pool(pool_name); + assert(pool_); + pool_->update_limit(kPageCacheLimit); +} + +inline void FSShim::capture_syscalls() { + open = (int (*)(const char *, int, mode_t))dlsym(RTLD_NEXT, "open"); + open64 = (int (*)(const char *, int, mode_t))dlsym(RTLD_NEXT, "open64"); + creat = (int (*)(const char *, int, mode_t))dlsym(RTLD_NEXT, "creat"); + creat64 = (int (*)(const char *, int, mode_t))dlsym(RTLD_NEXT, "creat64"); + openat = (int (*)(int, const char *, int, mode_t))dlsym(RTLD_NEXT, "openat"); + openat64 = + (int (*)(int, const char *, int, mode_t))dlsym(RTLD_NEXT, "openat64"); + dup = (int (*)(int))dlsym(RTLD_NEXT, "dup"); + dup2 = (int (*)(int, int))dlsym(RTLD_NEXT, "dup2"); + close = (int (*)(int))dlsym(RTLD_NEXT, "close"); + fopen = (FILE * (*)(const char *, const char *)) dlsym(RTLD_NEXT, "fopen"); + fopen64 = + (FILE * (*)(const char *, const char *)) dlsym(RTLD_NEXT, "fopen64"); + fclose = (int (*)(FILE *))dlsym(RTLD_NEXT, "fclose"); + + read = (ssize_t(*)(int, void *, size_t))dlsym(RTLD_NEXT, "read"); + write = (ssize_t(*)(int, const void *, size_t))dlsym(RTLD_NEXT, "write"); + pread = (ssize_t(*)(int, void *, size_t, off_t))dlsym(RTLD_NEXT, "pread"); + pwrite = + (ssize_t(*)(int, const void *, size_t, off_t))dlsym(RTLD_NEXT, "pwrite"); + fread = (size_t(*)(void *, size_t, size_t, FILE *))dlsym(RTLD_NEXT, "fread"); + fwrite = (size_t(*)(const void *, size_t, size_t, FILE *))dlsym(RTLD_NEXT, + "fwrite"); + lseek = (off_t(*)(int, off_t, int))dlsym(RTLD_NEXT, "lseek"); + + char *error = nullptr; + if ((error = dlerror()) != nullptr) + MIDAS_ABORT("%s", error); +} + +inline void FSShim::exclude_interpose(int fd) { + if (fd <= STDERR_FILENO) + return; + std::unique_lock ul(bl_mtx_); + black_list_.emplace(fd); + // MIDAS_LOG(kError) << fd; +} + +inline void FSShim::reset_interpose(int fd) { + if (fd <= STDERR_FILENO) + return; + std::unique_lock ul(bl_mtx_); + black_list_.erase(fd); + // MIDAS_LOG(kError) << fd; +} + +inline bool FSShim::is_excluded(int fd) { + if (fd <= STDERR_FILENO) + return true; + std::unique_lock ul(bl_mtx_); + return black_list_.find(fd) != black_list_.cend(); +} + +inline FSShim *FSShim::global_shim() { + static std::mutex mtx_; + static std::unique_ptr shim_; + + if (shim_) + return shim_.get(); + std::unique_lock ul(mtx_); + if (shim_) + return shim_.get(); + shim_ = std::make_unique(); + assert(shim_); + return shim_.get(); +} + +constexpr static ino_t INV_INODE = -1; +static inline ino_t get_inode(int fd) { + struct stat file_stat; + if (fstat(fd, &file_stat) < 0) + return INV_INODE; + return file_stat.st_ino; +} + +static inline bool invalid_inode(ino_t inode) { return inode == INV_INODE; } + +static inline bool is_shm_file(const char *pathname) { + constexpr static char shm_prefix[] = "/dev/shm"; + constexpr static int prefix_len = sizeof(shm_prefix) - 1; + return std::strncmp(pathname, shm_prefix, prefix_len) == 0; +} +} // namespace midas \ No newline at end of file diff --git a/inc/impl/log.ipp b/inc/impl/log.ipp new file mode 100644 index 0000000..c615a4b --- /dev/null +++ b/inc/impl/log.ipp @@ -0,0 +1,155 @@ +#pragma once + +namespace midas { + +/** LogSegment */ +inline LogSegment::LogSegment(LogAllocator *owner, int64_t rid, uint64_t addr) + : owner_(owner), alive_bytes_(kMaxAliveBytes), region_id_(rid), + start_addr_(addr), pos_(addr), sealed_(false), destroyed_(false) {} + +inline bool LogSegment::full() const noexcept { + return sealed_ || + pos_ - start_addr_ + sizeof(LargeObjectHdr) >= kLogSegmentSize; +} + +inline int32_t LogSegment::remaining_bytes() const noexcept { + return start_addr_ + kLogSegmentSize - pos_; +} + +inline void LogSegment::set_alive_bytes(int32_t alive_bytes) noexcept { + alive_bytes_ = alive_bytes; +} + +inline void LogSegment::seal() noexcept { sealed_ = true; } + +inline bool LogSegment::sealed() const noexcept { return sealed_; } + +inline bool LogSegment::destroyed() const noexcept { return destroyed_; } + +inline uint32_t LogSegment::size() const noexcept { return pos_ / kRegionSize; } + +inline float LogSegment::get_alive_ratio() const noexcept { + return static_cast(alive_bytes_) / kRegionSize; +} + +/** SegmentList */ +inline void SegmentList::push_back(std::shared_ptr segment) { + std::unique_lock ul_(lock_); + segments_.emplace_back(std::move(segment)); +} + +inline std::shared_ptr SegmentList::pop_front() { + std::unique_lock ul_(lock_); + if (segments_.empty()) + return nullptr; + auto max_retry = segments_.size(); + int retry = 0; + while (retry < max_retry) { + auto segment = segments_.front(); + segments_.pop_front(); + retry++; + if (segment->destroyed()) { + MIDAS_ABORT("impossible"); + continue; // remove destroyed segment (this should never happen though) + } else + return std::move(segment); + } + return nullptr; +} + +inline size_t SegmentList::size() noexcept { return segments_.size(); } + +inline bool SegmentList::empty() const noexcept { return segments_.empty(); } + +/** LogAllocator */ +inline LogAllocator::LogAllocator(CachePool *pool) : pool_(pool) { + assert(pool_); +} + +inline std::optional LogAllocator::alloc(size_t size) { + return alloc_(size, false); +} + +inline bool LogAllocator::alloc_to(size_t size, ObjectPtr *dst) { + auto optptr = alloc(size); + if (!optptr) + return false; + auto &ptr = *optptr; + *dst = ptr; + ptr.set_rref(dst); + return true; +} + +inline bool LogAllocator::free(ObjectPtr &ptr) { + return ptr.free() == RetCode::Succ; +} + +/* static functions */ +inline int64_t LogAllocator::total_access_cnt() noexcept { + return total_access_cnt_; +} + +inline void LogAllocator::reset_access_cnt() noexcept { total_access_cnt_ = 0; } + +inline int64_t LogAllocator::total_alive_cnt() noexcept { + return total_alive_cnt_; +} + +inline void LogAllocator::reset_alive_cnt() noexcept { total_alive_cnt_ = 0; } + +inline void LogAllocator::count_access() { + constexpr static int32_t kAccPrecision = 32; + access_cnt_++; + if (UNLIKELY(access_cnt_ >= kAccPrecision)) { + total_access_cnt_ += access_cnt_; + access_cnt_ = 0; + } +} + +inline void LogAllocator::count_alive(int val) { + constexpr static int32_t kAccPrecision = 32; + alive_cnt_ += val; + if (UNLIKELY(alive_cnt_ >= kAccPrecision || alive_cnt_ <= -kAccPrecision)) { + total_alive_cnt_ += alive_cnt_; + alive_cnt_ = 0; + } +} + +inline void LogAllocator::PCAB::thd_exit() { + if (local_seg) { // now only current PCAB holds the reference + local_seg->owner_->stashed_pcabs_.push_back(local_seg); + } + + if (access_cnt_) { + total_access_cnt_ += access_cnt_; + access_cnt_ = 0; + } + + if (alive_cnt_) { + total_alive_cnt_ += alive_cnt_; + alive_cnt_ = 0; + } +} + +/* A thread safe way to create a global allocator and get its reference. */ +inline std::shared_ptr +LogAllocator::global_allocator_shared_ptr() noexcept { + static std::mutex mtx_; + static std::shared_ptr allocator_(nullptr); + + if (LIKELY(allocator_.get() != nullptr)) + return allocator_; + + std::unique_lock lk(mtx_); + if (UNLIKELY(allocator_.get() != nullptr)) + return allocator_; + + allocator_ = std::make_unique(nullptr); + return allocator_; +} + +inline LogAllocator *LogAllocator::global_allocator() noexcept { + return global_allocator_shared_ptr().get(); +} + +} // namespace midas \ No newline at end of file diff --git a/inc/impl/obj_locker.ipp b/inc/impl/obj_locker.ipp new file mode 100644 index 0000000..4d5cff6 --- /dev/null +++ b/inc/impl/obj_locker.ipp @@ -0,0 +1,54 @@ +#pragma once + + +namespace midas { + +inline std::optional ObjLocker::try_lock(const TransientPtr &tptr) { + return _try_lock(tptr.ptr_); +} + +inline LockID ObjLocker::lock(const TransientPtr &tptr) { + return _lock(tptr.ptr_); +} + +inline void ObjLocker::unlock(LockID id) { mtxes_[id].unlock(); } + +inline std::optional ObjLocker::_try_lock(uint64_t obj_addr) { + int bucket = hash_val(obj_addr) % kNumMaps; + if (mtxes_[bucket].try_lock()) + return bucket; + + return std::nullopt; +} + +inline LockID ObjLocker::_lock(uint64_t obj_addr) { + if (obj_addr == 0) // obj is reset under the hood. + return -1; + int bucket = hash_val(obj_addr) % kNumMaps; + mtxes_[bucket].lock(); + return bucket; +} + +inline void ObjLocker::_unlock(uint64_t obj_addr) { + assert(obj_addr != 0); // obj is reset under the hood, this should not happen. + int bucket = hash_val(obj_addr) % kNumMaps; + mtxes_[bucket].unlock(); +} + +inline uint64_t ObjLocker::hash_val(uint64_t input) { + return robin_hood::hash_int(input); +} + +inline ObjLocker *ObjLocker::global_objlocker() noexcept { + static std::mutex mtx_; + static std::shared_ptr locker_; + if (locker_) + return locker_.get(); + std::unique_lock ul(mtx_); + if (locker_) + return locker_.get(); + locker_ = std::make_shared(); + return locker_.get(); +} + +} // namespace midas \ No newline at end of file diff --git a/inc/impl/object.ipp b/inc/impl/object.ipp new file mode 100644 index 0000000..da8ec79 --- /dev/null +++ b/inc/impl/object.ipp @@ -0,0 +1,512 @@ +#pragma once + +#include +#include + +#include "logging.hpp" +#include "utils.hpp" + +namespace midas { + +/** Generic Object */ +inline MetaObjectHdr::MetaObjectHdr() : flags(0) {} + +inline void MetaObjectHdr::set_invalid() noexcept { flags = kInvalidHdr; } + +inline bool MetaObjectHdr::is_valid() const noexcept { + return flags != kInvalidHdr; +} + +inline bool MetaObjectHdr::is_small_obj() const noexcept { + return (flags & (1ull << kSmallObjBit)); +} +inline void MetaObjectHdr::set_small_obj() noexcept { + flags |= (1ull << kSmallObjBit); +} + +inline void MetaObjectHdr::set_large_obj() noexcept { + flags &= ~(1ull << kSmallObjBit); +} + +inline bool MetaObjectHdr::is_present() const noexcept { + return flags & (1ull << kPresentBit); +} +inline void MetaObjectHdr::set_present() noexcept { + flags |= (1ull << kPresentBit); +} +inline void MetaObjectHdr::clr_present() noexcept { + flags &= ~(1ull << kPresentBit); +} + +inline bool MetaObjectHdr::is_accessed() const noexcept { + return flags & kAccessedMask; +} +inline void MetaObjectHdr::inc_accessed() noexcept { + int64_t accessed = (flags & kAccessedMask) >> kAccessedBit; + accessed = std::min(accessed + 1, 3ll); + + flags &= ~kAccessedMask; + flags |= (accessed << kAccessedBit); +} +inline void MetaObjectHdr::dec_accessed() noexcept { + int64_t accessed = (flags & kAccessedMask) >> kAccessedBit; + accessed = std::max(accessed - 1, 0ll); + + flags &= ~kAccessedMask; + flags |= (accessed << kAccessedBit); +} +inline void MetaObjectHdr::clr_accessed() noexcept { flags &= ~kAccessedMask; } + +inline bool MetaObjectHdr::is_evacuate() const noexcept { + return flags & (1ull << kEvacuateBit); +} +inline void MetaObjectHdr::set_evacuate() noexcept { + flags |= (1ull << kEvacuateBit); +} +inline void MetaObjectHdr::clr_evacuate() noexcept { + flags &= ~(1ull << kEvacuateBit); +} + +inline bool MetaObjectHdr::is_mutate() const noexcept { + return flags & (1ull << kMutateBit); +} +inline void MetaObjectHdr::set_mutate() noexcept { + flags |= (1ull << kMutateBit); +} +inline void MetaObjectHdr::clr_mutate() noexcept { + flags &= ~(1ull << kMutateBit); +} + +inline bool MetaObjectHdr::is_continue() const noexcept { + return flags & (1ull << kContinueBit); +} +inline void MetaObjectHdr::set_continue() noexcept { + flags |= (1ull << kContinueBit); +} +inline void MetaObjectHdr::clr_continue() noexcept { + flags &= ~(1ull << kContinueBit); +} + +inline MetaObjectHdr *MetaObjectHdr::cast_from(void *hdr) noexcept { + return reinterpret_cast(hdr); +} + +/** Small Object */ +inline SmallObjectHdr::SmallObjectHdr() : rref(0), size(0), flags(0) {} + +inline void SmallObjectHdr::init(uint32_t size, uint64_t rref) noexcept { + auto meta_hdr = reinterpret_cast(this); + meta_hdr->set_present(); + meta_hdr->set_small_obj(); + + set_size(size); + set_rref(rref); +} + +inline void SmallObjectHdr::set_invalid() noexcept { + auto *meta_hdr = reinterpret_cast(this); + meta_hdr->set_invalid(); + + auto bytes = *(reinterpret_cast(this)); + assert(bytes == kInvalidHdr); +} + +inline bool SmallObjectHdr::is_valid() noexcept { + auto *meta_hdr = reinterpret_cast(this); + return meta_hdr->is_valid(); +} + +inline void SmallObjectHdr::set_size(uint32_t size_) noexcept { + size_ = round_up_to_align(size_, kSmallObjSizeUnit); + assert(size_ < kSmallObjThreshold); + size = size_ / kSmallObjSizeUnit; +} +inline uint32_t SmallObjectHdr::get_size() const noexcept { + return size * kSmallObjSizeUnit; +} + +inline void SmallObjectHdr::set_rref(uint64_t addr) noexcept { rref = addr; } +inline uint64_t SmallObjectHdr::get_rref() const noexcept { return rref; } + +inline void SmallObjectHdr::set_flags(uint8_t flags_) noexcept { + flags = flags_; +} +inline uint8_t SmallObjectHdr::get_flags() const noexcept { return flags; } + +/** Large Object */ +inline LargeObjectHdr::LargeObjectHdr() : size(0), flags(0), rref(0), next(0) {} + +inline void LargeObjectHdr::init(uint32_t size_, bool is_head, + TransientPtr head_, + TransientPtr next_) noexcept { + auto *meta_hdr = MetaObjectHdr::cast_from(this); + meta_hdr->set_present(); + meta_hdr->set_large_obj(); + is_head ? meta_hdr->clr_continue() : meta_hdr->set_continue(); + + set_size(size_); + // for the first segment of a large obj, head_(rref_) must be 0 at this time. + assert(!is_head || head_.null()); + set_head(head_); + set_next(next_); +} + +inline void LargeObjectHdr::set_invalid() noexcept { + auto *meta_hdr = MetaObjectHdr::cast_from(this); + meta_hdr->set_invalid(); +} + +inline bool LargeObjectHdr::is_valid() noexcept { + auto *meta_hdr = MetaObjectHdr::cast_from(this); + return meta_hdr->is_valid(); +} + +inline void LargeObjectHdr::set_size(uint32_t size_) noexcept { size = size_; } +inline uint32_t LargeObjectHdr::get_size() const noexcept { return size; } + +inline void LargeObjectHdr::set_rref(uint64_t addr) noexcept { rref = addr; } +inline uint64_t LargeObjectHdr::get_rref() const noexcept { return rref; } + +inline void LargeObjectHdr::set_next(TransientPtr ptr) noexcept { + next = ptr.to_normal_address(); +} +inline TransientPtr LargeObjectHdr::get_next() const noexcept { + return TransientPtr(next, sizeof(LargeObjectHdr)); +} + +inline void LargeObjectHdr::set_head(TransientPtr ptr) noexcept { + rref = ptr.to_normal_address(); +} +inline TransientPtr LargeObjectHdr::get_head() const noexcept { + return TransientPtr(rref, sizeof(LargeObjectHdr)); +} + +inline void LargeObjectHdr::set_flags(uint32_t flags_) noexcept { + flags = flags_; +} +inline uint32_t LargeObjectHdr::get_flags() const noexcept { return flags; } + +/** ObjectPtr */ +inline ObjectPtr::ObjectPtr() + : small_obj_(true), head_obj_(true), victim_(false), size_(0), + deref_cnt_(0), obj_() {} + +inline bool ObjectPtr::null() const noexcept { return obj_.null(); } + +inline size_t ObjectPtr::obj_size(size_t data_size) noexcept { + data_size = round_up_to_align(data_size, kSmallObjSizeUnit); + return data_size < kSmallObjThreshold ? sizeof(SmallObjectHdr) + data_size + : sizeof(LargeObjectHdr) + data_size; +} + +inline size_t ObjectPtr::obj_size() const noexcept { + return hdr_size() + data_size_in_segment(); +} +inline size_t ObjectPtr::hdr_size() const noexcept { + return is_small_obj() ? sizeof(SmallObjectHdr) : sizeof(LargeObjectHdr); +} +/** Return data size in this segment. + * For small objects, it always equals to the data size; + * For large objects, the total size of the object can be larger if the + * object spans multiple segments. + */ +inline size_t ObjectPtr::data_size_in_segment() const noexcept { return size_; } + +/** Return total data size of the object for both small and large objects. */ +inline std::optional ObjectPtr::large_data_size() { + assert(!is_small_obj() && is_head_obj()); + size_t size = 0; + + ObjectPtr optr = *this; + while (!optr.null()) { + size += optr.data_size_in_segment(); + auto ret = iter_large(optr); + if (ret == RetCode::FaultLocal || ret == RetCode::FaultOther) + return std::nullopt; + else if (ret == RetCode::Fail) + break; + } + return size; +} + +inline bool ObjectPtr::is_small_obj() const noexcept { return small_obj_; } + +inline bool ObjectPtr::is_head_obj() const noexcept { return head_obj_; } + +inline void ObjectPtr::set_victim(bool victim) noexcept { victim_ = victim; } + +inline bool ObjectPtr::is_victim() const noexcept { return victim_; } + +inline bool ObjectPtr::contains(uint64_t addr) const noexcept { + auto stt_addr = obj_.to_normal_address(); + return stt_addr <= addr && addr < stt_addr + size_; +} + +using RetCode = ObjectPtr::RetCode; + +inline RetCode ObjectPtr::init_small(uint64_t stt_addr, size_t data_size) { + small_obj_ = true; + size_ = round_up_to_align(data_size, kSmallObjSizeUnit); + deref_cnt_ = 0; + + SmallObjectHdr hdr; + hdr.init(size_); + obj_ = TransientPtr(stt_addr, obj_size()); + return store_hdr(hdr, obj_) ? RetCode::Succ : RetCode::FaultLocal; +} + +inline RetCode ObjectPtr::init_large(uint64_t stt_addr, size_t data_size, + bool is_head, TransientPtr head, + TransientPtr next) { + assert(data_size <= kLogSegmentSize - sizeof(LargeObjectHdr)); + small_obj_ = false; + head_obj_ = is_head; + size_ = data_size; // YIFAN: check this later!! + deref_cnt_ = 0; + + LargeObjectHdr hdr; + hdr.init(data_size, is_head, head, next); + obj_ = TransientPtr(stt_addr, obj_size()); + return store_hdr(hdr, obj_) ? RetCode::Succ : RetCode::FaultLocal; +} + +inline RetCode ObjectPtr::init_from_soft(TransientPtr soft_ptr) { + deref_cnt_ = 0; + + MetaObjectHdr hdr; + obj_ = soft_ptr; + if (!load_hdr(hdr, *this)) + return RetCode::FaultLocal; + + if (!hdr.is_valid()) + return RetCode::Fail; + + if (hdr.is_small_obj()) { + SmallObjectHdr shdr = *(reinterpret_cast(&hdr)); + small_obj_ = true; + size_ = shdr.get_size(); + obj_ = TransientPtr(soft_ptr.to_normal_address(), obj_size() + size_); + } else { + LargeObjectHdr lhdr; + if (!load_hdr(lhdr, *this)) + return RetCode::FaultLocal; + small_obj_ = false; + head_obj_ = !MetaObjectHdr::cast_from(&lhdr)->is_continue(); + size_ = lhdr.get_size(); + obj_ = TransientPtr(soft_ptr.to_normal_address(), obj_size() + size_); + } + + return RetCode::Succ; +} + +inline RetCode ObjectPtr::free_small() noexcept { + assert(!null()); + + MetaObjectHdr meta_hdr; + if (!load_hdr(meta_hdr, *this)) + return RetCode::FaultLocal; + + if (!meta_hdr.is_valid()) + return RetCode::Fail; + meta_hdr.clr_present(); + auto ret = store_hdr(meta_hdr, *this) ? RetCode::Succ : RetCode::FaultLocal; + auto rref = reinterpret_cast(get_rref()); + if (rref) + rref->obj_.reset(); + return ret; +} + +inline RetCode ObjectPtr::free_large() noexcept { + assert(!null()); + + LargeObjectHdr hdr; + if (!load_hdr(hdr, *this)) + return RetCode::FaultLocal; + MetaObjectHdr meta_hdr = *MetaObjectHdr::cast_from(&hdr); + if (!meta_hdr.is_valid()) { + MIDAS_LOG(kError); + return RetCode::Fail; + } + meta_hdr.clr_present(); + auto ret = store_hdr(meta_hdr, *this) ? RetCode::Succ : RetCode::FaultLocal; + if (ret != RetCode::Succ) + return ret; + + auto rref = reinterpret_cast(get_rref()); + if (rref) + rref->obj_.reset(); + + auto next = hdr.get_next(); + while (!next.null()) { + ObjectPtr optr; + if (optr.init_from_soft(next) != RetCode::Succ || !load_hdr(hdr, optr)) + return RetCode::FaultOther; + next = hdr.get_next(); + + MetaObjectHdr::cast_from(&hdr)->clr_present(); + if (!store_hdr(hdr, optr)) + return RetCode::FaultOther; + } + return ret; +} + +inline bool ObjectPtr::set_rref(uint64_t addr) noexcept { + assert(!null()); + if (is_small_obj()) { + SmallObjectHdr hdr; + if (!load_hdr(hdr, *this)) + return false; + hdr.set_rref(addr); + if (!store_hdr(hdr, *this)) + return false; + } else { + LargeObjectHdr hdr; + if (!load_hdr(hdr, *this)) + return false; + hdr.set_rref(addr); + if (!store_hdr(hdr, *this)) + return false; + } + return true; +} + +inline bool ObjectPtr::set_rref(ObjectPtr *addr) noexcept { + return set_rref(reinterpret_cast(addr)); +} + +inline ObjectPtr *ObjectPtr::get_rref() noexcept { + assert(!null()); + if (is_small_obj()) { + SmallObjectHdr hdr; + if (!load_hdr(hdr, *this)) + return nullptr; + return reinterpret_cast(hdr.get_rref()); + } else { + LargeObjectHdr hdr; + if (!load_hdr(hdr, *this)) + return nullptr; + return reinterpret_cast(hdr.get_rref()); + } + MIDAS_ABORT("impossible to reach here!"); + return nullptr; +} + +inline RetCode ObjectPtr::upd_rref() noexcept { + auto *ref = get_rref(); + if (!ref) + return RetCode::Fail; + /* We should update obj_ and size_ atomically. As size_ can be protected by + * the object lock, the bottomline is to update obj_ atomically. So far we + * rely on 64bit CPU to do so. */ + ref->obj_ = this->obj_; + ref->size_ = this->size_; + // *ref = *this; + return RetCode::Succ; +} + +inline bool ObjectPtr::cmpxchg(int64_t offset, uint64_t oldval, + uint64_t newval) { + if (null()) + return false; + return obj_.cmpxchg(hdr_size() + offset, oldval, newval); +} + +inline bool ObjectPtr::copy_from(const void *src, size_t len, int64_t offset) { + return is_small_obj() ? copy_from_small(src, len, offset) + : copy_from_large(src, len, offset); +} + +inline bool ObjectPtr::copy_to(void *dst, size_t len, int64_t offset) { + return is_small_obj() ? copy_to_small(dst, len, offset) + : copy_to_large(dst, len, offset); +} + +inline RetCode ObjectPtr::move_from(ObjectPtr &src) { + if (null() || src.null()) + return RetCode::Fail; + + /* NOTE (YIFAN): the order of operations below can be tricky: + * 1. copy data from src to this. + * 2. free src (rref will be reset to nullptr). + * 3. mark this as present, finish setup. + * 4. update rref, let it point to this. + */ + if (src.is_small_obj()) { // small object + assert(src.obj_size() == this->obj_size()); + auto ret = RetCode::Fail; + if (!obj_.copy_from(src.obj_, src.obj_size())) + return RetCode::Fail; + ret = src.free(/* locked = */ true); + if (ret != RetCode::Succ) + return ret; + MetaObjectHdr meta_hdr; + if (!load_hdr(meta_hdr, *this)) + return ret; + meta_hdr.set_present(); + if (!store_hdr(meta_hdr, *this)) + return ret; + ret = upd_rref(); + if (ret != RetCode::Succ) + return ret; + } else { // large object + assert(src.is_head_obj()); + assert(!is_small_obj() && is_head_obj()); + // assert(*src.large_data_size() == *large_data_size()); + auto ret = move_large(src); + if (ret != RetCode::Succ) + return ret; + ret = src.free(/* locked = */ true); + LargeObjectHdr lhdr; + if (!load_hdr(lhdr, src)) + return RetCode::FaultLocal; // src is considered as local + auto rref = lhdr.get_rref(); + if (!load_hdr(lhdr, *this)) + return RetCode::FaultOther; // dst, hereby this, is considered as other + lhdr.set_rref(rref); + if (!store_hdr(lhdr, *this)) + return RetCode::FaultOther; + ret = upd_rref(); + if (ret != RetCode::Succ) + return ret; + } + return RetCode::Succ; +} + +inline RetCode ObjectPtr::iter_large(ObjectPtr &optr) { + LargeObjectHdr lhdr; + if (!load_hdr(lhdr, optr)) + return RetCode::FaultLocal; // fault on optr itself + auto next = lhdr.get_next(); + if (next.null()) + return RetCode::Fail; + if (optr.init_from_soft(next) == RetCode::Succ) + return RetCode::Succ; + return RetCode::FaultOther; // this must be fault on next segment +} + +inline const std::string ObjectPtr::to_string() noexcept { + std::stringstream sstream; + sstream << (is_small_obj() ? "Small" : "Large") << " Object @ " << std::hex + << obj_.to_normal_address(); + return sstream.str(); +} + +template inline bool load_hdr(T &hdr, TransientPtr &tptr) noexcept { + return tptr.copy_to(&hdr, sizeof(hdr)); +} + +template +inline bool store_hdr(const T &hdr, TransientPtr &tptr) noexcept { + return tptr.copy_from(&hdr, sizeof(hdr)); +} + +template inline bool load_hdr(T &hdr, ObjectPtr &optr) noexcept { + return load_hdr(hdr, optr.obj_); +} + +template +inline bool store_hdr(const T &hdr, ObjectPtr &optr) noexcept { + return store_hdr(hdr, optr.obj_); +} + +} // namespace midas \ No newline at end of file diff --git a/inc/impl/qpair.ipp b/inc/impl/qpair.ipp new file mode 100644 index 0000000..291807a --- /dev/null +++ b/inc/impl/qpair.ipp @@ -0,0 +1,142 @@ +#pragma once +#include +#include +#include + +#include "logging.hpp" + +namespace midas { + +namespace utils { +inline const std::string get_sq_name(std::string qpname, bool create) { + return create ? kSQPrefix + qpname : kRQPrefix + qpname; +} + +inline const std::string get_rq_name(std::string qpname, bool create) { + return get_sq_name(qpname, !create); +} + +inline const std::string get_ackq_name(std::string qpname, uint64_t id) { + return kSQPrefix + qpname + "-" + std::to_string(id); +} +} // namespace utils + +inline int QSingle::send(const void *buffer, size_t buffer_size) { + try { + q_->send(buffer, buffer_size, /* prio = */ 0); + } catch (boost::interprocess::interprocess_exception &e) { + MIDAS_LOG(kError) << e.what(); + return -1; + } + return 0; +} + +inline int QSingle::recv(void *buffer, size_t buffer_size) { + try { + unsigned priority; + size_t recvd_size; + q_->receive(buffer, buffer_size, recvd_size, priority); + if (recvd_size != buffer_size) { + MIDAS_LOG(kError) << "Q " << name_ << " recv error: " << recvd_size + << "!=" << buffer_size; + return -1; + } + } catch (boost::interprocess::interprocess_exception &e) { + MIDAS_LOG(kError) << e.what(); + return -1; + } + return 0; +} + +inline int QSingle::try_recv(void *buffer, size_t buffer_size) { + try { + unsigned priority; + size_t recvd_size; + if (!q_->try_receive(buffer, buffer_size, recvd_size, priority)) + return -1; + if (recvd_size != buffer_size) { + MIDAS_LOG(kError) << "Q " << name_ << " recv error: " << recvd_size + << "!=" << buffer_size; + return -1; + } + } catch (boost::interprocess::interprocess_exception &e) { + MIDAS_LOG(kError) << e.what(); + return -1; + } + return 0; +} + +inline int QSingle::timed_recv(void *buffer, size_t buffer_size, int timeout) { + try { + using namespace boost::posix_time; + using namespace boost::gregorian; + + unsigned priority; + size_t recvd_size = 0; + if (!q_->timed_receive( + buffer, buffer_size, recvd_size, priority, + second_clock::universal_time() + seconds(timeout))) + return -1; + if (recvd_size != buffer_size) { + MIDAS_LOG(kError) << "Q " << name_ << " recv error: " << recvd_size + << "!=" << buffer_size; + return -1; + } + } catch (boost::interprocess::interprocess_exception &e) { + MIDAS_LOG(kError) << e.what(); + return -1; + } + return 0; +} + +inline void QSingle::init(bool create) { + try { + if (create) { + boost::interprocess::permissions perms; + perms.set_unrestricted(); + q_ = std::make_shared(boost::interprocess::create_only, + name_.c_str(), qdepth_, msgsize_, perms); + } else + q_ = std::make_shared(boost::interprocess::open_only, + name_.c_str()); + } catch (boost::interprocess::interprocess_exception &e) { + std::cerr << e.what() << std::endl; + } +} + +inline QPair::QPair(std::string qpname, bool create, uint32_t qdepth, + uint32_t msgsize) + : sq_(std::make_shared(utils::get_sq_name(qpname, create), create, + qdepth, msgsize)), + rq_(std::make_shared(utils::get_rq_name(qpname, create), create, + qdepth, msgsize)) {} + +inline void QSingle::destroy() { + try { + MsgQueue::remove(name_.c_str()); + } catch (boost::interprocess::interprocess_exception &e) { + MIDAS_LOG(kError) << e.what(); + } +} + +inline void QPair::destroy() { + sq_->destroy(); + rq_->destroy(); +} + +inline int QPair::send(const void *buffer, size_t buffer_size) { + return sq_->send(buffer, buffer_size); +} + +inline int QPair::recv(void *buffer, size_t buffer_size) { + return rq_->recv(buffer, buffer_size); +} + +inline int QPair::try_recv(void *buffer, size_t buffer_size) { + return rq_->try_recv(buffer, buffer_size); +} + +inline int QPair::timed_recv(void *buffer, size_t buffer_size, int timeout) { + return rq_->timed_recv(buffer, buffer_size, timeout); +} +} // namespace midas \ No newline at end of file diff --git a/inc/impl/resilient_func.ipp b/inc/impl/resilient_func.ipp new file mode 100644 index 0000000..600c9c5 --- /dev/null +++ b/inc/impl/resilient_func.ipp @@ -0,0 +1,38 @@ +#include + +namespace midas { +static inline bool match_fp_prologue(uint64_t func_stt) { + /** + * X86-64 function frame pointer prologue: + * 0x55 push %rbp + * ... (parameter passing, etc.) + * 0x48 89 e5 mov %rsp %rbp + * ... + * function frame pointer epilogue: + * 0x5d pop %rbp + * 0xc3 ret + */ + // constexpr static uint32_t fp_prologue = 0xe5'89'48'55; + constexpr static uint8_t fp_prologue = 0x55; + uint8_t *code = reinterpret_cast(func_stt); + return code[0] == fp_prologue; +} + +inline ResilientFunc::ResilientFunc(uint64_t stt_ip_, uint64_t end_ip_) + : stt_ip(stt_ip_), end_ip(end_ip_), fail_entry(0) { + omitted_frame_pointer = !match_fp_prologue(stt_ip); + if (omitted_frame_pointer) { + constexpr static uint8_t int3 = 0xcc; + constexpr static uint8_t retq = 0xc3; + fail_entry = end_ip + 4; // we insert 4 int3 instructions in func_delimiter + while (*(reinterpret_cast(fail_entry)) == int3) + fail_entry++; + assert(*(reinterpret_cast(fail_entry)) == retq); + } +} + +inline bool ResilientFunc::contain(uint64_t fault_ip) { + return stt_ip < fault_ip && fault_ip < end_ip; +} + +} // namespace midas \ No newline at end of file diff --git a/inc/impl/resource_manager.ipp b/inc/impl/resource_manager.ipp new file mode 100644 index 0000000..2afb00d --- /dev/null +++ b/inc/impl/resource_manager.ipp @@ -0,0 +1,47 @@ +#pragma once + +namespace midas { + +static inline uint64_t get_unique_id() { + static std::random_device rd; + static std::mt19937 gen(rd()); + static std::uniform_int_distribution<> distr(1, 1ull << 20); + + uint64_t pid = boost::interprocess::ipcdetail::get_current_process_id(); + uint64_t rand = distr(gen); + + return (pid * 1'000'000'000'000ull) + rand; +} + +inline VRange ResourceManager::GetRegion(int64_t region_id) noexcept { + std::unique_lock lk(mtx_); + if (region_map_.find(region_id) == region_map_.cend()) + return VRange(); + auto ®ion = region_map_[region_id]; + return VRange(region->Addr(), region->Size()); +} + +inline uint64_t ResourceManager::NumRegionInUse() const noexcept { + // std::unique_lock lk(mtx_); + /* YIFAN: NOTE: so far we don't count regions in freelist as in-use since they + * are only for reducing IPC across client<-> daemon, and the app never uses + * free regions until they allocate. If in the future we want to count their + * usage, we need to enforce evacuator to skip freelist to avoid they eat up + * application's cache portion. */ + return region_map_.size(); + // return region_map_.size() + freelist_.size(); +} + +inline uint64_t ResourceManager::NumRegionLimit() const noexcept { + return region_limit_; +} + +inline int64_t ResourceManager::NumRegionAvail() const noexcept { + return static_cast(NumRegionLimit()) - NumRegionInUse(); +} + +inline ResourceManager *ResourceManager::global_manager() noexcept { + return global_manager_shared_ptr().get(); +} + +} // namespace midas \ No newline at end of file diff --git a/inc/impl/sig_handler.ipp b/inc/impl/sig_handler.ipp new file mode 100644 index 0000000..a326752 --- /dev/null +++ b/inc/impl/sig_handler.ipp @@ -0,0 +1,22 @@ +#include +#include + +namespace midas { +static inline bool in_volatile_range(uint64_t addr) { + return addr >= midas::kVolatileSttAddr && + addr < midas::kVolatileEndAddr; +} + +inline SigHandler *SigHandler::global_sighandler() { + static std::mutex mtx_; + static std::shared_ptr hdler_; + if (hdler_) + return hdler_.get(); + std::unique_lock ul(mtx_); + if (hdler_) + return hdler_.get(); + hdler_ = std::make_shared(); + return hdler_.get(); +} + +} // namespace midas \ No newline at end of file diff --git a/inc/impl/slab.ipp b/inc/impl/slab.ipp new file mode 100644 index 0000000..31b604e --- /dev/null +++ b/inc/impl/slab.ipp @@ -0,0 +1,39 @@ +#pragma once + +#ifdef ENABLE_SLAB + +namespace midas { +inline uint32_t SlabAllocator::get_slab_idx(uint32_t size) noexcept { + uint32_t rounded_size = utils::round_up_power_of_two(size); + assert(rounded_size <= kMaxSlabSize); + return utils::bsr_32(std::max(rounded_size >> kMinSlabShift, 1u)); +} + +inline uint32_t SlabAllocator::get_slab_size(uint32_t idx) noexcept { + return kMinSlabSize << idx; +} + +inline void *SlabAllocator::alloc(uint32_t size) { return _alloc(size); } + +template T *SlabAllocator::alloc(uint32_t cnt) { + return reinterpret_cast(_alloc(sizeof(T) * cnt)); +} + +/* A thread safe way to create a global allocator and get its reference. */ +inline SlabAllocator *SlabAllocator::global_allocator() { + static std::mutex mtx_; + static std::unique_ptr _allocator(nullptr); + + if (LIKELY(_allocator.get() != nullptr)) + return _allocator.get(); + + std::unique_lock lk(mtx_); + if (UNLIKELY(_allocator.get() != nullptr)) + return _allocator.get(); + + _allocator = std::make_unique(); + return _allocator.get(); +} +} // namespace midas + +#endif // ENABLE_SLAB \ No newline at end of file diff --git a/inc/impl/sync_hashmap.ipp b/inc/impl/sync_hashmap.ipp new file mode 100644 index 0000000..2d0146e --- /dev/null +++ b/inc/impl/sync_hashmap.ipp @@ -0,0 +1,259 @@ +#pragma once + +namespace midas { + +template +SyncHashMap::SyncHashMap() { + pool_ = CachePool::global_cache_pool(); + memset(buckets_, 0, sizeof(buckets_)); +} + +template +SyncHashMap::SyncHashMap( + CachePool *pool) + : pool_(pool) { + memset(buckets_, 0, sizeof(buckets_)); +} + +template +SyncHashMap::~SyncHashMap() { + clear(); +} + +template +template +std::unique_ptr +SyncHashMap::get(K1 &&k) { + Tp *stored_v = reinterpret_cast(::operator new(sizeof(Tp))); + if (get(std::forward(k), *stored_v)) + return std::unique_ptr(stored_v); + if (stored_v) + ::operator delete(stored_v); + return nullptr; +} + +template +template +bool SyncHashMap::get(K1 &&k, + Tp &v) { + auto hasher = Hash(); + auto key_hash = hasher(k); + auto bucket_idx = key_hash % NBuckets; + + auto ul = std::unique_lock(locks_[bucket_idx]); + + auto prev_next = &buckets_[bucket_idx]; + BNPtr node = buckets_[bucket_idx]; + bool found = false; + while (node) { + found = iterate_list(key_hash, k, prev_next, node); + if (found) + break; + } + if (!found) { + ul.unlock(); + pool_->inc_cache_miss(); + goto failed; + } + assert(node); + if (node->pair.null() || !node->pair.copy_to(&v, sizeof(Tp), sizeof(Key))) { + if (node->pair.is_victim()) + pool_->inc_cache_victim_hit(&node->pair); + node = delete_node(prev_next, node); + ul.unlock(); + goto failed; + } + ul.unlock(); + pool_->inc_cache_hit(); + LogAllocator::count_access(); + return true; +failed: + if (kEnableConstruct && pool_->get_construct_func()) { + ConstructArgs args = {&k, sizeof(k), &v, sizeof(v)}; + ConstructPlug plug; + pool_->construct_stt(plug); + bool succ = pool_->construct(&args) == 0; + if (!succ) // failed to re-construct + return false; + // successfully re-constructed + set(k, v); + pool_->construct_add(sizeof(v), plug); + pool_->construct_end(plug); + return succ; + } + return false; +} + +template +template +bool SyncHashMap::remove(K1 &&k) { + auto hasher = Hash(); + auto key_hash = hasher(k); + auto bucket_idx = key_hash % NBuckets; + + auto ul = std::unique_lock(locks_[bucket_idx]); + + auto prev_next = &buckets_[bucket_idx]; + BNPtr node = buckets_[bucket_idx]; + bool found = false; + while (node) { + found = iterate_list(key_hash, k, prev_next, node); + if (found) + break; + } + if (!found) + return false; + assert(node); + delete_node(prev_next, node); + /* should not count access for remove() */ + // LogAllocator::count_access(); + return true; +} + +template +template +bool SyncHashMap::set( + const K1 &k, const Tp1 &v) { + auto hasher = Hash(); + auto key_hash = hasher(k); + auto bucket_idx = key_hash % NBuckets; + + auto ul = std::unique_lock(locks_[bucket_idx]); + + auto prev_next = &buckets_[bucket_idx]; + auto node = buckets_[bucket_idx]; + while (node) { + auto found = iterate_list(key_hash, k, prev_next, node); + if (found) { + // Tp tmp_v = v; + assert(sizeof(v) <= sizeof(Tp)); + // try to set in place + if (!node->pair.null() && + node->pair.copy_from(&v, sizeof(Tp1), sizeof(Key))) { + ul.unlock(); + LogAllocator::count_access(); + return true; + } else { + node = delete_node(prev_next, node); + break; + } + } + } + + auto new_node = create_node(key_hash, k, v); + if (!new_node) + return false; + *prev_next = new_node; + ul.unlock(); + LogAllocator::count_access(); + return true; +} + +template +bool SyncHashMap::clear() { + for (int idx = 0; idx < NBuckets; idx++) { + auto ul = std::unique_lock(locks_[idx]); + auto prev_next = &buckets_[idx]; + auto node = buckets_[idx]; + while (node) + node = delete_node(prev_next, node); + } + return true; +} + +template +using BNPtr = + typename SyncHashMap::BucketNode + *; + +/** Utility functions */ +template +template +inline BNPtr +SyncHashMap::create_node( + uint64_t key_hash, K1 &&k, Tp1 &&v) { + // Tp tmp_v = v; + assert(sizeof(k) <= sizeof(Key)); + assert(sizeof(v) <= sizeof(Tp)); + // auto allocator = pool_->get_allocator(); + + auto *new_node = new BucketNode(); + if (!pool_->alloc_to(sizeof(Key) + sizeof(Tp), &new_node->pair) || + !new_node->pair.copy_from(&k, sizeof(Key)) || + !new_node->pair.copy_from(&v, sizeof(Tp1), sizeof(Key))) { + delete new_node; + return nullptr; + } + // assert(!new_node->pair.null()); + if (new_node->pair.null()) { + MIDAS_LOG(kError) << "new node KV pair is freed!"; + delete new_node; + return nullptr; + } + new_node->key_hash = key_hash; + new_node->next = nullptr; + return new_node; +} + +/** remove bucket_node from the list */ +// should always use as `node = delete_node()` when iterating the list +template +inline BNPtr +SyncHashMap::delete_node( + BNPtr *prev_next, BNPtr node) { + assert(*prev_next == node); + if (!node) + return nullptr; + auto next = node->next; + + // node->pair.free(); + // if (node->pair.is_victim()) + // pool_->get_vcache()->remove(&node->pair); + pool_->free(node->pair); + delete node; + + *prev_next = next; + return next; +} + +/** return: */ +template +template +inline bool +SyncHashMap::iterate_list( + uint64_t key_hash, K1 &&k, BNPtr *&prev_next, BNPtr &node) { + if (key_hash != node->key_hash) { + prev_next = &(node->next); + node = node->next; + return false; + } + std::byte k_buf[sizeof(Key)]; + auto tmp_k = std::launder(reinterpret_cast(&k_buf)); + if (node->pair.null() || !node->pair.copy_to(tmp_k, sizeof(Key))) { + if (node->pair.is_victim()) + pool_->inc_cache_victim_hit(&node->pair); + // prev remains the same when current node is deleted. + node = delete_node(prev_next, node); + return false; + } + if (!Pred()(k, *tmp_k)) { + prev_next = &(node->next); + node = node->next; + return false; + } + return true; +} + +} // namespace midas diff --git a/inc/impl/sync_kv.ipp b/inc/impl/sync_kv.ipp new file mode 100644 index 0000000..6168c0f --- /dev/null +++ b/inc/impl/sync_kv.ipp @@ -0,0 +1,784 @@ +#pragma once + +namespace midas { + +template +SyncKV::SyncKV() { + pool_ = CachePool::global_cache_pool(); + memset(buckets_, 0, sizeof(buckets_)); +} + +template +SyncKV::SyncKV(CachePool *pool) : pool_(pool) { + memset(buckets_, 0, sizeof(buckets_)); +} + +template +SyncKV::~SyncKV() { + clear(); +} + +/** Storage layout in soft memory: + * | KeyLen (8B) | ValueLen (8B) | Key (`KeyLen`B) | Value (`ValueLen`B) | + */ +namespace layout { +static inline size_t klen_offset() { return 0; } +static inline size_t vlen_offset() { return klen_offset() + sizeof(size_t); } +static inline size_t k_offset() { return vlen_offset() + sizeof(size_t); } +static inline size_t v_offset(size_t keylen) { return k_offset() + keylen; } +} // namespace layout + +namespace kv_utils { +template T make_(decltype(T::data) data, decltype(T::size) size) { + return T(data, size); +} +} // namespace kv_utils + +/** Base Interfaces */ +template +void *SyncKV::get(const void *k, size_t kn, size_t *vn) { + return get_(k, kn, nullptr, vn, nullptr, true); +} + +template +kv_types::Value SyncKV::get(const kv_types::Key &k) { + size_t vn = 0; + auto v = get(k.data, k.size, &vn); + return kv_utils::make_value(v, vn); +} + +template +template +kv_types::Value SyncKV::get(const K &k) { + return get(kv_utils::make_key(&k, sizeof(K))); +} + +template +template +std::unique_ptr SyncKV::get(const K &k) { + auto [raw_v, vn] = get(k); + auto v = std::unique_ptr(reinterpret_cast(raw_v)); + if (vn != sizeof(V)) + return nullptr; // this will free raw_v as well if it was allocated + return std::move(v); +} + +/* Use this func when the value has already had a buffer at @v with size @vn. */ +template +bool SyncKV::get(const void *k, size_t kn, void *v, + size_t vn) { + if (!v) + return false; + size_t stored_vn = 0; + if (get_(k, kn, v, &stored_vn, nullptr, true) == nullptr) + return false; + if (stored_vn < vn) // value size check + return false; + return true; +} + +template +bool SyncKV::remove(const void *k, size_t kn) { + auto key_hash = hash_(k, kn); + auto bucket_idx = key_hash % NBuckets; + + auto ul = std::unique_lock(locks_[bucket_idx]); + + auto prev_next = &buckets_[bucket_idx]; + BNPtr node = buckets_[bucket_idx]; + bool found = false; + while (node) { + found = iterate_list(key_hash, k, kn, nullptr, prev_next, node); + if (found) + break; + } + if (!found) + return false; + assert(node); + delete_node(prev_next, node); + /* should not count access for remove() */ + // LogAllocator::count_access(); + return true; +} + +template +bool SyncKV::remove(const kv_types::Key &k) { + return remove(k.data, k.size); +} + +template +template +bool SyncKV::remove(const K &k) { + return remove(&k, sizeof(K)); +} + +template +template +bool SyncKV::inc(const void *k, size_t kn, V offset, + V *value) { + auto key_hash = hash_(k, kn); + auto bucket_idx = key_hash % NBuckets; + + auto ul = std::unique_lock(locks_[bucket_idx]); + + auto prev_next = &buckets_[bucket_idx]; + BNPtr node = buckets_[bucket_idx]; + bool found = false; + size_t stored_vn = 0; + + while (node) { + found = iterate_list(key_hash, k, kn, &stored_vn, prev_next, node); + if (found) + break; + } + if (!found) + return false; + assert(node); + + if (stored_vn != sizeof(V) || node->pair.null() || + !node->pair.copy_to(value, sizeof(V), layout::v_offset(kn))) + return false; + *value = *value + offset; + if (!node->pair.copy_from(value, sizeof(V), layout::v_offset(kn))) + return false; + ul.unlock(); + LogAllocator::count_access(); + return true; +} + +template +int SyncKV::add(const void *k, size_t kn, const void *v, + size_t vn) { + auto key_hash = hash_(k, kn); + auto bucket_idx = key_hash % NBuckets; + + auto ul = std::unique_lock(locks_[bucket_idx]); + + size_t stored_vn = 0; + auto prev_next = &buckets_[bucket_idx]; + auto node = buckets_[bucket_idx]; + while (node) { + auto found = iterate_list(key_hash, k, kn, &stored_vn, prev_next, node); + if (found) + return kv_types::RetCode::Duplicated; + } + auto new_node = create_node(key_hash, k, kn, v, vn); + if (!new_node) + return kv_types::RetCode::Failed; + *prev_next = new_node; + ul.unlock(); + LogAllocator::count_access(); + return kv_types::RetCode::Succ; +} + +template +bool SyncKV::set(const void *k, size_t kn, const void *v, + size_t vn) { + auto key_hash = hash_(k, kn); + auto bucket_idx = key_hash % NBuckets; + + auto ul = std::unique_lock(locks_[bucket_idx]); + + size_t stored_vn = 0; + auto prev_next = &buckets_[bucket_idx]; + auto node = buckets_[bucket_idx]; + while (node) { + auto found = iterate_list(key_hash, k, kn, &stored_vn, prev_next, node); + if (found) { + if (vn <= stored_vn && !node->pair.null() && // try to set in place if fit + node->pair.copy_from(&vn, sizeof(size_t), layout::vlen_offset()) && + node->pair.copy_from(v, vn, layout::v_offset(kn))) { + LogAllocator::count_access(); + return true; + } else { + node = delete_node(prev_next, node); + break; + } + } + } + + auto new_node = create_node(key_hash, k, kn, v, vn); + if (!new_node) + return false; + *prev_next = new_node; + ul.unlock(); + LogAllocator::count_access(); + return true; +} + +template +bool SyncKV::set(const kv_types::Key &k, + const kv_types::CValue &v) { + return set(k.data, k.size, v.data, v.size); +} + +template +template +bool SyncKV::set(const K &k, const kv_types::CValue &v) { + return set(&k, sizeof(K), v.data, v.size); +} + +template +template +bool SyncKV::set(const K &k, const V &v) { + return set(&k, sizeof(K), &v, sizeof(V)); +} + +template +bool SyncKV::clear() { + for (int idx = 0; idx < NBuckets; idx++) { + auto &lock = locks_[idx]; + lock.lock(); + auto prev_next = &buckets_[idx]; + auto node = buckets_[idx]; + while (node) + node = delete_node(prev_next, node); + lock.unlock(); + } + return true; +} + +/** Ordered Set */ +/** Value Format: + * | NumEle (8B) | Len (8B) | Score (8B) | V1 (Len) | ... + * | Len (8B) | Score (8B) | Vn (Len) | + */ +namespace ordered_set { +struct OSetEle { + size_t len; + double score; + char data[]; // value + + static inline size_t total_size(size_t vn) { return sizeof(OSetEle) + vn; } + + inline void init(const void *v, size_t vn, double score) { + this->len = total_size(vn); + this->score = score; + std::memcpy(data, v, vn); + }; +}; +static_assert(sizeof(OSetEle) == sizeof(size_t) + sizeof(double), + "OSetEle is not corretly aligned!"); + +struct OSet { + size_t num_ele; + OSetEle data[]; // value array +}; +static_assert(sizeof(OSet) == sizeof(size_t), "OSet is not corretly aligned!"); + +static inline OSetEle *oset_iter(OSet *oset, size_t oset_len, OSetEle *pos) { + if (!pos || offset_ptrs(pos, oset) >= oset_len) + return nullptr; + size_t vn = pos->len; + auto next = ptr_offset(pos, vn); + return next; +} + +static inline OSet *oset_init(const void *v, size_t vn, double score) { + // allocate NumEle (size_t) and buffer for the single element + OSet *oset = reinterpret_cast( + malloc(sizeof(size_t) + OSetEle::total_size(vn))); + if (!oset) + return nullptr; + oset->num_ele = 1; + auto e = reinterpret_cast(oset->data); + e->init(v, vn, score); + return oset; +} + +static inline OSetEle *oset_search(OSet *oset, size_t oset_len, const void *v, + size_t vn) { + assert(oset->num_ele > 0); + + bool found = false; + OSetEle *iter = oset->data; + while (iter) { + if (iter->len == vn && std::memcmp(iter->data, v, vn) == 0) { + found = true; + break; + } + iter = oset_iter(oset, oset_len, iter); + } + if (found) + return iter; + return nullptr; +} + +/** Insert an element @v into ordered set. v must not in the set before. */ +static inline bool oset_insert(OSet *&oset_, size_t &oset_len_, const void *v, + size_t vn, double score) { + constexpr static bool DUP_CHECK = false; + OSet *oset = oset_; + size_t oset_len = oset_len_; + OSetEle *oset_end = reinterpret_cast(ptr_offset(oset, oset_len)); + size_t num_ele = *reinterpret_cast(oset); + assert(num_ele > 0); + auto data = ptr_offset(oset, sizeof(size_t)); + + OSetEle *iter = oset->data; + while (iter < oset_end) { + if (DUP_CHECK && iter->len == vn && std::memcmp(iter->data, v, vn) == 0) { + return false; + } + if (iter->score > score) { + break; + } + iter = oset_iter(oset, oset_len, iter); + } + size_t new_oset_len = oset_len + OSetEle::total_size(vn); + OSet *new_oset = reinterpret_cast(malloc(new_oset_len)); + if (!new_oset) + return false; + new_oset->num_ele = oset->num_ele + 1; + size_t offset = offset_ptrs(iter, oset->data); + size_t remaining_size = oset_len - sizeof(size_t) - offset; + if (offset) + std::memcpy(new_oset->data, oset->data, offset); + auto new_data = new_oset->data; + auto e = reinterpret_cast(ptr_offset(new_data, offset)); + e->init(v, vn, score); + if (remaining_size) + std::memcpy(ptr_offset(new_data, offset + OSetEle::total_size(vn)), + ptr_offset(data, offset), remaining_size); + + // update oset + free(oset); + oset_ = new_oset; + oset_len_ = new_oset_len; + return true; +} +} // namespace ordered_set + +/** insert a value into the ordered set by its timestamp */ +template +bool SyncKV::zadd(const void *k, size_t kn, + const void *v, size_t vn, double score, + UpdateType type) { + size_t oset_len = 0; + auto oset = reinterpret_cast( + get_(k, kn, nullptr, &oset_len, nullptr, false)); + if (!oset) { + if (type == UpdateType::EXIST) + return false; + else if (type == UpdateType::NOT_EXIST) { + oset = ordered_set::oset_init(v, vn, score); + if (!oset) + return false; + auto oset_len = sizeof(size_t) + ordered_set::OSetEle::total_size(vn); + return set(k, kn, oset, oset_len); + } else { + MIDAS_ABORT("Not implemented yet!"); + } + } + + auto *pos = ordered_set::oset_search(oset, oset_len, v, vn); + bool found = pos != nullptr; + + if (type == UpdateType::EXIST) { + if (!found) + goto failed; + pos->init(v, vn, score); // must have pos->len == vn + bool ret = set(k, kn, oset, oset_len); + return ret; + } else if (type == UpdateType::NOT_EXIST) { + if (found) + goto failed; + bool ret = ordered_set::oset_insert(oset, oset_len, v, vn, score); + if (!ret) + goto failed; + ret = set(k, kn, oset, oset_len); + free(oset); + return ret; + } + +failed: + if (oset) + free(oset); + return false; +} + +/** Fetch range [start, end] */ +template +bool SyncKV::zrange( + const void *key, size_t klen, int64_t start, int64_t end, + std::back_insert_iterator> bi) { + size_t oset_len; + auto oset = reinterpret_cast( + get_(key, klen, nullptr, &oset_len, nullptr, false)); + if (!oset) + return false; + if (start < 0 || end > oset->num_ele) + return false; + auto iter = oset->data; + for (size_t i = 0; i < start; i++) + iter = ordered_set::oset_iter(oset, oset_len, iter); + for (size_t i = start; i <= end; i++) { + auto len = iter->len - sizeof(ordered_set::OSetEle); + auto data = malloc(len); + std::memcpy(data, iter->data, len); + bi = kv_utils::make_value(data, len); + iter = ordered_set::oset_iter(oset, oset_len, iter); + } + return true; +} + +template +bool SyncKV::zrevrange( + const void *key, size_t klen, int64_t start, int64_t end, + std::back_insert_iterator> bi) { + std::vector values; + if (!zrange(key, klen, start, end, std::back_inserter(values))) + return false; + for (auto iter = values.rbegin(); iter != values.rend(); ++iter) + bi = *iter; + return true; +} + +/** Batched Interfaces */ +template +int SyncKV::bget(const std::vector &keys, + std::vector &values) { + kv_types::BatchPlug plug; + batch_stt(plug); + for (auto &k : keys) { + kv_types::Value v = bget_single(k, plug); + values.emplace_back(std::move(v)); + } + int succ = plug.hits; + assert(keys.size() == plug.batch_size); + batch_end(plug); + + return succ; +} + +template +template +int SyncKV::bget(const std::vector &keys, + std::vector &values) { + kv_types::BatchPlug plug; + batch_stt(plug); + for (const auto &k : keys) { + kv_types::Value v = bget_single(k, plug); + values.emplace_back(std::move(v)); + } + int succ = plug.hits; + assert(keys.size() == plug.batch_size); + batch_end(plug); + + return succ; +} + +template +template +int SyncKV::bget( + const std::vector &keys, std::vector> &values) { + kv_types::BatchPlug plug; + batch_stt(plug); + for (const auto &k : keys) { + std::unique_ptr v = bget_single(k, plug); + values.emplace_back(std::move(v)); + } + int succ = plug.hits; + assert(keys.size() == plug.batch_size); + batch_end(plug); + + return succ; +} + +template +int SyncKV::bset( + const std::vector &keys, + const std::vector &values) { + assert(keys.size() == values.size()); + int succ = 0; + auto nr_pairs = keys.size(); + for (int i = 0; i < nr_pairs; i++) + succ += set(keys[i], values[i]); + return succ; +} + +template +template +int SyncKV::bset( + const std::vector &keys, const std::vector &values) { + assert(keys.size() == values.size()); + int succ = 0; + auto nr_pairs = keys.size(); + for (int i = 0; i < nr_pairs; i++) + succ += set(keys[i], values[i]); + return succ; +} + +template +template +int SyncKV::bset(const std::vector &keys, + const std::vector &values) { + assert(keys.size() == values.size()); + int succ = 0; + auto nr_pairs = keys.size(); + for (int i = 0; i < nr_pairs; i++) + succ += set(keys[i], values[i]); + return succ; +} + +template +int SyncKV::bremove( + const std::vector &keys) { + int succ = 0; + for (auto &k : keys) + succ += remove(k); + return succ; +} + +template +template +int SyncKV::bremove(const std::vector &keys) { + int succ = 0; + for (auto &k : keys) + succ += remove(k); + return succ; +} + +template +void SyncKV::batch_stt(kv_types::BatchPlug &plug) { + plug.reset(); +} + +/* for batch operations, we count their cache stats only once here. */ +template +int SyncKV::batch_end(kv_types::BatchPlug &plug) { + int succ = plug.hits; + assert(plug.hits + plug.misses == plug.batch_size); + if (plug.hits == plug.batch_size) + pool_->inc_cache_hit(); + else { + if (plug.misses) + pool_->inc_cache_miss(); + if (plug.vhits == plug.misses) // count only if all missed items hit vcache + pool_->inc_cache_victim_hit(); + } + plug.reset(); + return succ; +} + +template +void *SyncKV::bget_single(const void *k, size_t kn, + size_t *vn, + kv_types::BatchPlug &plug) { + return get_(k, kn, nullptr, vn, &plug, false); +} + +template +kv_types::Value +SyncKV::bget_single(const kv_types::Key &key, + kv_types::BatchPlug &plug) { + size_t vn = 0; + auto v = get_(key.data, key.size, nullptr, &vn, &plug, false); + return kv_utils::make_value(v, vn); +} + +template +template +kv_types::Value +SyncKV::bget_single(const K &key, + kv_types::BatchPlug &plug) { + size_t vn = 0; + auto v = get_(&key, sizeof(K), nullptr, &vn, &plug, false); + return kv_utils::make_value(v, vn); +} + +template +template +std::unique_ptr +SyncKV::bget_single(const K &k, + kv_types::BatchPlug &plug) { + size_t vn = 0; + auto v = std::unique_ptr(get_(&k, sizeof(K), nullptr, &vn, &plug, false)); + if (vn != sizeof(V)) + return nullptr; + return std::move(v); +} + +/** Utility functions */ +/* if @v is given, then we will read the cache content into v directly; if v == + * nullptr, then this function will allocate a new buffer, reading the content + * into it, and return it. A nullptr ret value indicates a get failure. */ +template +void *SyncKV::get_(const void *k, size_t kn, void *v, + size_t *vn, kv_types::BatchPlug *plug, + bool construct) { + auto key_hash = hash_(k, kn); + auto bucket_idx = key_hash % NBuckets; + + auto ul = std::unique_lock(locks_[bucket_idx]); + + size_t stored_vn = 0; + void *stored_v = nullptr; + auto prev_next = &buckets_[bucket_idx]; + BNPtr node = buckets_[bucket_idx]; + bool found = false; + while (node) { + found = iterate_list(key_hash, k, kn, &stored_vn, prev_next, node); + if (found) + break; + } + if (!found) { + ul.unlock(); + goto failed; + } + assert(node); + stored_v = v ? v : malloc(stored_vn); + if (node->pair.null() || + !node->pair.copy_to(stored_v, stored_vn, layout::v_offset(kn))) { + if (node->pair.is_victim()) { + if (plug) + plug->vhits++; + else + pool_->inc_cache_victim_hit(&node->pair); + } + node = delete_node(prev_next, node); + ul.unlock(); + if (!v) { // stored_v is newly allocated + free(stored_v); + } + stored_v = nullptr; // reset stored_v + goto failed; + } + ul.unlock(); + if (vn) + *vn = stored_vn; + + if (plug) { + plug->hits++; + plug->batch_size++; + } else + pool_->inc_cache_hit(); + LogAllocator::count_access(); + return stored_v; + +failed: + assert(stored_v == nullptr); + if (plug) { + plug->misses++; + plug->batch_size++; + } else + pool_->inc_cache_miss(); + + // only re-construct for non-batched calls + if (kEnableConstruct && !plug && construct && pool_->get_construct_func()) { + ConstructArgs args = {k, kn, stored_v, stored_vn}; + ConstructPlug plug; + pool_->construct_stt(plug); + bool succ = pool_->construct(&args) == 0; + if (!succ) { // failed to re-construct + if (!v) // stored_v is newly allocated + free(stored_v); + return nullptr; + } + // successfully re-constructed + stored_v = args.value; + stored_vn = args.value_len; + set(k, kn, stored_v, stored_vn); + pool_->construct_add(stored_vn, plug); + pool_->construct_end(plug); + if (vn) + *vn = stored_vn; + return stored_v; + } + return nullptr; +} + +template +using BNPtr = typename SyncKV::BucketNode *; + +template +inline uint64_t SyncKV::hash_(const void *k, size_t kn) { + return kn == sizeof(uint64_t) + ? robin_hood::hash_int(*(reinterpret_cast(k))) + : robin_hood::hash_bytes(k, kn); +} + +template +inline BNPtr SyncKV::create_node( + uint64_t key_hash, const void *k, size_t kn, const void *v, size_t vn) { + auto *new_node = new BucketNode(); + if (!pool_->alloc_to(sizeof(size_t) * 2 + kn + vn, &new_node->pair) || + !new_node->pair.copy_from(&kn, sizeof(size_t), layout::klen_offset()) || + !new_node->pair.copy_from(&vn, sizeof(size_t), layout::vlen_offset()) || + !new_node->pair.copy_from(k, kn, layout::k_offset()) || + !new_node->pair.copy_from(v, vn, layout::v_offset(kn))) { + delete new_node; + return nullptr; + } + // assert(!new_node->pair.null()); + if (new_node->pair.null()) { + MIDAS_LOG(kError) << "new node KV pair is freed!"; + delete new_node; + return nullptr; + } + new_node->key_hash = key_hash; + new_node->next = nullptr; + return new_node; +} + +/** remove bucket_node from the list */ +// should always use as `node = delete_node()` when iterating the list +template +inline BNPtr +SyncKV::delete_node(BNPtr *prev_next, BNPtr node) { + assert(*prev_next == node); + if (!node) + return nullptr; + auto next = node->next; + + pool_->free(node->pair); + delete node; + + *prev_next = next; + return next; +} + +/** return: */ +template +inline bool +SyncKV::iterate_list(uint64_t key_hash, const void *k, + size_t kn, size_t *vn, + BNPtr *&prev_next, BNPtr &node) { + size_t stored_kn = 0; + void *stored_k = nullptr; + if (key_hash != node->key_hash) + goto notequal; + if (node->pair.null() || + !node->pair.copy_to(&stored_kn, sizeof(size_t), layout::klen_offset())) + goto faulted; + if (stored_kn != kn) + goto notequal; + stored_k = malloc(kn); + if (!node->pair.copy_to(stored_k, kn, layout::k_offset())) + goto faulted; + if (strncmp(reinterpret_cast(k), + reinterpret_cast(stored_k), kn) != 0) + goto notequal; + if (vn && !node->pair.copy_to(vn, sizeof(size_t), layout::vlen_offset())) + goto faulted; + if (stored_k) + free(stored_k); + return true; + +faulted: + if (stored_k) + free(stored_k); + if (node->pair.is_victim()) + pool_->inc_cache_victim_hit(&node->pair); + // prev remains the same when current node is deleted. + node = delete_node(prev_next, node); + return false; +notequal: + if (stored_k) + free(stored_k); + prev_next = &(node->next); + node = node->next; + return false; +} + +} // namespace midas diff --git a/inc/impl/sync_list.ipp b/inc/impl/sync_list.ipp new file mode 100644 index 0000000..c37671c --- /dev/null +++ b/inc/impl/sync_list.ipp @@ -0,0 +1,107 @@ +namespace midas { +template +SyncList::SyncList() : size_(0), list_(nullptr) { + pool_ = CachePool::global_cache_pool(); +} + +template +SyncList::SyncList(CachePool *pool) + : pool_(pool), size_(0), list_(nullptr) {} + +template +inline bool SyncList::empty() const noexcept { + return size_ == 0; +} + +template +inline bool SyncList::size() const noexcept { + return size_; +} + +template +inline std::unique_ptr SyncList::pop() { + Tp *stored_v = reinterpret_cast(::operator new(sizeof(Tp))); + if (pop(*stored_v)) + return std::unique_ptr(stored_v); + return nullptr; +} + +template +bool SyncList::pop(Tp &v) { + lock_.lock(); + if (!list_) { + lock_.unlock(); + pool_->inc_cache_miss(); + return false; + } + auto node = list_; + list_ = list_->next; + size_--; + lock_.unlock(); + if (node->obj.is_victim()) + pool_->inc_cache_victim_hit(&node->obj); + if (node->obj.null() || !node->obj.copy_to(&v, sizeof(Tp))) { + delete_node(node); + return false; + } + pool_->inc_cache_hit(); + LogAllocator::count_access(); + return true; +} + +template +bool SyncList::push(const Tp &v) { + ListNode *node = create_node(v); + if (!node) + return false; + lock_.lock(); + node->next = list_; + list_ = node; + size_++; + lock_.unlock(); + LogAllocator::count_access(); + return true; +} + +template +bool SyncList::clear() { + lock_.lock(); + while (list_) { + auto node = list_; + list_ = list_->next; + delete_node(node); + size_--; + } + lock_.unlock(); + return true; +} + +template +typename SyncList::ListNode * +SyncList::create_node(const Tp &v) { + auto allocator = pool_->get_allocator(); + ListNode *node = new ListNode(); + + if (!allocator->alloc_to(sizeof(Tp), &node->obj) || + !node->obj.copy_from(&v, sizeof(Tp))) { + delete node; + return nullptr; + } + // assert(!node->obj.null()); + if (node->obj.null()) { + MIDAS_LOG(kError) << "new list node is freed!"; + delete node; + return nullptr; + } + node->next = nullptr; + return node; +} + +template +void SyncList::delete_node(ListNode *node) { + // node->obj.free(); + pool_->free(node->obj); + delete node; +} + +} // namespace midas \ No newline at end of file diff --git a/inc/impl/time.ipp b/inc/impl/time.ipp new file mode 100644 index 0000000..9887a3d --- /dev/null +++ b/inc/impl/time.ipp @@ -0,0 +1,46 @@ +#pragma once + +namespace midas { +inline uint64_t Time::get_us_stt() { return cycles_to_us(rdtsc()); } + +inline uint64_t Time::get_us_end() { return cycles_to_us(rdtscp()); } + +inline uint64_t Time::get_cycles_stt() { return rdtsc(); } + +inline uint64_t Time::get_cycles_end() { return rdtscp(); } + +inline uint64_t Time::cycles_to_us(uint64_t cycles) noexcept { + return cycles / kCPUFreq; +} + +inline uint64_t Time::us_to_cycles(uint64_t us) noexcept { + return us * kCPUFreq; +} + +inline uint64_t Time::rdtsc() { + uint32_t a, d; + asm volatile("rdtsc" : "=a"(a), "=d"(d)); + return ((uint64_t)a) | (((uint64_t)d) << 32); +} + +inline uint64_t Time::rdtscp() { + uint32_t a, d, c; + asm volatile("rdtscp" : "=a"(a), "=d"(d), "=c"(c)); + return ((uint64_t)a) | (((uint64_t)d) << 32); +} + +namespace chrono_utils { +inline std::chrono::steady_clock::time_point now(); +inline double duration(const std::chrono::steady_clock::time_point &stt, + const std::chrono::steady_clock::time_point &end); + +inline std::chrono::steady_clock::time_point now() { + return std::chrono::steady_clock::now(); +} + +inline double duration(const std::chrono::steady_clock::time_point &stt, + const std::chrono::steady_clock::time_point &end) { + return std::chrono::duration(end - stt).count(); +} +} // namespace chrono_utils +} // namespace midas \ No newline at end of file diff --git a/inc/impl/transient_ptr.ipp b/inc/impl/transient_ptr.ipp new file mode 100644 index 0000000..5629c63 --- /dev/null +++ b/inc/impl/transient_ptr.ipp @@ -0,0 +1,149 @@ +#pragma once + +#include "logging.hpp" +#include "resilient_func.hpp" + +namespace midas { + +inline TransientPtr::TransientPtr() : ptr_(0) {} + +#ifdef BOUND_CHECK +inline TransientPtr::TransientPtr(uint64_t addr, size_t size) + : ptr_(addr), size_(size) {} +#else +inline TransientPtr::TransientPtr(uint64_t addr, size_t size) : ptr_(addr) {} +#endif // BOUND_CHECK + +inline bool TransientPtr::null() const noexcept { return ptr_ == 0; } + +inline bool TransientPtr::set(uint64_t addr, size_t size) { + // TODO: page-fault-aware logic + // if (!isValid(addr)) return false; + ptr_ = addr; +#ifdef BOUND_CHECK + size_ = size; +#endif // BOUND_CHECK + return true; +} + +inline bool TransientPtr::reset() noexcept { + ptr_ = 0; +#ifdef BOUND_CHECK + size_ = 0; +#endif // BOUND_CHECK + return true; +} + +inline size_t TransientPtr::size() const noexcept { return 0; } + +inline TransientPtr TransientPtr::slice(int64_t offset) const { +#ifdef BOUND_CHECK + return null() ? TransientPtr() : TransientPtr(ptr_ + offset, size_ - offset); +#else // !BOUND_CHECK + return null() ? TransientPtr() : TransientPtr(ptr_ + offset, 0); +#endif // BOUND_CHECK +} + +inline TransientPtr TransientPtr::slice(int64_t offset, size_t size) const { + return null() ? TransientPtr() : TransientPtr(ptr_ + offset, size); +} + +/** + * Atomic operations + */ +inline bool TransientPtr::cmpxchg(int64_t offset, uint64_t oldval, + uint64_t newval) { + auto addr = reinterpret_cast(ptr_ + offset); + return __sync_val_compare_and_swap(addr, oldval, newval); +} + +inline int64_t TransientPtr::atomic_add(int64_t offset, int64_t val) { + auto addr = reinterpret_cast(ptr_ + offset); + return __sync_fetch_and_add(addr, val); +} + +inline bool TransientPtr::copy_from(const void *src, size_t len, + int64_t offset) { + if (null()) + return false; +#ifdef BOUND_CHECK + if (offset + len > size_) { + MIDAS_LOG(kError); + return false; + } +#endif // BOUND_CHECK + bool ret = rmemcpy(reinterpret_cast(ptr_ + offset), src, len); + return ret; +} + +inline bool TransientPtr::copy_to(void *dst, size_t len, int64_t offset) { + if (null()) + return false; +#ifdef BOUND_CHECK + if (offset + len > size_) { + MIDAS_LOG(kError); + return false; + } +#endif // BOUND_CHECK + bool ret = rmemcpy(dst, reinterpret_cast(ptr_ + offset), len); + return ret; +} + +inline bool TransientPtr::copy_from(const TransientPtr &src, size_t len, + int64_t from_offset, int64_t to_offset) { + if (null()) + return false; +#ifdef BOUND_CHECK + if (from_offset + len > src.size_ || to_offset + len > this->size_) { + MIDAS_LOG(kError); + return false; + } +#endif // BOUND_CHECK + bool ret = rmemcpy(reinterpret_cast(this->ptr_ + to_offset), + reinterpret_cast(src.ptr_ + from_offset), len); + return ret; +} + +inline bool TransientPtr::copy_to(TransientPtr &dst, size_t len, + int64_t from_offset, int64_t to_offset) { + if (null()) + return false; +#ifdef BOUND_CHECK + if (from_offset + len > dst.size_ || to_offset + len > this->size_) { + MIDAS_LOG(kError); + return false; + } +#endif // BOUND_CHECK + bool ret = rmemcpy(reinterpret_cast(dst.ptr_ + to_offset), + reinterpret_cast(this->ptr_ + from_offset), len); + return ret; +} + +inline bool TransientPtr::assign_to_non_volatile(TransientPtr *dst) { + if (null()) + return false; + // TODO: page-fault-aware logic + *dst = *this; + return true; +} + +inline bool TransientPtr::assign_to_local_region(TransientPtr *dst) { + if (null()) + return false; + // TODO: page-fault-aware logic + *dst = *this; + return true; +} + +inline bool TransientPtr::assign_to_foreign_region(TransientPtr *dst) { + if (null()) + return false; + // TODO: page-fault-aware logic + *dst = *this; + return true; +} + +inline uint64_t TransientPtr::to_normal_address() const noexcept { + return ptr_; +} +} // namespace midas \ No newline at end of file diff --git a/inc/impl/victim_cache.ipp b/inc/impl/victim_cache.ipp new file mode 100644 index 0000000..00cfa5d --- /dev/null +++ b/inc/impl/victim_cache.ipp @@ -0,0 +1,100 @@ +#pragma once + +namespace midas { +inline VCEntry::VCEntry() : optr(nullptr), construct_args(nullptr), size(0) {} + +inline VCEntry::VCEntry(ObjectPtr *optr_, void *construct_args_) + : optr(optr_), construct_args(construct_args_), + size(optr_->data_size_in_segment()) {} + +inline void VCEntry::reset(ObjectPtr *optr_, void *construct_args_) noexcept { + optr = optr_; + size = optr_->data_size_in_segment(); + construct_args = construct_args_; +} + +inline VictimCache::VictimCache(int64_t size_limit, int64_t cnt_limit) + : size_limit_(size_limit), cnt_limit_(cnt_limit), size_(0), cnt_(0) {} + +inline VictimCache::~VictimCache() { + std::unique_lock ul(mtx_); + map_.clear(); + entries_.clear(); +} + +inline int64_t VictimCache::size() const noexcept { return size_; } +inline int64_t VictimCache::count() const noexcept { return cnt_; } + +inline bool VictimCache::get(ObjectPtr *optr_addr) noexcept { + if (!kEnableVictimCache) + return true; + + std::unique_lock ul(mtx_); + auto iter = map_.find(optr_addr); + if (iter != map_.cend()) { + entries_.splice(entries_.begin(), entries_, iter->second); + } + return true; +} + +inline bool VictimCache::put(ObjectPtr *optr_addr, + void *construct_args) noexcept { + if (!kEnableVictimCache) + return true; + + std::unique_lock ul(mtx_); + auto iter = map_.find(optr_addr); + if (iter != map_.cend()) { + auto entry = iter->second; + size_ -= entry->size; + MIDAS_LOG(kDebug) << "Replace an existing optr in victim cache " + << optr_addr << ", prev size " << entry->size + << ", curr size " << optr_addr->data_size_in_segment(); + entry->reset(optr_addr, construct_args); + size_ += entry->size; + optr_addr->set_victim(true); + entries_.splice(entries_.begin(), entries_, entry); + return true; + } + + auto entry = entries_.emplace_front(optr_addr, construct_args); + entry.optr->set_victim(true); + map_[optr_addr] = entries_.begin(); + cnt_++; + size_ += entry.size; + + while (size() > size_limit_ || count() > cnt_limit_) { + pop_back_locked(); + } + + return true; +} + +inline bool VictimCache::remove(ObjectPtr *optr_addr) noexcept { + if (!kEnableVictimCache) + return true; + + std::unique_lock ul(mtx_); + auto iter = map_.find(optr_addr); + if (iter == map_.cend()) + return false; + + auto entry = iter->second; + cnt_--; + size_ -= entry->size; + entry->optr->set_victim(false); + map_.erase(iter->first); + entries_.erase(entry); + return true; +} + +/** Util functions. All functions should be called with lock */ +inline void VictimCache::pop_back_locked() noexcept { + auto entry = entries_.back(); + cnt_--; + size_ -= entry.size; + entry.optr->set_victim(false); + map_.erase(entry.optr); + entries_.pop_back(); +} +} // namespace midas \ No newline at end of file diff --git a/inc/impl/zipf.ipp b/inc/impl/zipf.ipp new file mode 100644 index 0000000..cbaab1d --- /dev/null +++ b/inc/impl/zipf.ipp @@ -0,0 +1,47 @@ +#pragma once + +namespace midas { + +template +inline zipf_table_distribution::zipf_table_distribution( + const IntType n, const RealType q) + : n_(init(n, q)), q_(q), dist_(pdf_.begin(), pdf_.end()) {} + +template +inline void zipf_table_distribution::reset() {} + +template +inline IntType +zipf_table_distribution::operator()(std::mt19937 &rng) { + return dist_(rng) - 1; +} + +template +inline RealType zipf_table_distribution::s() const { + return q_; +} + +template +inline IntType +zipf_table_distribution::min() const { + return 0; +} + +template +inline IntType +zipf_table_distribution::max() const { + return n_ - 1; +} + +template +inline IntType +zipf_table_distribution::init(const IntType n, + const RealType q) { + pdf_.reserve(n + 1); + pdf_.emplace_back(0.0); + for (IntType i = 1; i <= n; i++) { + pdf_.emplace_back(std::pow((double)i, -q)); + } + return n; +} +} // namespace midas diff --git a/inc/log.hpp b/inc/log.hpp new file mode 100644 index 0000000..f0f0afe --- /dev/null +++ b/inc/log.hpp @@ -0,0 +1,130 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "object.hpp" +#include "transient_ptr.hpp" +#include "utils.hpp" + +namespace midas { + +class LogAllocator; + +class LogSegment { +public: + LogSegment(LogAllocator *owner, int64_t rid, uint64_t addr); + std::optional alloc_small(size_t size); + std::optional> + alloc_large(size_t size, const TransientPtr head_addr, + TransientPtr prev_addr); + bool free(ObjectPtr &ptr); + void seal() noexcept; + void destroy() noexcept; + + uint32_t size() const noexcept; + bool sealed() const noexcept; + bool destroyed() const noexcept; + bool full() const noexcept; + int32_t remaining_bytes() const noexcept; + float get_alive_ratio() const noexcept; + +private: + void init(uint64_t addr); + void iterate(size_t pos); + + void set_alive_bytes(int32_t alive_bytes) noexcept; + + static_assert(kRegionSize % kLogSegmentSize == 0, + "Region size must equal to segment size"); + + LogAllocator *owner_; + bool sealed_; + bool destroyed_; + int32_t alive_bytes_; + uint64_t start_addr_; + uint64_t pos_; + + int64_t region_id_; + + friend class Evacuator; + friend class LogAllocator; +}; + +class SegmentList { +public: + void push_back(std::shared_ptr segment); + std::shared_ptr pop_front(); + size_t size() noexcept; + bool empty() const noexcept; + +private: + std::mutex lock_; + std::list> segments_; +}; + +class CachePool; // defined in cache_manager.hpp + +class LogAllocator { +public: + LogAllocator(CachePool *pool); + std::optional alloc(size_t size); + bool alloc_to(size_t size, ObjectPtr *dst); + bool free(ObjectPtr &ptr); + + // accessing internal counters + static inline int64_t total_access_cnt() noexcept; + static inline void reset_access_cnt() noexcept; + static inline int64_t total_alive_cnt() noexcept; + static inline void reset_alive_cnt() noexcept; + static inline void count_access(); + static inline void count_alive(int val); + + static inline void thd_exit(); + + static inline std::shared_ptr + global_allocator_shared_ptr() noexcept; + static inline LogAllocator *global_allocator() noexcept; + +private: + std::optional alloc_(size_t size, bool overcommit); + std::optional alloc_large(size_t size, bool overcommit); + std::shared_ptr allocSegment(bool overcommit = false); + + /** Management */ + CachePool *pool_; + + /** Allocation */ + SegmentList segments_; + SegmentList stashed_pcabs_; + + /** Counters */ + static std::atomic_int64_t total_access_cnt_; + static std::atomic_int64_t total_alive_cnt_; + static void signal_scanner(); + + friend class Evacuator; + friend class LogSegment; + + /** Thread-local variables */ + // Per Core Allocation Buffer + // YIFAN: currently implemented as thread local buffers + static thread_local struct PCAB { + ~PCAB() { thd_exit(); } + std::shared_ptr local_seg; + + private: + void thd_exit(); + } pcab_; + static thread_local int32_t access_cnt_; + static thread_local int32_t alive_cnt_; +}; + +} // namespace midas + +#include "impl/log.ipp" diff --git a/inc/logging.hpp b/inc/logging.hpp new file mode 100644 index 0000000..8b9bf6a --- /dev/null +++ b/inc/logging.hpp @@ -0,0 +1,83 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace midas { + +enum LogVerbosity { + kError = 0, + kWarning = 1, + kInfo = 2, // information may interests users. + kDebug = 3, // information only interesting to developers. + kAll = 4, // log all information +}; + +constexpr LogVerbosity kGlobalVerbose = kInfo; +constexpr bool kLogFlagTime = false; +constexpr bool kLogFlagLoc = true; + +class Logger { +public: + Logger(const std::string &file, const std::string &func, int line, + LogVerbosity verbose, const std::string &verbose_str) noexcept + : _verbose(verbose) { + if (_verbose > kGlobalVerbose) + return; + if (kLogFlagTime) { + auto now = std::chrono::system_clock::to_time_t( + std::chrono::system_clock::now()); + std::cerr << "[" << std::put_time(std::localtime(&now), "%c") << "]"; + } + std::cerr << "[" << verbose_str << "]"; + if (kLogFlagLoc) { + std::cerr << "(" << file << ":" << std::dec << line << ", in " << func + << "()):"; + } + std::cerr << " "; + } + + template Logger &operator<<(const T &v) { + if (_verbose > kGlobalVerbose) + return *this; + std::cerr << v; + return *this; + } + + ~Logger() { + if (_verbose > kGlobalVerbose) + return; + std::cerr << std::endl; + } + +private: + LogVerbosity _verbose; +}; + +#define MIDAS_LOG(verbose) \ + Logger(__FILE__, __func__, __LINE__, (verbose), #verbose) + +#define MIDAS_LOG_PRINTF(verbose, ...) \ + do { \ + if ((verbose) <= kGlobalVerbose) { \ + fprintf(stderr, "[%s](%s:%d, in %s()): ", #verbose, __FILE__, __LINE__, \ + __func__); \ + fprintf(stderr, ##__VA_ARGS__); \ + } \ + } while (0) + +#define MIDAS_ABORT(...) \ + do { \ + fprintf(stderr, "[Abort](%s:%d, in %s()): ", __FILE__, __LINE__, \ + __func__); \ + fprintf(stderr, ##__VA_ARGS__); \ + fprintf(stderr, "\n"); \ + exit(-1); \ + } while (0) + +} // namespace midas \ No newline at end of file diff --git a/inc/obj_locker.hpp b/inc/obj_locker.hpp new file mode 100644 index 0000000..448b6c1 --- /dev/null +++ b/inc/obj_locker.hpp @@ -0,0 +1,37 @@ +#pragma once +#include +#include +#include +#include + +#include "robinhood.h" + +#include "transient_ptr.hpp" + +namespace midas { + +constexpr static uint32_t INV_LOCK_ID = -1; +using LockID = uint32_t; // need to be the same as in object.hpp + +class ObjLocker { +public: + std::optional try_lock(const TransientPtr &tptr); + LockID lock(const TransientPtr &tptr); + void unlock(LockID id); + + static inline ObjLocker *global_objlocker() noexcept; + +private: + constexpr static uint32_t kNumMaps = 1 << 16; + + std::optional _try_lock(uint64_t obj_addr); + LockID _lock(uint64_t obj_addr); + void _unlock(uint64_t obj_addr); + + uint64_t hash_val(uint64_t); + std::mutex mtxes_[kNumMaps]; +}; + +}; // namespace midas + +#include "impl/obj_locker.ipp" diff --git a/inc/object.hpp b/inc/object.hpp new file mode 100644 index 0000000..aa48c12 --- /dev/null +++ b/inc/object.hpp @@ -0,0 +1,263 @@ +#pragma once + +#include +#include +#include + +#include "transient_ptr.hpp" +#include "utils.hpp" + +namespace midas { + +constexpr static uint32_t kSmallObjThreshold = kSmallObjSizeUnit << 10; + +/** flags = 0, size = 0; rref = 0x1f1f1f1f1f1f */ +constexpr static uint64_t kInvalidHdr = 0x0'000'1f1f1f1f1f1f; +constexpr static uint64_t kInvalidFlags = 0; + +struct MetaObjectHdr { +public: + uint64_t flags; + + MetaObjectHdr(); + void set_invalid() noexcept; + bool is_valid() const noexcept; + + bool is_small_obj() const noexcept; + void set_small_obj() noexcept; + void set_large_obj() noexcept; + + bool is_present() const noexcept; + void set_present() noexcept; + void clr_present() noexcept; + bool is_accessed() const noexcept; + void inc_accessed() noexcept; + void dec_accessed() noexcept; + void clr_accessed() noexcept; + bool is_evacuate() const noexcept; + void set_evacuate() noexcept; + void clr_evacuate() noexcept; + bool is_mutate() const noexcept; + void set_mutate() noexcept; + void clr_mutate() noexcept; + bool is_continue() const noexcept; + void set_continue() noexcept; + void clr_continue() noexcept; + + static MetaObjectHdr *cast_from(void *hdr) noexcept; + +private: + constexpr static uint32_t kFlagShift = + sizeof(flags) * 8; // start from the highest bit + constexpr static decltype(flags) kPresentBit = kFlagShift - 1; + constexpr static decltype(flags) kSmallObjBit = kFlagShift - 2; + constexpr static decltype(flags) kEvacuateBit = kFlagShift - 3; + constexpr static decltype(flags) kMutateBit = kFlagShift - 4; + constexpr static decltype(flags) kAccessedBit = kFlagShift - 6; + constexpr static decltype(flags) kAccessedMask = (3ull << kAccessedBit); + + constexpr static decltype(flags) kContinueBit = kFlagShift - 7; +}; + +static_assert(sizeof(MetaObjectHdr) <= sizeof(uint64_t), + "GenericObjHdr is not correctly aligned!"); + +struct SmallObjectHdr { + // Format: + // I) |P(1b)|S(1b)|E(1b)|M(1b)|A(2b)| Obj Size(10b) | Reverse Ref (48b) | + // P: present bit. + // S: small obj bit -- the object is a small obj + // (size <= 8B * 2^10 == 8KB). + // E: evacuate bit -- the pointed data is being evacuated. + // M: mutate bit -- the pointed data is being mutated. + // A: accessed bits. + // Object Size: the size of the pointed object. + // Reverse reference: the only pointer referencing this object. +public: + SmallObjectHdr(); + + void init(uint32_t size_, uint64_t rref = 0) noexcept; + void free() noexcept; + + void set_invalid() noexcept; + bool is_valid() noexcept; + + void set_size(uint32_t size_) noexcept; + uint32_t get_size() const noexcept; + + void set_rref(uint64_t addr) noexcept; + uint64_t get_rref() const noexcept; + + void set_flags(uint8_t flags) noexcept; + uint8_t get_flags() const noexcept; + +private: +#pragma pack(push, 1) + uint64_t rref : 48; // reverse reference to the single to-be-updated + uint16_t size : 10; // object size / 8 (8 Bytes is the base unit) + uint8_t flags : 6; +#pragma pack(pop) +}; + +static_assert(sizeof(SmallObjectHdr) <= sizeof(uint64_t), + "SmallObjHdr is not correctly aligned!"); + +struct LargeObjectHdr { + // Format: + // I) | P(1b) | S(1b) | E(1b) | M(1b) | A(2b) |C(1b)|00(27b)| Obj Size(32b) | + // II) | 0...0(16b) | Reverse Reference (48b) | + // P: present bit. + // S: small obj bit -- the object is a small obj + // (size <= 8B * 2^10 == 8KB). + // E: evacuate bit -- the pointed data is being evacuated. + // M: mutate bit -- the pointed data is being mutated. + // A: accessed bits.. + // C: continue bit, meaning the objct is a large obj and + // the current segment is a continued segment. + // Object Size: the size of the pointed object. If object spans + // multiple segments, then size only represents the partial + // object size in the current segment. + // Reverse reference: the only pointer referencing this object. + // For the continued segments, rref stores the pointer to + // the head segment. +public: + LargeObjectHdr(); + + void init(uint32_t size_, bool is_head, TransientPtr head, + TransientPtr next) noexcept; + void free() noexcept; + + void set_invalid() noexcept; + bool is_valid() noexcept; + + void set_size(uint32_t size_) noexcept; + uint32_t get_size() const noexcept; + + void set_rref(uint64_t addr) noexcept; + uint64_t get_rref() const noexcept; + + void set_next(TransientPtr ptr) noexcept; + TransientPtr get_next() const noexcept; + + void set_flags(uint32_t flags) noexcept; + uint32_t get_flags() const noexcept; + + // Used by the continued parts of a large object only. + // head is stored in the `rref` field to reuse the space for cont'd parts. + void set_head(TransientPtr hdr) noexcept; + TransientPtr get_head() const noexcept; + +private: +#pragma pack(push, 1) + uint32_t size; + uint32_t flags; + uint64_t rref; // reverse reference + uint64_t next; // pointer to the next segment +#pragma pack(pop) +}; + +static_assert(sizeof(LargeObjectHdr) <= 24, + "LargeObjHdr is not correctly aligned!"); + +struct ObjectPtr { +public: + ObjectPtr(); + + /** Trinary return code : + * Form 1: {Fault, False, True} for get_*() / is_*() operations; + * Form 2: {Fault, Fail , Succ} for set_*() / upd_*() operations. + * We keep Fault as 0 to adapt to fault handler's design so that it can + * always return 0 to indicate a fault. + */ + enum class RetCode { + FaultLocal = 0, + FaultOther = 1, + False = 2, + True = 3, + Fail = False, + Succ = True + }; + + RetCode init_small(uint64_t stt_addr, size_t data_size); + RetCode init_large(uint64_t stt_addr, size_t data_size, bool is_head, + TransientPtr head, TransientPtr next); + RetCode init_from_soft(TransientPtr soft_addr); + RetCode free(bool locked = false) noexcept; + bool null() const noexcept; + + /** Header related */ + bool is_small_obj() const noexcept; + bool is_head_obj() const noexcept; + static size_t obj_size(size_t data_size) noexcept; + size_t obj_size() const noexcept; + size_t hdr_size() const noexcept; + size_t data_size_in_segment() const noexcept; + std::optional large_data_size(); + + bool contains(uint64_t addr) const noexcept; + + void set_victim(bool victim) noexcept; + bool is_victim() const noexcept; + + RetCode set_invalid() noexcept; + RetCode is_valid() noexcept; + + bool set_rref(uint64_t addr) noexcept; + bool set_rref(ObjectPtr *addr) noexcept; + ObjectPtr *get_rref() noexcept; + + RetCode upd_rref() noexcept; + + /** Data related */ + bool cmpxchg(int64_t offset, uint64_t oldval, uint64_t newval); + + bool copy_from(const void *src, size_t len, int64_t offset = 0); + bool copy_to(void *dst, size_t len, int64_t offset = 0); + + /** Evacuation related */ + RetCode move_from(ObjectPtr &src); + + /** Synchronization between Mutator and GC threads */ + using LockID = uint32_t; // need to be the same as in obj_locker.hpp + LockID lock(); + static void unlock(LockID id); + + /** Print & Debug */ + const std::string to_string() noexcept; + +private: + RetCode free_small() noexcept; + RetCode free_large() noexcept; + bool copy_from_small(const void *src, size_t len, int64_t offset); + bool copy_to_small(void *dst, size_t len, int64_t offset); + bool copy_from_large(const void *src, size_t len, int64_t offset); + bool copy_to_large(void *dst, size_t len, int64_t offset); + static RetCode iter_large(ObjectPtr &obj); + RetCode copy_from_large(const TransientPtr &src, size_t len, + int64_t from_offset, int64_t to_offset); + RetCode move_large(ObjectPtr &src) noexcept; + +#pragma pack(push, 1) + bool small_obj_ : 1; + bool head_obj_ : 1; + bool victim_: 1; + uint32_t size_ : 29; // Support up to 2^29 = 512MB. + uint32_t deref_cnt_; // not in use for now. + TransientPtr obj_; +#pragma pack(pop) + + template friend bool load_hdr(T &hdr, ObjectPtr &optr) noexcept; + template + friend bool store_hdr(const T &hdr, ObjectPtr &optr) noexcept; +}; + +static_assert(sizeof(ObjectPtr) <= 16, "ObjectPtr is not correctly aligned!"); + +template bool load_hdr(T &hdr, ObjectPtr &optr) noexcept; +template bool store_hdr(const T &hdr, ObjectPtr &optr) noexcept; + +template bool load_hdr(T &hdr, TransientPtr &tptr) noexcept; +template bool store_hdr(const T &hdr, TransientPtr &tptr) noexcept; +} // namespace midas + +#include "impl/object.ipp" \ No newline at end of file diff --git a/inc/perf.hpp b/inc/perf.hpp new file mode 100644 index 0000000..13f1b4a --- /dev/null +++ b/inc/perf.hpp @@ -0,0 +1,87 @@ +#pragma once + +#include +#include +#include +#include + +namespace midas { + +constexpr static uint64_t to_us = 1000 * 1000; // 1s = 10^6 us +constexpr static uint64_t kMissDDL = 100ul * to_us; // 100 s -> us + +struct PerfRequest { + virtual ~PerfRequest() = default; +}; + +struct PerfRequestWithTime { + uint64_t start_us; + std::unique_ptr req; +}; + +struct Trace { + uint64_t absl_start_us; + uint64_t start_us; + uint64_t duration_us; + Trace(); + Trace(uint64_t absl_start_us_, uint64_t start_us_, uint64_t duration_us_); +}; + +class PerfAdapter { +public: + virtual std::unique_ptr gen_req(int tid) = 0; + virtual bool serve_req(int tid, const PerfRequest *req) = 0; +}; + +// Closed-loop, possion arrival. +class Perf { +public: + Perf(PerfAdapter &adapter); + void reset(); + void run(uint32_t num_threads, double target_kops, uint64_t duration_us, + uint64_t warmup_us = 0, uint64_t miss_ddl_thresh_us = kMissDDL); + void run_phased(uint32_t num_threads, std::vector target_kops_vec, + std::vector &duration_us_vec, + std::vector &transition_us_vec, double warmup_kops, + uint64_t warmup_us = 0, + uint64_t miss_ddl_thresh_us = kMissDDL); + + uint64_t get_average_lat(); + uint64_t get_nth_lat(double nth); + std::vector get_timeseries_nth_lats(uint64_t interval_us, double nth); + double get_real_kops() const; + const std::vector &get_traces() const; + +private: + enum TraceFormat { kUnsorted, kSortedByDuration, kSortedByStart }; + + PerfAdapter &adapter_; + std::vector traces_; + TraceFormat trace_format_; + double real_kops_; + std::vector tputs_; + std::atomic_int32_t succ_ops; + friend class Test; + + uint64_t gen_reqs(std::vector *all_reqs, + uint32_t num_threads, double target_kops, + uint64_t duration_us, uint64_t start_us = 0); + uint64_t gen_phased_reqs(std::vector *all_reqs, + uint32_t num_threads, + std::vector &target_kops_vec, + std::vector &duration_us_vec, + std::vector &transition_us_vec, + uint64_t start_us = 0); + std::vector benchmark(std::vector *all_reqs, + uint32_t num_threads, + uint64_t miss_ddl_thresh_us); + + void report_tput(uint64_t duration_us); + void dump_tput(); + + constexpr static bool kEnableReporter = true; + constexpr static int kReportInterval = 5; // seconds + constexpr static int kReportBatch = 10; // update counter batch size + constexpr static int kTransSteps = 10; +}; +} // namespace midas \ No newline at end of file diff --git a/inc/qpair.hpp b/inc/qpair.hpp new file mode 100644 index 0000000..b3d65c2 --- /dev/null +++ b/inc/qpair.hpp @@ -0,0 +1,85 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "utils.hpp" +#include "shm_types.hpp" + +namespace midas { +constexpr static uint32_t kDaemonQDepth = 16384; +constexpr static uint32_t kClientQDepth = 1024; +constexpr static uint32_t kMaxMsgSize = sizeof(CtrlMsg); + +constexpr static char kNameCtrlQ[] = "daemon_ctrl_mq"; +constexpr static char kSQPrefix[] = "sendq-"; +constexpr static char kRQPrefix[] = "recvq-"; + +namespace utils { +const std::string get_sq_name(std::string qpname, bool create); +const std::string get_rq_name(std::string qpname, bool create); +const std::string get_ackq_name(std::string qpname, uint64_t id); +} // namespace utils + +class QSingle { +public: + using MsgQueue = boost::interprocess::message_queue; + // QSingle() = default; + QSingle(std::string name, bool create, uint32_t qdepth = kClientQDepth, + uint32_t msgsize = kMaxMsgSize) + : qdepth_(qdepth), msgsize_(msgsize), name_(name) { + init(create); + } + + inline int send(const void *buffer, size_t buffer_size); + inline int recv(void *buffer, size_t buffer_size); + inline int try_recv(void *buffer, size_t buffer_size); + inline int timed_recv(void *buffer, size_t buffer_size, int timeout); + + inline void init(bool create); + + /* destroy() will remove the shm file so it should be called only once + * manually by the owner of the QP, usually in its destructor. */ + inline void destroy(); + +private: + uint32_t qdepth_; + uint32_t msgsize_; + std::string name_; + std::shared_ptr q_; +}; + +class QPair { +public: + // QPair() = default; + QPair(std::string qpname, bool create, uint32_t qdepth = kClientQDepth, + uint32_t msgsize = kMaxMsgSize); + QPair(std::shared_ptr sq, std::shared_ptr rq) + : sq_(sq), rq_(rq) {} + + inline int send(const void *buffer, size_t buffer_size); + inline int recv(void *buffer, size_t buffer_size); + inline int try_recv(void *buffer, size_t buffer_size); + inline int timed_recv(void *buffer, size_t buffer_size, int timeout); + + inline QSingle &SendQ() const noexcept { return *sq_; } + inline QSingle &RecvQ() const noexcept { return *rq_; } + + /* destroy() will remove the shm file so it should be called only once + * manually by the owner of the QP, usually in its destructor. */ + inline void destroy(); + +private: + std::shared_ptr sq_; + std::shared_ptr rq_; +}; + + +} // namespace midas + +#include "impl/qpair.ipp" \ No newline at end of file diff --git a/inc/resilient_func.hpp b/inc/resilient_func.hpp new file mode 100644 index 0000000..23752b7 --- /dev/null +++ b/inc/resilient_func.hpp @@ -0,0 +1,26 @@ +#pragma once + +#include +#include + +#include "utils.hpp" + +namespace midas { + +class ResilientFunc { +public: + ResilientFunc(uint64_t stt_ip_, uint64_t end_ip_); + void init(void *func_addr); + bool contain(uint64_t fault_ip); + bool omitted_frame_pointer; + uint64_t fail_entry; + +private: + uint64_t stt_ip; + uint64_t end_ip; +}; + +DECL_RESILIENT_FUNC(bool, rmemcpy, void *dst, const void *src, size_t len); +} // namespace midas + +#include "impl/resilient_func.ipp" \ No newline at end of file diff --git a/inc/resource_manager.hpp b/inc/resource_manager.hpp new file mode 100644 index 0000000..c0adeac --- /dev/null +++ b/inc/resource_manager.hpp @@ -0,0 +1,154 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "qpair.hpp" +#include "shm_types.hpp" +#include "utils.hpp" + +namespace midas { + +using SharedMemObj = boost::interprocess::shared_memory_object; +using MsgQueue = boost::interprocess::message_queue; +using MappedRegion = boost::interprocess::mapped_region; + +class Region { +public: + Region(uint64_t pid, uint64_t region_id) noexcept; + ~Region() noexcept; + + void map() noexcept; + void unmap() noexcept; + void free() noexcept; + + inline bool mapped() const noexcept { return vrid_ == INVALID_VRID; }; + + inline friend bool operator<(const Region &lhs, const Region &rhs) noexcept { + return lhs.prid_ < rhs.prid_; + } + + inline void *Addr() const noexcept { + return reinterpret_cast( + vrid_ == INVALID_VRID ? INVALID_VRID + : kVolatileSttAddr + vrid_ * kRegionSize); + } + inline uint64_t ID() const noexcept { return prid_; } + inline int64_t Size() const noexcept { return size_; } + +private: + // generating unique name for the region shared memory file + uint64_t pid_; + uint64_t prid_; // physical memory region id + uint64_t vrid_; // mapped virtual memory region id + std::unique_ptr shm_region_; + int64_t size_; // int64_t to adapt to boost::interprocess::offset_t + + static std::atomic_int64_t + global_mapped_rid_; // never reuse virtual addresses + + constexpr static uint64_t INVALID_VRID = -1ul; +}; + +class CachePool; +class ResourceManager { +public: + ResourceManager(CachePool *cpool = nullptr, + const std::string &daemon_name = kNameCtrlQ) noexcept; + ~ResourceManager() noexcept; + + int64_t AllocRegion(bool overcommit = false) noexcept; + void FreeRegion(int64_t rid) noexcept; + void FreeRegions(size_t size = kRegionSize) noexcept; + inline VRange GetRegion(int64_t region_id) noexcept; + + void UpdateLimit(size_t size) noexcept; + void SetWeight(float weight) noexcept; + void SetLatCritical(bool value) noexcept; + + uint64_t NumRegionInUse() const noexcept; + uint64_t NumRegionLimit() const noexcept; + int64_t NumRegionAvail() const noexcept; + + /** trigger evacuation */ + bool reclaim_trigger() noexcept; + int64_t reclaim_target() noexcept; + int32_t reclaim_nr_thds() noexcept; + int32_t reclaim_headroom() noexcept; + + /** profiling stats */ + void prof_alloc_tput(); + // called by Evacuator to calculate reclaim tput + void prof_reclaim_stt(); + void prof_reclaim_end(int nr_thds, double dur_s); + + static std::shared_ptr global_manager_shared_ptr() noexcept; + static ResourceManager *global_manager() noexcept; + +private: + int connect(const std::string &daemon_name = kNameCtrlQ) noexcept; + int disconnect() noexcept; + size_t free_region(std::shared_ptr region, bool enforce) noexcept; + + void pressure_handler(); + void do_update_limit(CtrlMsg &msg); + void do_force_reclaim(CtrlMsg &msg); + void do_profile_stats(CtrlMsg &msg); + void do_disconnect(CtrlMsg &msg); + + bool reclaim(); + bool force_reclaim(); + + CachePool *cpool_; + + // inter-process comm + uint64_t id_; + std::mutex mtx_; + std::condition_variable cv_; + QPair txqp_; + QPair rxqp_; + + // regions + uint64_t region_limit_; + std::map> region_map_; + std::list> freelist_; + + std::atomic_int_fast64_t nr_pending_; + std::shared_ptr handler_thd_; + bool stop_; + + // stats + struct AllocTputStats { + // Updated by ResourceManager + std::atomic_int_fast64_t nr_alloced{0}; + std::atomic_int_fast64_t nr_evac_alloced{ + 0}; // temporarily allocated by the evacuator during evacuation + std::atomic_int_fast64_t nr_freed{0}; + uint64_t prev_time{0}; + int64_t prev_alloced{0}; + float alloc_tput{0}; + float reclaim_tput{0}; // per-evacuator-thread + float reclaim_dur{0}; // duration of each reclamation round + uint32_t headroom{0}; + // Updated by Evacuator + int64_t prev_evac_alloced{0}; + int64_t prev_freed{0}; + int32_t evac_cnt{0}; + float accum_evac_dur{0}; + float accum_nr_reclaimed{0}; + } stats_; +}; + +} // namespace midas + +#include "impl/resource_manager.ipp" \ No newline at end of file diff --git a/inc/robinhood.h b/inc/robinhood.h new file mode 100644 index 0000000..d38dd73 --- /dev/null +++ b/inc/robinhood.h @@ -0,0 +1,2544 @@ +// ______ _____ ______ _________ +// ______________ ___ /_ ___(_)_______ ___ /_ ______ ______ ______ / +// __ ___/_ __ \__ __ \__ / __ __ \ __ __ \_ __ \_ __ \_ __ / +// _ / / /_/ /_ /_/ /_ / _ / / / _ / / // /_/ // /_/ // /_/ / +// /_/ \____/ /_.___/ /_/ /_/ /_/ ________/_/ /_/ \____/ \____/ \__,_/ +// _/_____/ +// +// Fast & memory efficient hashtable based on robin hood hashing for C++11/14/17/20 +// https://github.com/martinus/robin-hood-hashing +// +// Licensed under the MIT License . +// SPDX-License-Identifier: MIT +// Copyright (c) 2018-2021 Martin Ankerl +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#ifndef ROBIN_HOOD_H_INCLUDED +#define ROBIN_HOOD_H_INCLUDED + +// see https://semver.org/ +#define ROBIN_HOOD_VERSION_MAJOR 3 // for incompatible API changes +#define ROBIN_HOOD_VERSION_MINOR 11 // for adding functionality in a backwards-compatible manner +#define ROBIN_HOOD_VERSION_PATCH 5 // for backwards-compatible bug fixes + +#include +#include +#include +#include +#include +#include // only to support hash of smart pointers +#include +#include +#include +#include +#if __cplusplus >= 201703L +# include +#endif + +// #define ROBIN_HOOD_LOG_ENABLED +#ifdef ROBIN_HOOD_LOG_ENABLED +# include +# define ROBIN_HOOD_LOG(...) \ + std::cout << __FUNCTION__ << "@" << __LINE__ << ": " << __VA_ARGS__ << std::endl; +#else +# define ROBIN_HOOD_LOG(x) +#endif + +// #define ROBIN_HOOD_TRACE_ENABLED +#ifdef ROBIN_HOOD_TRACE_ENABLED +# include +# define ROBIN_HOOD_TRACE(...) \ + std::cout << __FUNCTION__ << "@" << __LINE__ << ": " << __VA_ARGS__ << std::endl; +#else +# define ROBIN_HOOD_TRACE(x) +#endif + +// #define ROBIN_HOOD_COUNT_ENABLED +#ifdef ROBIN_HOOD_COUNT_ENABLED +# include +# define ROBIN_HOOD_COUNT(x) ++counts().x; +namespace robin_hood { +struct Counts { + uint64_t shiftUp{}; + uint64_t shiftDown{}; +}; +inline std::ostream& operator<<(std::ostream& os, Counts const& c) { + return os << c.shiftUp << " shiftUp" << std::endl << c.shiftDown << " shiftDown" << std::endl; +} + +static Counts& counts() { + static Counts counts{}; + return counts; +} +} // namespace robin_hood +#else +# define ROBIN_HOOD_COUNT(x) +#endif + +// all non-argument macros should use this facility. See +// https://www.fluentcpp.com/2019/05/28/better-macros-better-flags/ +#define ROBIN_HOOD(x) ROBIN_HOOD_PRIVATE_DEFINITION_##x() + +// mark unused members with this macro +#define ROBIN_HOOD_UNUSED(identifier) + +// bitness +#if SIZE_MAX == UINT32_MAX +# define ROBIN_HOOD_PRIVATE_DEFINITION_BITNESS() 32 +#elif SIZE_MAX == UINT64_MAX +# define ROBIN_HOOD_PRIVATE_DEFINITION_BITNESS() 64 +#else +# error Unsupported bitness +#endif + +// endianess +#ifdef _MSC_VER +# define ROBIN_HOOD_PRIVATE_DEFINITION_LITTLE_ENDIAN() 1 +# define ROBIN_HOOD_PRIVATE_DEFINITION_BIG_ENDIAN() 0 +#else +# define ROBIN_HOOD_PRIVATE_DEFINITION_LITTLE_ENDIAN() \ + (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) +# define ROBIN_HOOD_PRIVATE_DEFINITION_BIG_ENDIAN() (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +#endif + +// inline +#ifdef _MSC_VER +# define ROBIN_HOOD_PRIVATE_DEFINITION_NOINLINE() __declspec(noinline) +#else +# define ROBIN_HOOD_PRIVATE_DEFINITION_NOINLINE() __attribute__((noinline)) +#endif + +// exceptions +#if !defined(__cpp_exceptions) && !defined(__EXCEPTIONS) && !defined(_CPPUNWIND) +# define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_EXCEPTIONS() 0 +#else +# define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_EXCEPTIONS() 1 +#endif + +// count leading/trailing bits +#if !defined(ROBIN_HOOD_DISABLE_INTRINSICS) +# ifdef _MSC_VER +# if ROBIN_HOOD(BITNESS) == 32 +# define ROBIN_HOOD_PRIVATE_DEFINITION_BITSCANFORWARD() _BitScanForward +# else +# define ROBIN_HOOD_PRIVATE_DEFINITION_BITSCANFORWARD() _BitScanForward64 +# endif +# include +# pragma intrinsic(ROBIN_HOOD(BITSCANFORWARD)) +# define ROBIN_HOOD_COUNT_TRAILING_ZEROES(x) \ + [](size_t mask) noexcept -> int { \ + unsigned long index; \ + return ROBIN_HOOD(BITSCANFORWARD)(&index, mask) ? static_cast(index) \ + : ROBIN_HOOD(BITNESS); \ + }(x) +# else +# if ROBIN_HOOD(BITNESS) == 32 +# define ROBIN_HOOD_PRIVATE_DEFINITION_CTZ() __builtin_ctzl +# define ROBIN_HOOD_PRIVATE_DEFINITION_CLZ() __builtin_clzl +# else +# define ROBIN_HOOD_PRIVATE_DEFINITION_CTZ() __builtin_ctzll +# define ROBIN_HOOD_PRIVATE_DEFINITION_CLZ() __builtin_clzll +# endif +# define ROBIN_HOOD_COUNT_LEADING_ZEROES(x) ((x) ? ROBIN_HOOD(CLZ)(x) : ROBIN_HOOD(BITNESS)) +# define ROBIN_HOOD_COUNT_TRAILING_ZEROES(x) ((x) ? ROBIN_HOOD(CTZ)(x) : ROBIN_HOOD(BITNESS)) +# endif +#endif + +// fallthrough +#ifndef __has_cpp_attribute // For backwards compatibility +# define __has_cpp_attribute(x) 0 +#endif +#if __has_cpp_attribute(clang::fallthrough) +# define ROBIN_HOOD_PRIVATE_DEFINITION_FALLTHROUGH() [[clang::fallthrough]] +#elif __has_cpp_attribute(gnu::fallthrough) +# define ROBIN_HOOD_PRIVATE_DEFINITION_FALLTHROUGH() [[gnu::fallthrough]] +#else +# define ROBIN_HOOD_PRIVATE_DEFINITION_FALLTHROUGH() +#endif + +// likely/unlikely +#ifdef _MSC_VER +# define ROBIN_HOOD_LIKELY(condition) condition +# define ROBIN_HOOD_UNLIKELY(condition) condition +#else +# define ROBIN_HOOD_LIKELY(condition) __builtin_expect(condition, 1) +# define ROBIN_HOOD_UNLIKELY(condition) __builtin_expect(condition, 0) +#endif + +// detect if native wchar_t type is availiable in MSVC +#ifdef _MSC_VER +# ifdef _NATIVE_WCHAR_T_DEFINED +# define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_NATIVE_WCHART() 1 +# else +# define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_NATIVE_WCHART() 0 +# endif +#else +# define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_NATIVE_WCHART() 1 +#endif + +// detect if MSVC supports the pair(std::piecewise_construct_t,...) consructor being constexpr +#ifdef _MSC_VER +# if _MSC_VER <= 1900 +# define ROBIN_HOOD_PRIVATE_DEFINITION_BROKEN_CONSTEXPR() 1 +# else +# define ROBIN_HOOD_PRIVATE_DEFINITION_BROKEN_CONSTEXPR() 0 +# endif +#else +# define ROBIN_HOOD_PRIVATE_DEFINITION_BROKEN_CONSTEXPR() 0 +#endif + +// workaround missing "is_trivially_copyable" in g++ < 5.0 +// See https://stackoverflow.com/a/31798726/48181 +#if defined(__GNUC__) && __GNUC__ < 5 +# define ROBIN_HOOD_IS_TRIVIALLY_COPYABLE(...) __has_trivial_copy(__VA_ARGS__) +#else +# define ROBIN_HOOD_IS_TRIVIALLY_COPYABLE(...) std::is_trivially_copyable<__VA_ARGS__>::value +#endif + +// helpers for C++ versions, see https://gcc.gnu.org/onlinedocs/cpp/Standard-Predefined-Macros.html +#define ROBIN_HOOD_PRIVATE_DEFINITION_CXX() __cplusplus +#define ROBIN_HOOD_PRIVATE_DEFINITION_CXX98() 199711L +#define ROBIN_HOOD_PRIVATE_DEFINITION_CXX11() 201103L +#define ROBIN_HOOD_PRIVATE_DEFINITION_CXX14() 201402L +#define ROBIN_HOOD_PRIVATE_DEFINITION_CXX17() 201703L + +#if ROBIN_HOOD(CXX) >= ROBIN_HOOD(CXX17) +# define ROBIN_HOOD_PRIVATE_DEFINITION_NODISCARD() [[nodiscard]] +#else +# define ROBIN_HOOD_PRIVATE_DEFINITION_NODISCARD() +#endif + +namespace robin_hood { + +#if ROBIN_HOOD(CXX) >= ROBIN_HOOD(CXX14) +# define ROBIN_HOOD_STD std +#else + +// c++11 compatibility layer +namespace ROBIN_HOOD_STD { +template +struct alignment_of + : std::integral_constant::type)> {}; + +template +class integer_sequence { +public: + using value_type = T; + static_assert(std::is_integral::value, "not integral type"); + static constexpr std::size_t size() noexcept { + return sizeof...(Ints); + } +}; +template +using index_sequence = integer_sequence; + +namespace detail_ { +template +struct IntSeqImpl { + using TValue = T; + static_assert(std::is_integral::value, "not integral type"); + static_assert(Begin >= 0 && Begin < End, "unexpected argument (Begin<0 || Begin<=End)"); + + template + struct IntSeqCombiner; + + template + struct IntSeqCombiner, integer_sequence> { + using TResult = integer_sequence; + }; + + using TResult = + typename IntSeqCombiner::TResult, + typename IntSeqImpl::TResult>::TResult; +}; + +template +struct IntSeqImpl { + using TValue = T; + static_assert(std::is_integral::value, "not integral type"); + static_assert(Begin >= 0, "unexpected argument (Begin<0)"); + using TResult = integer_sequence; +}; + +template +struct IntSeqImpl { + using TValue = T; + static_assert(std::is_integral::value, "not integral type"); + static_assert(Begin >= 0, "unexpected argument (Begin<0)"); + using TResult = integer_sequence; +}; +} // namespace detail_ + +template +using make_integer_sequence = typename detail_::IntSeqImpl::TResult; + +template +using make_index_sequence = make_integer_sequence; + +template +using index_sequence_for = make_index_sequence; + +} // namespace ROBIN_HOOD_STD + +#endif + +namespace detail { + +// make sure we static_cast to the correct type for hash_int +#if ROBIN_HOOD(BITNESS) == 64 +using SizeT = uint64_t; +#else +using SizeT = uint32_t; +#endif + +template +T rotr(T x, unsigned k) { + return (x >> k) | (x << (8U * sizeof(T) - k)); +} + +// This cast gets rid of warnings like "cast from 'uint8_t*' {aka 'unsigned char*'} to +// 'uint64_t*' {aka 'long unsigned int*'} increases required alignment of target type". Use with +// care! +template +inline T reinterpret_cast_no_cast_align_warning(void* ptr) noexcept { + return reinterpret_cast(ptr); +} + +template +inline T reinterpret_cast_no_cast_align_warning(void const* ptr) noexcept { + return reinterpret_cast(ptr); +} + +// make sure this is not inlined as it is slow and dramatically enlarges code, thus making other +// inlinings more difficult. Throws are also generally the slow path. +template +[[noreturn]] ROBIN_HOOD(NOINLINE) +#if ROBIN_HOOD(HAS_EXCEPTIONS) + void doThrow(Args&&... args) { + // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-array-to-pointer-decay) + throw E(std::forward(args)...); +} +#else + void doThrow(Args&&... ROBIN_HOOD_UNUSED(args) /*unused*/) { + abort(); +} +#endif + +template +T* assertNotNull(T* t, Args&&... args) { + if (ROBIN_HOOD_UNLIKELY(nullptr == t)) { + doThrow(std::forward(args)...); + } + return t; +} + +template +inline T unaligned_load(void const* ptr) noexcept { + // using memcpy so we don't get into unaligned load problems. + // compiler should optimize this very well anyways. + T t; + std::memcpy(&t, ptr, sizeof(T)); + return t; +} + +// Allocates bulks of memory for objects of type T. This deallocates the memory in the destructor, +// and keeps a linked list of the allocated memory around. Overhead per allocation is the size of a +// pointer. +template +class BulkPoolAllocator { +public: + BulkPoolAllocator() noexcept = default; + + // does not copy anything, just creates a new allocator. + BulkPoolAllocator(const BulkPoolAllocator& ROBIN_HOOD_UNUSED(o) /*unused*/) noexcept + : mHead(nullptr) + , mListForFree(nullptr) {} + + BulkPoolAllocator(BulkPoolAllocator&& o) noexcept + : mHead(o.mHead) + , mListForFree(o.mListForFree) { + o.mListForFree = nullptr; + o.mHead = nullptr; + } + + BulkPoolAllocator& operator=(BulkPoolAllocator&& o) noexcept { + reset(); + mHead = o.mHead; + mListForFree = o.mListForFree; + o.mListForFree = nullptr; + o.mHead = nullptr; + return *this; + } + + BulkPoolAllocator& + // NOLINTNEXTLINE(bugprone-unhandled-self-assignment,cert-oop54-cpp) + operator=(const BulkPoolAllocator& ROBIN_HOOD_UNUSED(o) /*unused*/) noexcept { + // does not do anything + return *this; + } + + ~BulkPoolAllocator() noexcept { + reset(); + } + + // Deallocates all allocated memory. + void reset() noexcept { + while (mListForFree) { + T* tmp = *mListForFree; + ROBIN_HOOD_LOG("std::free") + std::free(mListForFree); + mListForFree = reinterpret_cast_no_cast_align_warning(tmp); + } + mHead = nullptr; + } + + // allocates, but does NOT initialize. Use in-place new constructor, e.g. + // T* obj = pool.allocate(); + // ::new (static_cast(obj)) T(); + T* allocate() { + T* tmp = mHead; + if (!tmp) { + tmp = performAllocation(); + } + + mHead = *reinterpret_cast_no_cast_align_warning(tmp); + return tmp; + } + + // does not actually deallocate but puts it in store. + // make sure you have already called the destructor! e.g. with + // obj->~T(); + // pool.deallocate(obj); + void deallocate(T* obj) noexcept { + *reinterpret_cast_no_cast_align_warning(obj) = mHead; + mHead = obj; + } + + // Adds an already allocated block of memory to the allocator. This allocator is from now on + // responsible for freeing the data (with free()). If the provided data is not large enough to + // make use of, it is immediately freed. Otherwise it is reused and freed in the destructor. + void addOrFree(void* ptr, const size_t numBytes) noexcept { + // calculate number of available elements in ptr + if (numBytes < ALIGNMENT + ALIGNED_SIZE) { + // not enough data for at least one element. Free and return. + ROBIN_HOOD_LOG("std::free") + std::free(ptr); + } else { + ROBIN_HOOD_LOG("add to buffer") + add(ptr, numBytes); + } + } + + void swap(BulkPoolAllocator& other) noexcept { + using std::swap; + swap(mHead, other.mHead); + swap(mListForFree, other.mListForFree); + } + +private: + // iterates the list of allocated memory to calculate how many to alloc next. + // Recalculating this each time saves us a size_t member. + // This ignores the fact that memory blocks might have been added manually with addOrFree. In + // practice, this should not matter much. + ROBIN_HOOD(NODISCARD) size_t calcNumElementsToAlloc() const noexcept { + auto tmp = mListForFree; + size_t numAllocs = MinNumAllocs; + + while (numAllocs * 2 <= MaxNumAllocs && tmp) { + auto x = reinterpret_cast(tmp); + tmp = *x; + numAllocs *= 2; + } + + return numAllocs; + } + + // WARNING: Underflow if numBytes < ALIGNMENT! This is guarded in addOrFree(). + void add(void* ptr, const size_t numBytes) noexcept { + const size_t numElements = (numBytes - ALIGNMENT) / ALIGNED_SIZE; + + auto data = reinterpret_cast(ptr); + + // link free list + auto x = reinterpret_cast(data); + *x = mListForFree; + mListForFree = data; + + // create linked list for newly allocated data + auto* const headT = + reinterpret_cast_no_cast_align_warning(reinterpret_cast(ptr) + ALIGNMENT); + + auto* const head = reinterpret_cast(headT); + + // Visual Studio compiler automatically unrolls this loop, which is pretty cool + for (size_t i = 0; i < numElements; ++i) { + *reinterpret_cast_no_cast_align_warning(head + i * ALIGNED_SIZE) = + head + (i + 1) * ALIGNED_SIZE; + } + + // last one points to 0 + *reinterpret_cast_no_cast_align_warning(head + (numElements - 1) * ALIGNED_SIZE) = + mHead; + mHead = headT; + } + + // Called when no memory is available (mHead == 0). + // Don't inline this slow path. + ROBIN_HOOD(NOINLINE) T* performAllocation() { + size_t const numElementsToAlloc = calcNumElementsToAlloc(); + + // alloc new memory: [prev |T, T, ... T] + size_t const bytes = ALIGNMENT + ALIGNED_SIZE * numElementsToAlloc; + ROBIN_HOOD_LOG("std::malloc " << bytes << " = " << ALIGNMENT << " + " << ALIGNED_SIZE + << " * " << numElementsToAlloc) + add(assertNotNull(std::malloc(bytes)), bytes); + return mHead; + } + + // enforce byte alignment of the T's +#if ROBIN_HOOD(CXX) >= ROBIN_HOOD(CXX14) + static constexpr size_t ALIGNMENT = + (std::max)(std::alignment_of::value, std::alignment_of::value); +#else + static const size_t ALIGNMENT = + (ROBIN_HOOD_STD::alignment_of::value > ROBIN_HOOD_STD::alignment_of::value) + ? ROBIN_HOOD_STD::alignment_of::value + : +ROBIN_HOOD_STD::alignment_of::value; // the + is for walkarround +#endif + + static constexpr size_t ALIGNED_SIZE = ((sizeof(T) - 1) / ALIGNMENT + 1) * ALIGNMENT; + + static_assert(MinNumAllocs >= 1, "MinNumAllocs"); + static_assert(MaxNumAllocs >= MinNumAllocs, "MaxNumAllocs"); + static_assert(ALIGNED_SIZE >= sizeof(T*), "ALIGNED_SIZE"); + static_assert(0 == (ALIGNED_SIZE % sizeof(T*)), "ALIGNED_SIZE mod"); + static_assert(ALIGNMENT >= sizeof(T*), "ALIGNMENT"); + + T* mHead{nullptr}; + T** mListForFree{nullptr}; +}; + +template +struct NodeAllocator; + +// dummy allocator that does nothing +template +struct NodeAllocator { + + // we are not using the data, so just free it. + void addOrFree(void* ptr, size_t ROBIN_HOOD_UNUSED(numBytes) /*unused*/) noexcept { + ROBIN_HOOD_LOG("std::free") + std::free(ptr); + } +}; + +template +struct NodeAllocator : public BulkPoolAllocator {}; + +// c++14 doesn't have is_nothrow_swappable, and clang++ 6.0.1 doesn't like it either, so I'm making +// my own here. +namespace swappable { +#if ROBIN_HOOD(CXX) < ROBIN_HOOD(CXX17) +using std::swap; +template +struct nothrow { + static const bool value = noexcept(swap(std::declval(), std::declval())); +}; +#else +template +struct nothrow { + static const bool value = std::is_nothrow_swappable::value; +}; +#endif +} // namespace swappable + +} // namespace detail + +struct is_transparent_tag {}; + +// A custom pair implementation is used in the map because std::pair is not is_trivially_copyable, +// which means it would not be allowed to be used in std::memcpy. This struct is copyable, which is +// also tested. +template +struct pair { + using first_type = T1; + using second_type = T2; + + template ::value && + std::is_default_constructible::value>::type> + constexpr pair() noexcept(noexcept(U1()) && noexcept(U2())) + : first() + , second() {} + + // pair constructors are explicit so we don't accidentally call this ctor when we don't have to. + explicit constexpr pair(std::pair const& o) noexcept( + noexcept(T1(std::declval())) && noexcept(T2(std::declval()))) + : first(o.first) + , second(o.second) {} + + // pair constructors are explicit so we don't accidentally call this ctor when we don't have to. + explicit constexpr pair(std::pair&& o) noexcept(noexcept( + T1(std::move(std::declval()))) && noexcept(T2(std::move(std::declval())))) + : first(std::move(o.first)) + , second(std::move(o.second)) {} + + constexpr pair(T1&& a, T2&& b) noexcept(noexcept( + T1(std::move(std::declval()))) && noexcept(T2(std::move(std::declval())))) + : first(std::move(a)) + , second(std::move(b)) {} + + template + constexpr pair(U1&& a, U2&& b) noexcept(noexcept(T1(std::forward( + std::declval()))) && noexcept(T2(std::forward(std::declval())))) + : first(std::forward(a)) + , second(std::forward(b)) {} + + template + // MSVC 2015 produces error "C2476: ‘constexpr’ constructor does not initialize all members" + // if this constructor is constexpr +#if !ROBIN_HOOD(BROKEN_CONSTEXPR) + constexpr +#endif + pair(std::piecewise_construct_t /*unused*/, std::tuple a, + std::tuple + b) noexcept(noexcept(pair(std::declval&>(), + std::declval&>(), + ROBIN_HOOD_STD::index_sequence_for(), + ROBIN_HOOD_STD::index_sequence_for()))) + : pair(a, b, ROBIN_HOOD_STD::index_sequence_for(), + ROBIN_HOOD_STD::index_sequence_for()) { + } + + // constructor called from the std::piecewise_construct_t ctor + template + pair(std::tuple& a, std::tuple& b, ROBIN_HOOD_STD::index_sequence /*unused*/, ROBIN_HOOD_STD::index_sequence /*unused*/) noexcept( + noexcept(T1(std::forward(std::get( + std::declval&>()))...)) && noexcept(T2(std:: + forward(std::get( + std::declval&>()))...))) + : first(std::forward(std::get(a))...) + , second(std::forward(std::get(b))...) { + // make visual studio compiler happy about warning about unused a & b. + // Visual studio's pair implementation disables warning 4100. + (void)a; + (void)b; + } + + void swap(pair& o) noexcept((detail::swappable::nothrow::value) && + (detail::swappable::nothrow::value)) { + using std::swap; + swap(first, o.first); + swap(second, o.second); + } + + T1 first; // NOLINT(misc-non-private-member-variables-in-classes) + T2 second; // NOLINT(misc-non-private-member-variables-in-classes) +}; + +template +inline void swap(pair& a, pair& b) noexcept( + noexcept(std::declval&>().swap(std::declval&>()))) { + a.swap(b); +} + +template +inline constexpr bool operator==(pair const& x, pair const& y) { + return (x.first == y.first) && (x.second == y.second); +} +template +inline constexpr bool operator!=(pair const& x, pair const& y) { + return !(x == y); +} +template +inline constexpr bool operator<(pair const& x, pair const& y) noexcept(noexcept( + std::declval() < std::declval()) && noexcept(std::declval() < + std::declval())) { + return x.first < y.first || (!(y.first < x.first) && x.second < y.second); +} +template +inline constexpr bool operator>(pair const& x, pair const& y) { + return y < x; +} +template +inline constexpr bool operator<=(pair const& x, pair const& y) { + return !(x > y); +} +template +inline constexpr bool operator>=(pair const& x, pair const& y) { + return !(x < y); +} + +inline size_t hash_bytes(void const* ptr, size_t len) noexcept { + static constexpr uint64_t m = UINT64_C(0xc6a4a7935bd1e995); + static constexpr uint64_t seed = UINT64_C(0xe17a1465); + static constexpr unsigned int r = 47; + + auto const* const data64 = static_cast(ptr); + uint64_t h = seed ^ (len * m); + + size_t const n_blocks = len / 8; + for (size_t i = 0; i < n_blocks; ++i) { + auto k = detail::unaligned_load(data64 + i); + + k *= m; + k ^= k >> r; + k *= m; + + h ^= k; + h *= m; + } + + auto const* const data8 = reinterpret_cast(data64 + n_blocks); + switch (len & 7U) { + case 7: + h ^= static_cast(data8[6]) << 48U; + ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH + case 6: + h ^= static_cast(data8[5]) << 40U; + ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH + case 5: + h ^= static_cast(data8[4]) << 32U; + ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH + case 4: + h ^= static_cast(data8[3]) << 24U; + ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH + case 3: + h ^= static_cast(data8[2]) << 16U; + ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH + case 2: + h ^= static_cast(data8[1]) << 8U; + ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH + case 1: + h ^= static_cast(data8[0]); + h *= m; + ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH + default: + break; + } + + h ^= h >> r; + + // not doing the final step here, because this will be done by keyToIdx anyways + // h *= m; + // h ^= h >> r; + return static_cast(h); +} + +inline size_t hash_int(uint64_t x) noexcept { + // tried lots of different hashes, let's stick with murmurhash3. It's simple, fast, well tested, + // and doesn't need any special 128bit operations. + x ^= x >> 33U; + x *= UINT64_C(0xff51afd7ed558ccd); + x ^= x >> 33U; + + // not doing the final step here, because this will be done by keyToIdx anyways + // x *= UINT64_C(0xc4ceb9fe1a85ec53); + // x ^= x >> 33U; + return static_cast(x); +} + +// A thin wrapper around std::hash, performing an additional simple mixing step of the result. +template +struct hash : public std::hash { + size_t operator()(T const& obj) const + noexcept(noexcept(std::declval>().operator()(std::declval()))) { + // call base hash + auto result = std::hash::operator()(obj); + // return mixed of that, to be save against identity has + return hash_int(static_cast(result)); + } +}; + +template +struct hash> { + size_t operator()(std::basic_string const& str) const noexcept { + return hash_bytes(str.data(), sizeof(CharT) * str.size()); + } +}; + +#if ROBIN_HOOD(CXX) >= ROBIN_HOOD(CXX17) +template +struct hash> { + size_t operator()(std::basic_string_view const& sv) const noexcept { + return hash_bytes(sv.data(), sizeof(CharT) * sv.size()); + } +}; +#endif + +template +struct hash { + size_t operator()(T* ptr) const noexcept { + return hash_int(reinterpret_cast(ptr)); + } +}; + +template +struct hash> { + size_t operator()(std::unique_ptr const& ptr) const noexcept { + return hash_int(reinterpret_cast(ptr.get())); + } +}; + +template +struct hash> { + size_t operator()(std::shared_ptr const& ptr) const noexcept { + return hash_int(reinterpret_cast(ptr.get())); + } +}; + +template +struct hash::value>::type> { + size_t operator()(Enum e) const noexcept { + using Underlying = typename std::underlying_type::type; + return hash{}(static_cast(e)); + } +}; + +#define ROBIN_HOOD_HASH_INT(T) \ + template <> \ + struct hash { \ + size_t operator()(T const& obj) const noexcept { \ + return hash_int(static_cast(obj)); \ + } \ + } + +#if defined(__GNUC__) && !defined(__clang__) +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wuseless-cast" +#endif +// see https://en.cppreference.com/w/cpp/utility/hash +ROBIN_HOOD_HASH_INT(bool); +ROBIN_HOOD_HASH_INT(char); +ROBIN_HOOD_HASH_INT(signed char); +ROBIN_HOOD_HASH_INT(unsigned char); +ROBIN_HOOD_HASH_INT(char16_t); +ROBIN_HOOD_HASH_INT(char32_t); +#if ROBIN_HOOD(HAS_NATIVE_WCHART) +ROBIN_HOOD_HASH_INT(wchar_t); +#endif +ROBIN_HOOD_HASH_INT(short); +ROBIN_HOOD_HASH_INT(unsigned short); +ROBIN_HOOD_HASH_INT(int); +ROBIN_HOOD_HASH_INT(unsigned int); +ROBIN_HOOD_HASH_INT(long); +ROBIN_HOOD_HASH_INT(long long); +ROBIN_HOOD_HASH_INT(unsigned long); +ROBIN_HOOD_HASH_INT(unsigned long long); +#if defined(__GNUC__) && !defined(__clang__) +# pragma GCC diagnostic pop +#endif +namespace detail { + +template +struct void_type { + using type = void; +}; + +template +struct has_is_transparent : public std::false_type {}; + +template +struct has_is_transparent::type> + : public std::true_type {}; + +// using wrapper classes for hash and key_equal prevents the diamond problem when the same type +// is used. see https://stackoverflow.com/a/28771920/48181 +template +struct WrapHash : public T { + WrapHash() = default; + explicit WrapHash(T const& o) noexcept(noexcept(T(std::declval()))) + : T(o) {} +}; + +template +struct WrapKeyEqual : public T { + WrapKeyEqual() = default; + explicit WrapKeyEqual(T const& o) noexcept(noexcept(T(std::declval()))) + : T(o) {} +}; + +// A highly optimized hashmap implementation, using the Robin Hood algorithm. +// +// In most cases, this map should be usable as a drop-in replacement for std::unordered_map, but +// be about 2x faster in most cases and require much less allocations. +// +// This implementation uses the following memory layout: +// +// [Node, Node, ... Node | info, info, ... infoSentinel ] +// +// * Node: either a DataNode that directly has the std::pair as member, +// or a DataNode with a pointer to std::pair. Which DataNode representation to use +// depends on how fast the swap() operation is. Heuristically, this is automatically choosen +// based on sizeof(). there are always 2^n Nodes. +// +// * info: Each Node in the map has a corresponding info byte, so there are 2^n info bytes. +// Each byte is initialized to 0, meaning the corresponding Node is empty. Set to 1 means the +// corresponding node contains data. Set to 2 means the corresponding Node is filled, but it +// actually belongs to the previous position and was pushed out because that place is already +// taken. +// +// * infoSentinel: Sentinel byte set to 1, so that iterator's ++ can stop at end() without the +// need for a idx variable. +// +// According to STL, order of templates has effect on throughput. That's why I've moved the +// boolean to the front. +// https://www.reddit.com/r/cpp/comments/ahp6iu/compile_time_binary_size_reductions_and_cs_future/eeguck4/ +template +class Table + : public WrapHash, + public WrapKeyEqual, + detail::NodeAllocator< + typename std::conditional< + std::is_void::value, Key, + robin_hood::pair::type, T>>::type, + 4, 16384, IsFlat> { +public: + static constexpr bool is_flat = IsFlat; + static constexpr bool is_map = !std::is_void::value; + static constexpr bool is_set = !is_map; + static constexpr bool is_transparent = + has_is_transparent::value && has_is_transparent::value; + + using key_type = Key; + using mapped_type = T; + using value_type = typename std::conditional< + is_set, Key, + robin_hood::pair::type, T>>::type; + using size_type = size_t; + using hasher = Hash; + using key_equal = KeyEqual; + using Self = Table; + +private: + static_assert(MaxLoadFactor100 > 10 && MaxLoadFactor100 < 100, + "MaxLoadFactor100 needs to be >10 && < 100"); + + using WHash = WrapHash; + using WKeyEqual = WrapKeyEqual; + + // configuration defaults + + // make sure we have 8 elements, needed to quickly rehash mInfo + static constexpr size_t InitialNumElements = sizeof(uint64_t); + static constexpr uint32_t InitialInfoNumBits = 5; + static constexpr uint8_t InitialInfoInc = 1U << InitialInfoNumBits; + static constexpr size_t InfoMask = InitialInfoInc - 1U; + static constexpr uint8_t InitialInfoHashShift = 0; + using DataPool = detail::NodeAllocator; + + // type needs to be wider than uint8_t. + using InfoType = uint32_t; + + // DataNode //////////////////////////////////////////////////////// + + // Primary template for the data node. We have special implementations for small and big + // objects. For large objects it is assumed that swap() is fairly slow, so we allocate these + // on the heap so swap merely swaps a pointer. + template + class DataNode {}; + + // Small: just allocate on the stack. + template + class DataNode final { + public: + template + explicit DataNode(M& ROBIN_HOOD_UNUSED(map) /*unused*/, Args&&... args) noexcept( + noexcept(value_type(std::forward(args)...))) + : mData(std::forward(args)...) {} + + DataNode(M& ROBIN_HOOD_UNUSED(map) /*unused*/, DataNode&& n) noexcept( + std::is_nothrow_move_constructible::value) + : mData(std::move(n.mData)) {} + + // doesn't do anything + void destroy(M& ROBIN_HOOD_UNUSED(map) /*unused*/) noexcept {} + void destroyDoNotDeallocate() noexcept {} + + value_type const* operator->() const noexcept { + return &mData; + } + value_type* operator->() noexcept { + return &mData; + } + + const value_type& operator*() const noexcept { + return mData; + } + + value_type& operator*() noexcept { + return mData; + } + + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::type getFirst() noexcept { + return mData.first; + } + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::type getFirst() noexcept { + return mData; + } + + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::type + getFirst() const noexcept { + return mData.first; + } + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::type getFirst() const noexcept { + return mData; + } + + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::type getSecond() noexcept { + return mData.second; + } + + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::type getSecond() const noexcept { + return mData.second; + } + + void swap(DataNode& o) noexcept( + noexcept(std::declval().swap(std::declval()))) { + mData.swap(o.mData); + } + + private: + value_type mData; + }; + + // big object: allocate on heap. + template + class DataNode { + public: + template + explicit DataNode(M& map, Args&&... args) + : mData(map.allocate()) { + ::new (static_cast(mData)) value_type(std::forward(args)...); + } + + DataNode(M& ROBIN_HOOD_UNUSED(map) /*unused*/, DataNode&& n) noexcept + : mData(std::move(n.mData)) {} + + void destroy(M& map) noexcept { + // don't deallocate, just put it into list of datapool. + mData->~value_type(); + map.deallocate(mData); + } + + void destroyDoNotDeallocate() noexcept { + mData->~value_type(); + } + + value_type const* operator->() const noexcept { + return mData; + } + + value_type* operator->() noexcept { + return mData; + } + + const value_type& operator*() const { + return *mData; + } + + value_type& operator*() { + return *mData; + } + + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::type getFirst() noexcept { + return mData->first; + } + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::type getFirst() noexcept { + return *mData; + } + + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::type + getFirst() const noexcept { + return mData->first; + } + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::type getFirst() const noexcept { + return *mData; + } + + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::type getSecond() noexcept { + return mData->second; + } + + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::type getSecond() const noexcept { + return mData->second; + } + + void swap(DataNode& o) noexcept { + using std::swap; + swap(mData, o.mData); + } + + private: + value_type* mData; + }; + + using Node = DataNode; + + // helpers for insertKeyPrepareEmptySpot: extract first entry (only const required) + ROBIN_HOOD(NODISCARD) key_type const& getFirstConst(Node const& n) const noexcept { + return n.getFirst(); + } + + // in case we have void mapped_type, we are not using a pair, thus we just route k through. + // No need to disable this because it's just not used if not applicable. + ROBIN_HOOD(NODISCARD) key_type const& getFirstConst(key_type const& k) const noexcept { + return k; + } + + // in case we have non-void mapped_type, we have a standard robin_hood::pair + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::value, key_type const&>::type + getFirstConst(value_type const& vt) const noexcept { + return vt.first; + } + + // Cloner ////////////////////////////////////////////////////////// + + template + struct Cloner; + + // fast path: Just copy data, without allocating anything. + template + struct Cloner { + void operator()(M const& source, M& target) const { + auto const* const src = reinterpret_cast(source.mKeyVals); + auto* tgt = reinterpret_cast(target.mKeyVals); + auto const numElementsWithBuffer = target.calcNumElementsWithBuffer(target.mMask + 1); + std::copy(src, src + target.calcNumBytesTotal(numElementsWithBuffer), tgt); + } + }; + + template + struct Cloner { + void operator()(M const& s, M& t) const { + auto const numElementsWithBuffer = t.calcNumElementsWithBuffer(t.mMask + 1); + std::copy(s.mInfo, s.mInfo + t.calcNumBytesInfo(numElementsWithBuffer), t.mInfo); + + for (size_t i = 0; i < numElementsWithBuffer; ++i) { + if (t.mInfo[i]) { + ::new (static_cast(t.mKeyVals + i)) Node(t, *s.mKeyVals[i]); + } + } + } + }; + + // Destroyer /////////////////////////////////////////////////////// + + template + struct Destroyer {}; + + template + struct Destroyer { + void nodes(M& m) const noexcept { + m.mNumElements = 0; + } + + void nodesDoNotDeallocate(M& m) const noexcept { + m.mNumElements = 0; + } + }; + + template + struct Destroyer { + void nodes(M& m) const noexcept { + m.mNumElements = 0; + // clear also resets mInfo to 0, that's sometimes not necessary. + auto const numElementsWithBuffer = m.calcNumElementsWithBuffer(m.mMask + 1); + + for (size_t idx = 0; idx < numElementsWithBuffer; ++idx) { + if (0 != m.mInfo[idx]) { + Node& n = m.mKeyVals[idx]; + n.destroy(m); + n.~Node(); + } + } + } + + void nodesDoNotDeallocate(M& m) const noexcept { + m.mNumElements = 0; + // clear also resets mInfo to 0, that's sometimes not necessary. + auto const numElementsWithBuffer = m.calcNumElementsWithBuffer(m.mMask + 1); + for (size_t idx = 0; idx < numElementsWithBuffer; ++idx) { + if (0 != m.mInfo[idx]) { + Node& n = m.mKeyVals[idx]; + n.destroyDoNotDeallocate(); + n.~Node(); + } + } + } + }; + + // Iter //////////////////////////////////////////////////////////// + + struct fast_forward_tag {}; + + // generic iterator for both const_iterator and iterator. + template + // NOLINTNEXTLINE(hicpp-special-member-functions,cppcoreguidelines-special-member-functions) + class Iter { + private: + using NodePtr = typename std::conditional::type; + + public: + using difference_type = std::ptrdiff_t; + using value_type = typename Self::value_type; + using reference = typename std::conditional::type; + using pointer = typename std::conditional::type; + using iterator_category = std::forward_iterator_tag; + + // default constructed iterator can be compared to itself, but WON'T return true when + // compared to end(). + Iter() = default; + + // Rule of zero: nothing specified. The conversion constructor is only enabled for + // iterator to const_iterator, so it doesn't accidentally work as a copy ctor. + + // Conversion constructor from iterator to const_iterator. + template ::type> + // NOLINTNEXTLINE(hicpp-explicit-conversions) + Iter(Iter const& other) noexcept + : mKeyVals(other.mKeyVals) + , mInfo(other.mInfo) {} + + Iter(NodePtr valPtr, uint8_t const* infoPtr) noexcept + : mKeyVals(valPtr) + , mInfo(infoPtr) {} + + Iter(NodePtr valPtr, uint8_t const* infoPtr, + fast_forward_tag ROBIN_HOOD_UNUSED(tag) /*unused*/) noexcept + : mKeyVals(valPtr) + , mInfo(infoPtr) { + fastForward(); + } + + template ::type> + Iter& operator=(Iter const& other) noexcept { + mKeyVals = other.mKeyVals; + mInfo = other.mInfo; + return *this; + } + + // prefix increment. Undefined behavior if we are at end()! + Iter& operator++() noexcept { + mInfo++; + mKeyVals++; + fastForward(); + return *this; + } + + Iter operator++(int) noexcept { + Iter tmp = *this; + ++(*this); + return tmp; + } + + reference operator*() const { + return **mKeyVals; + } + + pointer operator->() const { + return &**mKeyVals; + } + + template + bool operator==(Iter const& o) const noexcept { + return mKeyVals == o.mKeyVals; + } + + template + bool operator!=(Iter const& o) const noexcept { + return mKeyVals != o.mKeyVals; + } + + private: + // fast forward to the next non-free info byte + // I've tried a few variants that don't depend on intrinsics, but unfortunately they are + // quite a bit slower than this one. So I've reverted that change again. See map_benchmark. + void fastForward() noexcept { + size_t n = 0; + while (0U == (n = detail::unaligned_load(mInfo))) { + mInfo += sizeof(size_t); + mKeyVals += sizeof(size_t); + } +#if defined(ROBIN_HOOD_DISABLE_INTRINSICS) + // we know for certain that within the next 8 bytes we'll find a non-zero one. + if (ROBIN_HOOD_UNLIKELY(0U == detail::unaligned_load(mInfo))) { + mInfo += 4; + mKeyVals += 4; + } + if (ROBIN_HOOD_UNLIKELY(0U == detail::unaligned_load(mInfo))) { + mInfo += 2; + mKeyVals += 2; + } + if (ROBIN_HOOD_UNLIKELY(0U == *mInfo)) { + mInfo += 1; + mKeyVals += 1; + } +#else +# if ROBIN_HOOD(LITTLE_ENDIAN) + auto inc = ROBIN_HOOD_COUNT_TRAILING_ZEROES(n) / 8; +# else + auto inc = ROBIN_HOOD_COUNT_LEADING_ZEROES(n) / 8; +# endif + mInfo += inc; + mKeyVals += inc; +#endif + } + + friend class Table; + NodePtr mKeyVals{nullptr}; + uint8_t const* mInfo{nullptr}; + }; + + //////////////////////////////////////////////////////////////////// + + // highly performance relevant code. + // Lower bits are used for indexing into the array (2^n size) + // The upper 1-5 bits need to be a reasonable good hash, to save comparisons. + template + void keyToIdx(HashKey&& key, size_t* idx, InfoType* info) const { + // In addition to whatever hash is used, add another mul & shift so we get better hashing. + // This serves as a bad hash prevention, if the given data is + // badly mixed. + auto h = static_cast(WHash::operator()(key)); + + h *= mHashMultiplier; + h ^= h >> 33U; + + // the lower InitialInfoNumBits are reserved for info. + *info = mInfoInc + static_cast((h & InfoMask) >> mInfoHashShift); + *idx = (static_cast(h) >> InitialInfoNumBits) & mMask; + } + + // forwards the index by one, wrapping around at the end + void next(InfoType* info, size_t* idx) const noexcept { + *idx = *idx + 1; + *info += mInfoInc; + } + + void nextWhileLess(InfoType* info, size_t* idx) const noexcept { + // unrolling this by hand did not bring any speedups. + while (*info < mInfo[*idx]) { + next(info, idx); + } + } + + // Shift everything up by one element. Tries to move stuff around. + void + shiftUp(size_t startIdx, + size_t const insertion_idx) noexcept(std::is_nothrow_move_assignable::value) { + auto idx = startIdx; + ::new (static_cast(mKeyVals + idx)) Node(std::move(mKeyVals[idx - 1])); + while (--idx != insertion_idx) { + mKeyVals[idx] = std::move(mKeyVals[idx - 1]); + } + + idx = startIdx; + while (idx != insertion_idx) { + ROBIN_HOOD_COUNT(shiftUp) + mInfo[idx] = static_cast(mInfo[idx - 1] + mInfoInc); + if (ROBIN_HOOD_UNLIKELY(mInfo[idx] + mInfoInc > 0xFF)) { + mMaxNumElementsAllowed = 0; + } + --idx; + } + } + + void shiftDown(size_t idx) noexcept(std::is_nothrow_move_assignable::value) { + // until we find one that is either empty or has zero offset. + // TODO(martinus) we don't need to move everything, just the last one for the same + // bucket. + mKeyVals[idx].destroy(*this); + + // until we find one that is either empty or has zero offset. + while (mInfo[idx + 1] >= 2 * mInfoInc) { + ROBIN_HOOD_COUNT(shiftDown) + mInfo[idx] = static_cast(mInfo[idx + 1] - mInfoInc); + mKeyVals[idx] = std::move(mKeyVals[idx + 1]); + ++idx; + } + + mInfo[idx] = 0; + // don't destroy, we've moved it + // mKeyVals[idx].destroy(*this); + mKeyVals[idx].~Node(); + } + + // copy of find(), except that it returns iterator instead of const_iterator. + template + ROBIN_HOOD(NODISCARD) + size_t findIdx(Other const& key) const { + size_t idx{}; + InfoType info{}; + keyToIdx(key, &idx, &info); + + do { + // unrolling this twice gives a bit of a speedup. More unrolling did not help. + if (info == mInfo[idx] && + ROBIN_HOOD_LIKELY(WKeyEqual::operator()(key, mKeyVals[idx].getFirst()))) { + return idx; + } + next(&info, &idx); + if (info == mInfo[idx] && + ROBIN_HOOD_LIKELY(WKeyEqual::operator()(key, mKeyVals[idx].getFirst()))) { + return idx; + } + next(&info, &idx); + } while (info <= mInfo[idx]); + + // nothing found! + return mMask == 0 ? 0 + : static_cast(std::distance( + mKeyVals, reinterpret_cast_no_cast_align_warning(mInfo))); + } + + void cloneData(const Table& o) { + Cloner()(o, *this); + } + + // inserts a keyval that is guaranteed to be new, e.g. when the hashmap is resized. + // @return True on success, false if something went wrong + void insert_move(Node&& keyval) { + // we don't retry, fail if overflowing + // don't need to check max num elements + if (0 == mMaxNumElementsAllowed && !try_increase_info()) { + throwOverflowError(); + } + + size_t idx{}; + InfoType info{}; + keyToIdx(keyval.getFirst(), &idx, &info); + + // skip forward. Use <= because we are certain that the element is not there. + while (info <= mInfo[idx]) { + idx = idx + 1; + info += mInfoInc; + } + + // key not found, so we are now exactly where we want to insert it. + auto const insertion_idx = idx; + auto const insertion_info = static_cast(info); + if (ROBIN_HOOD_UNLIKELY(insertion_info + mInfoInc > 0xFF)) { + mMaxNumElementsAllowed = 0; + } + + // find an empty spot + while (0 != mInfo[idx]) { + next(&info, &idx); + } + + auto& l = mKeyVals[insertion_idx]; + if (idx == insertion_idx) { + ::new (static_cast(&l)) Node(std::move(keyval)); + } else { + shiftUp(idx, insertion_idx); + l = std::move(keyval); + } + + // put at empty spot + mInfo[insertion_idx] = insertion_info; + + ++mNumElements; + } + +public: + using iterator = Iter; + using const_iterator = Iter; + + Table() noexcept(noexcept(Hash()) && noexcept(KeyEqual())) + : WHash() + , WKeyEqual() { + ROBIN_HOOD_TRACE(this) + } + + // Creates an empty hash map. Nothing is allocated yet, this happens at the first insert. + // This tremendously speeds up ctor & dtor of a map that never receives an element. The + // penalty is payed at the first insert, and not before. Lookup of this empty map works + // because everybody points to DummyInfoByte::b. parameter bucket_count is dictated by the + // standard, but we can ignore it. + explicit Table( + size_t ROBIN_HOOD_UNUSED(bucket_count) /*unused*/, const Hash& h = Hash{}, + const KeyEqual& equal = KeyEqual{}) noexcept(noexcept(Hash(h)) && noexcept(KeyEqual(equal))) + : WHash(h) + , WKeyEqual(equal) { + ROBIN_HOOD_TRACE(this) + } + + template + Table(Iter first, Iter last, size_t ROBIN_HOOD_UNUSED(bucket_count) /*unused*/ = 0, + const Hash& h = Hash{}, const KeyEqual& equal = KeyEqual{}) + : WHash(h) + , WKeyEqual(equal) { + ROBIN_HOOD_TRACE(this) + insert(first, last); + } + + Table(std::initializer_list initlist, + size_t ROBIN_HOOD_UNUSED(bucket_count) /*unused*/ = 0, const Hash& h = Hash{}, + const KeyEqual& equal = KeyEqual{}) + : WHash(h) + , WKeyEqual(equal) { + ROBIN_HOOD_TRACE(this) + insert(initlist.begin(), initlist.end()); + } + + Table(Table&& o) noexcept + : WHash(std::move(static_cast(o))) + , WKeyEqual(std::move(static_cast(o))) + , DataPool(std::move(static_cast(o))) { + ROBIN_HOOD_TRACE(this) + if (o.mMask) { + mHashMultiplier = std::move(o.mHashMultiplier); + mKeyVals = std::move(o.mKeyVals); + mInfo = std::move(o.mInfo); + mNumElements = std::move(o.mNumElements); + mMask = std::move(o.mMask); + mMaxNumElementsAllowed = std::move(o.mMaxNumElementsAllowed); + mInfoInc = std::move(o.mInfoInc); + mInfoHashShift = std::move(o.mInfoHashShift); + // set other's mask to 0 so its destructor won't do anything + o.init(); + } + } + + Table& operator=(Table&& o) noexcept { + ROBIN_HOOD_TRACE(this) + if (&o != this) { + if (o.mMask) { + // only move stuff if the other map actually has some data + destroy(); + mHashMultiplier = std::move(o.mHashMultiplier); + mKeyVals = std::move(o.mKeyVals); + mInfo = std::move(o.mInfo); + mNumElements = std::move(o.mNumElements); + mMask = std::move(o.mMask); + mMaxNumElementsAllowed = std::move(o.mMaxNumElementsAllowed); + mInfoInc = std::move(o.mInfoInc); + mInfoHashShift = std::move(o.mInfoHashShift); + WHash::operator=(std::move(static_cast(o))); + WKeyEqual::operator=(std::move(static_cast(o))); + DataPool::operator=(std::move(static_cast(o))); + + o.init(); + + } else { + // nothing in the other map => just clear us. + clear(); + } + } + return *this; + } + + Table(const Table& o) + : WHash(static_cast(o)) + , WKeyEqual(static_cast(o)) + , DataPool(static_cast(o)) { + ROBIN_HOOD_TRACE(this) + if (!o.empty()) { + // not empty: create an exact copy. it is also possible to just iterate through all + // elements and insert them, but copying is probably faster. + + auto const numElementsWithBuffer = calcNumElementsWithBuffer(o.mMask + 1); + auto const numBytesTotal = calcNumBytesTotal(numElementsWithBuffer); + + ROBIN_HOOD_LOG("std::malloc " << numBytesTotal << " = calcNumBytesTotal(" + << numElementsWithBuffer << ")") + mHashMultiplier = o.mHashMultiplier; + mKeyVals = static_cast( + detail::assertNotNull(std::malloc(numBytesTotal))); + // no need for calloc because clonData does memcpy + mInfo = reinterpret_cast(mKeyVals + numElementsWithBuffer); + mNumElements = o.mNumElements; + mMask = o.mMask; + mMaxNumElementsAllowed = o.mMaxNumElementsAllowed; + mInfoInc = o.mInfoInc; + mInfoHashShift = o.mInfoHashShift; + cloneData(o); + } + } + + // Creates a copy of the given map. Copy constructor of each entry is used. + // Not sure why clang-tidy thinks this doesn't handle self assignment, it does + // NOLINTNEXTLINE(bugprone-unhandled-self-assignment,cert-oop54-cpp) + Table& operator=(Table const& o) { + ROBIN_HOOD_TRACE(this) + if (&o == this) { + // prevent assigning of itself + return *this; + } + + // we keep using the old allocator and not assign the new one, because we want to keep + // the memory available. when it is the same size. + if (o.empty()) { + if (0 == mMask) { + // nothing to do, we are empty too + return *this; + } + + // not empty: destroy what we have there + // clear also resets mInfo to 0, that's sometimes not necessary. + destroy(); + init(); + WHash::operator=(static_cast(o)); + WKeyEqual::operator=(static_cast(o)); + DataPool::operator=(static_cast(o)); + + return *this; + } + + // clean up old stuff + Destroyer::value>{}.nodes(*this); + + if (mMask != o.mMask) { + // no luck: we don't have the same array size allocated, so we need to realloc. + if (0 != mMask) { + // only deallocate if we actually have data! + ROBIN_HOOD_LOG("std::free") + std::free(mKeyVals); + } + + auto const numElementsWithBuffer = calcNumElementsWithBuffer(o.mMask + 1); + auto const numBytesTotal = calcNumBytesTotal(numElementsWithBuffer); + ROBIN_HOOD_LOG("std::malloc " << numBytesTotal << " = calcNumBytesTotal(" + << numElementsWithBuffer << ")") + mKeyVals = static_cast( + detail::assertNotNull(std::malloc(numBytesTotal))); + + // no need for calloc here because cloneData performs a memcpy. + mInfo = reinterpret_cast(mKeyVals + numElementsWithBuffer); + // sentinel is set in cloneData + } + WHash::operator=(static_cast(o)); + WKeyEqual::operator=(static_cast(o)); + DataPool::operator=(static_cast(o)); + mHashMultiplier = o.mHashMultiplier; + mNumElements = o.mNumElements; + mMask = o.mMask; + mMaxNumElementsAllowed = o.mMaxNumElementsAllowed; + mInfoInc = o.mInfoInc; + mInfoHashShift = o.mInfoHashShift; + cloneData(o); + + return *this; + } + + // Swaps everything between the two maps. + void swap(Table& o) { + ROBIN_HOOD_TRACE(this) + using std::swap; + swap(o, *this); + } + + // Clears all data, without resizing. + void clear() { + ROBIN_HOOD_TRACE(this) + if (empty()) { + // don't do anything! also important because we don't want to write to + // DummyInfoByte::b, even though we would just write 0 to it. + return; + } + + Destroyer::value>{}.nodes(*this); + + auto const numElementsWithBuffer = calcNumElementsWithBuffer(mMask + 1); + // clear everything, then set the sentinel again + uint8_t const z = 0; + std::fill(mInfo, mInfo + calcNumBytesInfo(numElementsWithBuffer), z); + mInfo[numElementsWithBuffer] = 1; + + mInfoInc = InitialInfoInc; + mInfoHashShift = InitialInfoHashShift; + } + + // Destroys the map and all it's contents. + ~Table() { + ROBIN_HOOD_TRACE(this) + destroy(); + } + + // Checks if both tables contain the same entries. Order is irrelevant. + bool operator==(const Table& other) const { + ROBIN_HOOD_TRACE(this) + if (other.size() != size()) { + return false; + } + for (auto const& otherEntry : other) { + if (!has(otherEntry)) { + return false; + } + } + + return true; + } + + bool operator!=(const Table& other) const { + ROBIN_HOOD_TRACE(this) + return !operator==(other); + } + + template + typename std::enable_if::value, Q&>::type operator[](const key_type& key) { + ROBIN_HOOD_TRACE(this) + auto idxAndState = insertKeyPrepareEmptySpot(key); + switch (idxAndState.second) { + case InsertionState::key_found: + break; + + case InsertionState::new_node: + ::new (static_cast(&mKeyVals[idxAndState.first])) + Node(*this, std::piecewise_construct, std::forward_as_tuple(key), + std::forward_as_tuple()); + break; + + case InsertionState::overwrite_node: + mKeyVals[idxAndState.first] = Node(*this, std::piecewise_construct, + std::forward_as_tuple(key), std::forward_as_tuple()); + break; + + case InsertionState::overflow_error: + throwOverflowError(); + } + + return mKeyVals[idxAndState.first].getSecond(); + } + + template + typename std::enable_if::value, Q&>::type operator[](key_type&& key) { + ROBIN_HOOD_TRACE(this) + auto idxAndState = insertKeyPrepareEmptySpot(key); + switch (idxAndState.second) { + case InsertionState::key_found: + break; + + case InsertionState::new_node: + ::new (static_cast(&mKeyVals[idxAndState.first])) + Node(*this, std::piecewise_construct, std::forward_as_tuple(std::move(key)), + std::forward_as_tuple()); + break; + + case InsertionState::overwrite_node: + mKeyVals[idxAndState.first] = + Node(*this, std::piecewise_construct, std::forward_as_tuple(std::move(key)), + std::forward_as_tuple()); + break; + + case InsertionState::overflow_error: + throwOverflowError(); + } + + return mKeyVals[idxAndState.first].getSecond(); + } + + template + void insert(Iter first, Iter last) { + for (; first != last; ++first) { + // value_type ctor needed because this might be called with std::pair's + insert(value_type(*first)); + } + } + + void insert(std::initializer_list ilist) { + for (auto&& vt : ilist) { + insert(std::move(vt)); + } + } + + template + std::pair emplace(Args&&... args) { + ROBIN_HOOD_TRACE(this) + Node n{*this, std::forward(args)...}; + auto idxAndState = insertKeyPrepareEmptySpot(getFirstConst(n)); + switch (idxAndState.second) { + case InsertionState::key_found: + n.destroy(*this); + break; + + case InsertionState::new_node: + ::new (static_cast(&mKeyVals[idxAndState.first])) Node(*this, std::move(n)); + break; + + case InsertionState::overwrite_node: + mKeyVals[idxAndState.first] = std::move(n); + break; + + case InsertionState::overflow_error: + n.destroy(*this); + throwOverflowError(); + break; + } + + return std::make_pair(iterator(mKeyVals + idxAndState.first, mInfo + idxAndState.first), + InsertionState::key_found != idxAndState.second); + } + + template + iterator emplace_hint(const_iterator position, Args&&... args) { + (void)position; + return emplace(std::forward(args)...).first; + } + + template + std::pair try_emplace(const key_type& key, Args&&... args) { + return try_emplace_impl(key, std::forward(args)...); + } + + template + std::pair try_emplace(key_type&& key, Args&&... args) { + return try_emplace_impl(std::move(key), std::forward(args)...); + } + + template + iterator try_emplace(const_iterator hint, const key_type& key, Args&&... args) { + (void)hint; + return try_emplace_impl(key, std::forward(args)...).first; + } + + template + iterator try_emplace(const_iterator hint, key_type&& key, Args&&... args) { + (void)hint; + return try_emplace_impl(std::move(key), std::forward(args)...).first; + } + + template + std::pair insert_or_assign(const key_type& key, Mapped&& obj) { + return insertOrAssignImpl(key, std::forward(obj)); + } + + template + std::pair insert_or_assign(key_type&& key, Mapped&& obj) { + return insertOrAssignImpl(std::move(key), std::forward(obj)); + } + + template + iterator insert_or_assign(const_iterator hint, const key_type& key, Mapped&& obj) { + (void)hint; + return insertOrAssignImpl(key, std::forward(obj)).first; + } + + template + iterator insert_or_assign(const_iterator hint, key_type&& key, Mapped&& obj) { + (void)hint; + return insertOrAssignImpl(std::move(key), std::forward(obj)).first; + } + + std::pair insert(const value_type& keyval) { + ROBIN_HOOD_TRACE(this) + return emplace(keyval); + } + + iterator insert(const_iterator hint, const value_type& keyval) { + (void)hint; + return emplace(keyval).first; + } + + std::pair insert(value_type&& keyval) { + return emplace(std::move(keyval)); + } + + iterator insert(const_iterator hint, value_type&& keyval) { + (void)hint; + return emplace(std::move(keyval)).first; + } + + // Returns 1 if key is found, 0 otherwise. + size_t count(const key_type& key) const { // NOLINT(modernize-use-nodiscard) + ROBIN_HOOD_TRACE(this) + auto kv = mKeyVals + findIdx(key); + if (kv != reinterpret_cast_no_cast_align_warning(mInfo)) { + return 1; + } + return 0; + } + + template + // NOLINTNEXTLINE(modernize-use-nodiscard) + typename std::enable_if::type count(const OtherKey& key) const { + ROBIN_HOOD_TRACE(this) + auto kv = mKeyVals + findIdx(key); + if (kv != reinterpret_cast_no_cast_align_warning(mInfo)) { + return 1; + } + return 0; + } + + bool contains(const key_type& key) const { // NOLINT(modernize-use-nodiscard) + return 1U == count(key); + } + + template + // NOLINTNEXTLINE(modernize-use-nodiscard) + typename std::enable_if::type contains(const OtherKey& key) const { + return 1U == count(key); + } + + // Returns a reference to the value found for key. + // Throws std::out_of_range if element cannot be found + template + // NOLINTNEXTLINE(modernize-use-nodiscard) + typename std::enable_if::value, Q&>::type at(key_type const& key) { + ROBIN_HOOD_TRACE(this) + auto kv = mKeyVals + findIdx(key); + if (kv == reinterpret_cast_no_cast_align_warning(mInfo)) { + doThrow("key not found"); + } + return kv->getSecond(); + } + + // Returns a reference to the value found for key. + // Throws std::out_of_range if element cannot be found + template + // NOLINTNEXTLINE(modernize-use-nodiscard) + typename std::enable_if::value, Q const&>::type at(key_type const& key) const { + ROBIN_HOOD_TRACE(this) + auto kv = mKeyVals + findIdx(key); + if (kv == reinterpret_cast_no_cast_align_warning(mInfo)) { + doThrow("key not found"); + } + return kv->getSecond(); + } + + const_iterator find(const key_type& key) const { // NOLINT(modernize-use-nodiscard) + ROBIN_HOOD_TRACE(this) + const size_t idx = findIdx(key); + return const_iterator{mKeyVals + idx, mInfo + idx}; + } + + template + const_iterator find(const OtherKey& key, is_transparent_tag /*unused*/) const { + ROBIN_HOOD_TRACE(this) + const size_t idx = findIdx(key); + return const_iterator{mKeyVals + idx, mInfo + idx}; + } + + template + typename std::enable_if::type // NOLINT(modernize-use-nodiscard) + find(const OtherKey& key) const { // NOLINT(modernize-use-nodiscard) + ROBIN_HOOD_TRACE(this) + const size_t idx = findIdx(key); + return const_iterator{mKeyVals + idx, mInfo + idx}; + } + + iterator find(const key_type& key) { + ROBIN_HOOD_TRACE(this) + const size_t idx = findIdx(key); + return iterator{mKeyVals + idx, mInfo + idx}; + } + + template + iterator find(const OtherKey& key, is_transparent_tag /*unused*/) { + ROBIN_HOOD_TRACE(this) + const size_t idx = findIdx(key); + return iterator{mKeyVals + idx, mInfo + idx}; + } + + template + typename std::enable_if::type find(const OtherKey& key) { + ROBIN_HOOD_TRACE(this) + const size_t idx = findIdx(key); + return iterator{mKeyVals + idx, mInfo + idx}; + } + + iterator begin() { + ROBIN_HOOD_TRACE(this) + if (empty()) { + return end(); + } + return iterator(mKeyVals, mInfo, fast_forward_tag{}); + } + const_iterator begin() const { // NOLINT(modernize-use-nodiscard) + ROBIN_HOOD_TRACE(this) + return cbegin(); + } + const_iterator cbegin() const { // NOLINT(modernize-use-nodiscard) + ROBIN_HOOD_TRACE(this) + if (empty()) { + return cend(); + } + return const_iterator(mKeyVals, mInfo, fast_forward_tag{}); + } + + iterator end() { + ROBIN_HOOD_TRACE(this) + // no need to supply valid info pointer: end() must not be dereferenced, and only node + // pointer is compared. + return iterator{reinterpret_cast_no_cast_align_warning(mInfo), nullptr}; + } + const_iterator end() const { // NOLINT(modernize-use-nodiscard) + ROBIN_HOOD_TRACE(this) + return cend(); + } + const_iterator cend() const { // NOLINT(modernize-use-nodiscard) + ROBIN_HOOD_TRACE(this) + return const_iterator{reinterpret_cast_no_cast_align_warning(mInfo), nullptr}; + } + + iterator erase(const_iterator pos) { + ROBIN_HOOD_TRACE(this) + // its safe to perform const cast here + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-const-cast) + return erase(iterator{const_cast(pos.mKeyVals), const_cast(pos.mInfo)}); + } + + // Erases element at pos, returns iterator to the next element. + iterator erase(iterator pos) { + ROBIN_HOOD_TRACE(this) + // we assume that pos always points to a valid entry, and not end(). + auto const idx = static_cast(pos.mKeyVals - mKeyVals); + + shiftDown(idx); + --mNumElements; + + if (*pos.mInfo) { + // we've backward shifted, return this again + return pos; + } + + // no backward shift, return next element + return ++pos; + } + + size_t erase(const key_type& key) { + ROBIN_HOOD_TRACE(this) + size_t idx{}; + InfoType info{}; + keyToIdx(key, &idx, &info); + + // check while info matches with the source idx + do { + if (info == mInfo[idx] && WKeyEqual::operator()(key, mKeyVals[idx].getFirst())) { + shiftDown(idx); + --mNumElements; + return 1; + } + next(&info, &idx); + } while (info <= mInfo[idx]); + + // nothing found to delete + return 0; + } + + // reserves space for the specified number of elements. Makes sure the old data fits. + // exactly the same as reserve(c). + void rehash(size_t c) { + // forces a reserve + reserve(c, true); + } + + // reserves space for the specified number of elements. Makes sure the old data fits. + // Exactly the same as rehash(c). Use rehash(0) to shrink to fit. + void reserve(size_t c) { + // reserve, but don't force rehash + reserve(c, false); + } + + // If possible reallocates the map to a smaller one. This frees the underlying table. + // Does not do anything if load_factor is too large for decreasing the table's size. + void compact() { + ROBIN_HOOD_TRACE(this) + auto newSize = InitialNumElements; + while (calcMaxNumElementsAllowed(newSize) < mNumElements && newSize != 0) { + newSize *= 2; + } + if (ROBIN_HOOD_UNLIKELY(newSize == 0)) { + throwOverflowError(); + } + + ROBIN_HOOD_LOG("newSize > mMask + 1: " << newSize << " > " << mMask << " + 1") + + // only actually do anything when the new size is bigger than the old one. This prevents to + // continuously allocate for each reserve() call. + if (newSize < mMask + 1) { + rehashPowerOfTwo(newSize, true); + } + } + + size_type size() const noexcept { // NOLINT(modernize-use-nodiscard) + ROBIN_HOOD_TRACE(this) + return mNumElements; + } + + size_type max_size() const noexcept { // NOLINT(modernize-use-nodiscard) + ROBIN_HOOD_TRACE(this) + return static_cast(-1); + } + + ROBIN_HOOD(NODISCARD) bool empty() const noexcept { + ROBIN_HOOD_TRACE(this) + return 0 == mNumElements; + } + + float max_load_factor() const noexcept { // NOLINT(modernize-use-nodiscard) + ROBIN_HOOD_TRACE(this) + return MaxLoadFactor100 / 100.0F; + } + + // Average number of elements per bucket. Since we allow only 1 per bucket + float load_factor() const noexcept { // NOLINT(modernize-use-nodiscard) + ROBIN_HOOD_TRACE(this) + return static_cast(size()) / static_cast(mMask + 1); + } + + ROBIN_HOOD(NODISCARD) size_t mask() const noexcept { + ROBIN_HOOD_TRACE(this) + return mMask; + } + + ROBIN_HOOD(NODISCARD) size_t calcMaxNumElementsAllowed(size_t maxElements) const noexcept { + if (ROBIN_HOOD_LIKELY(maxElements <= (std::numeric_limits::max)() / 100)) { + return maxElements * MaxLoadFactor100 / 100; + } + + // we might be a bit inprecise, but since maxElements is quite large that doesn't matter + return (maxElements / 100) * MaxLoadFactor100; + } + + ROBIN_HOOD(NODISCARD) size_t calcNumBytesInfo(size_t numElements) const noexcept { + // we add a uint64_t, which houses the sentinel (first byte) and padding so we can load + // 64bit types. + return numElements + sizeof(uint64_t); + } + + ROBIN_HOOD(NODISCARD) + size_t calcNumElementsWithBuffer(size_t numElements) const noexcept { + auto maxNumElementsAllowed = calcMaxNumElementsAllowed(numElements); + return numElements + (std::min)(maxNumElementsAllowed, (static_cast(0xFF))); + } + + // calculation only allowed for 2^n values + ROBIN_HOOD(NODISCARD) size_t calcNumBytesTotal(size_t numElements) const { +#if ROBIN_HOOD(BITNESS) == 64 + return numElements * sizeof(Node) + calcNumBytesInfo(numElements); +#else + // make sure we're doing 64bit operations, so we are at least safe against 32bit overflows. + auto const ne = static_cast(numElements); + auto const s = static_cast(sizeof(Node)); + auto const infos = static_cast(calcNumBytesInfo(numElements)); + + auto const total64 = ne * s + infos; + auto const total = static_cast(total64); + + if (ROBIN_HOOD_UNLIKELY(static_cast(total) != total64)) { + throwOverflowError(); + } + return total; +#endif + } + +private: + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::value, bool>::type has(const value_type& e) const { + ROBIN_HOOD_TRACE(this) + auto it = find(e.first); + return it != end() && it->second == e.second; + } + + template + ROBIN_HOOD(NODISCARD) + typename std::enable_if::value, bool>::type has(const value_type& e) const { + ROBIN_HOOD_TRACE(this) + return find(e) != end(); + } + + void reserve(size_t c, bool forceRehash) { + ROBIN_HOOD_TRACE(this) + auto const minElementsAllowed = (std::max)(c, mNumElements); + auto newSize = InitialNumElements; + while (calcMaxNumElementsAllowed(newSize) < minElementsAllowed && newSize != 0) { + newSize *= 2; + } + if (ROBIN_HOOD_UNLIKELY(newSize == 0)) { + throwOverflowError(); + } + + ROBIN_HOOD_LOG("newSize > mMask + 1: " << newSize << " > " << mMask << " + 1") + + // only actually do anything when the new size is bigger than the old one. This prevents to + // continuously allocate for each reserve() call. + if (forceRehash || newSize > mMask + 1) { + rehashPowerOfTwo(newSize, false); + } + } + + // reserves space for at least the specified number of elements. + // only works if numBuckets if power of two + // True on success, false otherwise + void rehashPowerOfTwo(size_t numBuckets, bool forceFree) { + ROBIN_HOOD_TRACE(this) + + Node* const oldKeyVals = mKeyVals; + uint8_t const* const oldInfo = mInfo; + + const size_t oldMaxElementsWithBuffer = calcNumElementsWithBuffer(mMask + 1); + + // resize operation: move stuff + initData(numBuckets); + if (oldMaxElementsWithBuffer > 1) { + for (size_t i = 0; i < oldMaxElementsWithBuffer; ++i) { + if (oldInfo[i] != 0) { + // might throw an exception, which is really bad since we are in the middle of + // moving stuff. + insert_move(std::move(oldKeyVals[i])); + // destroy the node but DON'T destroy the data. + oldKeyVals[i].~Node(); + } + } + + // this check is not necessary as it's guarded by the previous if, but it helps + // silence g++'s overeager "attempt to free a non-heap object 'map' + // [-Werror=free-nonheap-object]" warning. + if (oldKeyVals != reinterpret_cast_no_cast_align_warning(&mMask)) { + // don't destroy old data: put it into the pool instead + if (forceFree) { + std::free(oldKeyVals); + } else { + DataPool::addOrFree(oldKeyVals, calcNumBytesTotal(oldMaxElementsWithBuffer)); + } + } + } + } + + ROBIN_HOOD(NOINLINE) void throwOverflowError() const { +#if ROBIN_HOOD(HAS_EXCEPTIONS) + throw std::overflow_error("robin_hood::map overflow"); +#else + abort(); +#endif + } + + template + std::pair try_emplace_impl(OtherKey&& key, Args&&... args) { + ROBIN_HOOD_TRACE(this) + auto idxAndState = insertKeyPrepareEmptySpot(key); + switch (idxAndState.second) { + case InsertionState::key_found: + break; + + case InsertionState::new_node: + ::new (static_cast(&mKeyVals[idxAndState.first])) Node( + *this, std::piecewise_construct, std::forward_as_tuple(std::forward(key)), + std::forward_as_tuple(std::forward(args)...)); + break; + + case InsertionState::overwrite_node: + mKeyVals[idxAndState.first] = Node(*this, std::piecewise_construct, + std::forward_as_tuple(std::forward(key)), + std::forward_as_tuple(std::forward(args)...)); + break; + + case InsertionState::overflow_error: + throwOverflowError(); + break; + } + + return std::make_pair(iterator(mKeyVals + idxAndState.first, mInfo + idxAndState.first), + InsertionState::key_found != idxAndState.second); + } + + template + std::pair insertOrAssignImpl(OtherKey&& key, Mapped&& obj) { + ROBIN_HOOD_TRACE(this) + auto idxAndState = insertKeyPrepareEmptySpot(key); + switch (idxAndState.second) { + case InsertionState::key_found: + mKeyVals[idxAndState.first].getSecond() = std::forward(obj); + break; + + case InsertionState::new_node: + ::new (static_cast(&mKeyVals[idxAndState.first])) Node( + *this, std::piecewise_construct, std::forward_as_tuple(std::forward(key)), + std::forward_as_tuple(std::forward(obj))); + break; + + case InsertionState::overwrite_node: + mKeyVals[idxAndState.first] = Node(*this, std::piecewise_construct, + std::forward_as_tuple(std::forward(key)), + std::forward_as_tuple(std::forward(obj))); + break; + + case InsertionState::overflow_error: + throwOverflowError(); + break; + } + + return std::make_pair(iterator(mKeyVals + idxAndState.first, mInfo + idxAndState.first), + InsertionState::key_found != idxAndState.second); + } + + void initData(size_t max_elements) { + mNumElements = 0; + mMask = max_elements - 1; + mMaxNumElementsAllowed = calcMaxNumElementsAllowed(max_elements); + + auto const numElementsWithBuffer = calcNumElementsWithBuffer(max_elements); + + // malloc & zero mInfo. Faster than calloc everything. + auto const numBytesTotal = calcNumBytesTotal(numElementsWithBuffer); + ROBIN_HOOD_LOG("std::calloc " << numBytesTotal << " = calcNumBytesTotal(" + << numElementsWithBuffer << ")") + mKeyVals = reinterpret_cast( + detail::assertNotNull(std::malloc(numBytesTotal))); + mInfo = reinterpret_cast(mKeyVals + numElementsWithBuffer); + std::memset(mInfo, 0, numBytesTotal - numElementsWithBuffer * sizeof(Node)); + + // set sentinel + mInfo[numElementsWithBuffer] = 1; + + mInfoInc = InitialInfoInc; + mInfoHashShift = InitialInfoHashShift; + } + + enum class InsertionState { overflow_error, key_found, new_node, overwrite_node }; + + // Finds key, and if not already present prepares a spot where to pot the key & value. + // This potentially shifts nodes out of the way, updates mInfo and number of inserted + // elements, so the only operation left to do is create/assign a new node at that spot. + template + std::pair insertKeyPrepareEmptySpot(OtherKey&& key) { + for (int i = 0; i < 256; ++i) { + size_t idx{}; + InfoType info{}; + keyToIdx(key, &idx, &info); + nextWhileLess(&info, &idx); + + // while we potentially have a match + while (info == mInfo[idx]) { + if (WKeyEqual::operator()(key, mKeyVals[idx].getFirst())) { + // key already exists, do NOT insert. + // see http://en.cppreference.com/w/cpp/container/unordered_map/insert + return std::make_pair(idx, InsertionState::key_found); + } + next(&info, &idx); + } + + // unlikely that this evaluates to true + if (ROBIN_HOOD_UNLIKELY(mNumElements >= mMaxNumElementsAllowed)) { + if (!increase_size()) { + return std::make_pair(size_t(0), InsertionState::overflow_error); + } + continue; + } + + // key not found, so we are now exactly where we want to insert it. + auto const insertion_idx = idx; + auto const insertion_info = info; + if (ROBIN_HOOD_UNLIKELY(insertion_info + mInfoInc > 0xFF)) { + mMaxNumElementsAllowed = 0; + } + + // find an empty spot + while (0 != mInfo[idx]) { + next(&info, &idx); + } + + if (idx != insertion_idx) { + shiftUp(idx, insertion_idx); + } + // put at empty spot + mInfo[insertion_idx] = static_cast(insertion_info); + ++mNumElements; + return std::make_pair(insertion_idx, idx == insertion_idx + ? InsertionState::new_node + : InsertionState::overwrite_node); + } + + // enough attempts failed, so finally give up. + return std::make_pair(size_t(0), InsertionState::overflow_error); + } + + bool try_increase_info() { + ROBIN_HOOD_LOG("mInfoInc=" << mInfoInc << ", numElements=" << mNumElements + << ", maxNumElementsAllowed=" + << calcMaxNumElementsAllowed(mMask + 1)) + if (mInfoInc <= 2) { + // need to be > 2 so that shift works (otherwise undefined behavior!) + return false; + } + // we got space left, try to make info smaller + mInfoInc = static_cast(mInfoInc >> 1U); + + // remove one bit of the hash, leaving more space for the distance info. + // This is extremely fast because we can operate on 8 bytes at once. + ++mInfoHashShift; + auto const numElementsWithBuffer = calcNumElementsWithBuffer(mMask + 1); + + for (size_t i = 0; i < numElementsWithBuffer; i += 8) { + auto val = unaligned_load(mInfo + i); + val = (val >> 1U) & UINT64_C(0x7f7f7f7f7f7f7f7f); + std::memcpy(mInfo + i, &val, sizeof(val)); + } + // update sentinel, which might have been cleared out! + mInfo[numElementsWithBuffer] = 1; + + mMaxNumElementsAllowed = calcMaxNumElementsAllowed(mMask + 1); + return true; + } + + // True if resize was possible, false otherwise + bool increase_size() { + // nothing allocated yet? just allocate InitialNumElements + if (0 == mMask) { + initData(InitialNumElements); + return true; + } + + auto const maxNumElementsAllowed = calcMaxNumElementsAllowed(mMask + 1); + if (mNumElements < maxNumElementsAllowed && try_increase_info()) { + return true; + } + + ROBIN_HOOD_LOG("mNumElements=" << mNumElements << ", maxNumElementsAllowed=" + << maxNumElementsAllowed << ", load=" + << (static_cast(mNumElements) * 100.0 / + (static_cast(mMask) + 1))) + + if (mNumElements * 2 < calcMaxNumElementsAllowed(mMask + 1)) { + // we have to resize, even though there would still be plenty of space left! + // Try to rehash instead. Delete freed memory so we don't steadyily increase mem in case + // we have to rehash a few times + nextHashMultiplier(); + rehashPowerOfTwo(mMask + 1, true); + } else { + // we've reached the capacity of the map, so the hash seems to work nice. Keep using it. + rehashPowerOfTwo((mMask + 1) * 2, false); + } + return true; + } + + void nextHashMultiplier() { + // adding an *even* number, so that the multiplier will always stay odd. This is necessary + // so that the hash stays a mixing function (and thus doesn't have any information loss). + mHashMultiplier += UINT64_C(0xc4ceb9fe1a85ec54); + } + + void destroy() { + if (0 == mMask) { + // don't deallocate! + return; + } + + Destroyer::value>{} + .nodesDoNotDeallocate(*this); + + // This protection against not deleting mMask shouldn't be needed as it's sufficiently + // protected with the 0==mMask check, but I have this anyways because g++ 7 otherwise + // reports a compile error: attempt to free a non-heap object 'fm' + // [-Werror=free-nonheap-object] + if (mKeyVals != reinterpret_cast_no_cast_align_warning(&mMask)) { + ROBIN_HOOD_LOG("std::free") + std::free(mKeyVals); + } + } + + void init() noexcept { + mKeyVals = reinterpret_cast_no_cast_align_warning(&mMask); + mInfo = reinterpret_cast(&mMask); + mNumElements = 0; + mMask = 0; + mMaxNumElementsAllowed = 0; + mInfoInc = InitialInfoInc; + mInfoHashShift = InitialInfoHashShift; + } + + // members are sorted so no padding occurs + uint64_t mHashMultiplier = UINT64_C(0xc4ceb9fe1a85ec53); // 8 byte 8 + Node* mKeyVals = reinterpret_cast_no_cast_align_warning(&mMask); // 8 byte 16 + uint8_t* mInfo = reinterpret_cast(&mMask); // 8 byte 24 + size_t mNumElements = 0; // 8 byte 32 + size_t mMask = 0; // 8 byte 40 + size_t mMaxNumElementsAllowed = 0; // 8 byte 48 + InfoType mInfoInc = InitialInfoInc; // 4 byte 52 + InfoType mInfoHashShift = InitialInfoHashShift; // 4 byte 56 + // 16 byte 56 if NodeAllocator +}; + +} // namespace detail + +// map + +template , + typename KeyEqual = std::equal_to, size_t MaxLoadFactor100 = 80> +using unordered_flat_map = detail::Table; + +template , + typename KeyEqual = std::equal_to, size_t MaxLoadFactor100 = 80> +using unordered_node_map = detail::Table; + +template , + typename KeyEqual = std::equal_to, size_t MaxLoadFactor100 = 80> +using unordered_map = + detail::Table) <= sizeof(size_t) * 6 && + std::is_nothrow_move_constructible>::value && + std::is_nothrow_move_assignable>::value, + MaxLoadFactor100, Key, T, Hash, KeyEqual>; + +// set + +template , typename KeyEqual = std::equal_to, + size_t MaxLoadFactor100 = 80> +using unordered_flat_set = detail::Table; + +template , typename KeyEqual = std::equal_to, + size_t MaxLoadFactor100 = 80> +using unordered_node_set = detail::Table; + +template , typename KeyEqual = std::equal_to, + size_t MaxLoadFactor100 = 80> +using unordered_set = detail::Table::value && + std::is_nothrow_move_assignable::value, + MaxLoadFactor100, Key, void, Hash, KeyEqual>; + +} // namespace robin_hood + +#endif \ No newline at end of file diff --git a/inc/shm_types.hpp b/inc/shm_types.hpp new file mode 100644 index 0000000..55985d6 --- /dev/null +++ b/inc/shm_types.hpp @@ -0,0 +1,77 @@ +#pragma once + +#include +#include +#include + +namespace midas { + +enum CtrlOpCode { + CONNECT, + DISCONNECT, + ALLOC, + OVERCOMMIT, + FREE, + UPDLIMIT, + UPDLIMIT_REQ, + FORCE_RECLAIM, + PROF_STATS, + SET_WEIGHT, + SET_LAT_CRITICAL, +}; + +enum CtrlRetCode { + CONN_SUCC, + CONN_FAIL, + MEM_SUCC, + MEM_FAIL, +}; + +struct MemMsg { + int64_t region_id; + union { + uint64_t size; + float weight; + bool lat_critical; + }; +}; + +struct CtrlMsg { + uint64_t id; + CtrlOpCode op; + CtrlRetCode ret; + MemMsg mmsg; +}; + +struct StatsMsg { + // cache stats + uint64_t hits; + uint64_t misses; + double miss_penalty; + // victim cache stats + uint32_t vhits; + // full threshold + uint32_t headroom; +}; +static_assert(sizeof(CtrlMsg) == sizeof(StatsMsg), + "CtrlMsg and StatsMsg have different size!"); + +struct VRange { + VRange() = default; + VRange(void *addr_, size_t size_) : stt_addr(addr_), size(size_) {} + + void *stt_addr; + size_t size; + + bool contains(const void *ptr) const noexcept { + return ptr > stt_addr && ptr < reinterpret_cast(stt_addr) + size; + } +}; + +namespace utils { +static inline const std::string get_region_name(uint64_t pid, uint64_t rid) { + return "region-" + std::to_string(pid) + "-" + std::to_string(rid); +} +} // namespace utils + +} // namespace midas \ No newline at end of file diff --git a/inc/sig_handler.hpp b/inc/sig_handler.hpp new file mode 100644 index 0000000..a965ab3 --- /dev/null +++ b/inc/sig_handler.hpp @@ -0,0 +1,34 @@ +#pragma once + +#include +#include +extern "C" { +#include +#include +} + +#include "utils.hpp" +#include "resilient_func.hpp" + +namespace midas { + +class SigHandler { +public: + SigHandler(); + void init(); + void register_func(uint64_t stt_ip, uint64_t end_ip); + bool softfault_handler(siginfo_t *info, ucontext_t *ctx); + + static SigHandler *global_sighandler(); + +private: + ResilientFunc *dispatcher(uint64_t ip); + + std::vector funcs; +}; +} // namespace midas + +/** Implemented in stacktrace.cpp */ +extern "C" void print_callstack(siginfo_t *info, ucontext_t *ctx); + +#include "impl/sig_handler.ipp" \ No newline at end of file diff --git a/inc/slab.hpp b/inc/slab.hpp new file mode 100644 index 0000000..6f2a082 --- /dev/null +++ b/inc/slab.hpp @@ -0,0 +1,120 @@ +#pragma once +// #define ENABLE_SLAB +#undef ENABLE_SLAB // disable slab allocator by default + +#ifdef ENABLE_SLAB + +#include +#include +#include +#include +#include +#include + +#include "resource_manager.hpp" +#include "utils.hpp" + +namespace midas { + +class SlabRegion; +struct SlabHeader { + uint32_t slab_id; + uint32_t slab_size; + SlabRegion *region; +}; + +constexpr static uint32_t kAlignment = 16; +static_assert(sizeof(SlabHeader) % kAlignment == 0, + "SlabHeader is not aligned correctly"); + +class SlabRegion : public VRange { +public: + SlabRegion(uint32_t slab_id_, uint32_t slab_size_, void *addr, + size_t region_size) + : VRange(addr, region_size), slab_id(slab_id_), slab_size(slab_size_), + capacity(region_size / slab_size - + 1 /* the first element is for SlabHeader */), + nr_alloced(capacity), nr_freed(0), + ret_mtx(std::make_shared()) { + // TODO: lazily init to save memory + init(); + } + void push(void *addr); + void *pop(); + + inline bool full() const noexcept { return nr_alloced == capacity; } + inline uint32_t size() const noexcept { return nr_alloced; } + + void evacuate(SlabRegion *dst) {} + + using VRange::contains; + friend class SlabAllocator; + +private: + uint32_t init(); + const uint32_t slab_id; + const uint32_t slab_size; + const uint32_t capacity; + uint32_t nr_alloced; + uint32_t nr_freed; + struct FreeSlot { + void *addr; + FreeSlot *next; + }; + FreeSlot *slots; + + std::shared_ptr ret_mtx; + FreeSlot *ret_slots; +}; + +class SlabAllocator { +public: + constexpr static uint8_t kNumSlabClasses = 8; + // minial slab size should be 16 bytes for a FreeSlot (2 pointers) + constexpr static uint32_t kMinSlabShift = 4; + constexpr static uint32_t kMinSlabSize = (1 << kMinSlabShift); + constexpr static uint32_t kMaxSlabSize = + (kMinSlabSize << (kNumSlabClasses - 1)); + static_assert(sizeof(SlabHeader) <= kMinSlabSize, + "SlabHeader is larger than kMinSlabSize"); + + void *alloc(uint32_t size); + template T *alloc(uint32_t cnt); + void free(void *addr); + + void FreeRegion() {} + + static inline SlabAllocator *global_allocator(); + +private: + void *_alloc(uint32_t size); + static inline uint32_t get_slab_idx(uint32_t size) noexcept; + static inline uint32_t get_slab_size(uint32_t idx) noexcept; + static thread_local std::vector> + slab_regions[kNumSlabClasses]; +}; + +namespace utils { +/* From AIFM */ +static inline uint32_t bsr_32(uint32_t a) { + uint32_t ret; + asm("BSR %k1, %k0 \n" : "=r"(ret) : "rm"(a)); + return ret; +} + +static inline uint64_t bsr_64(uint64_t a) { + uint64_t ret; + asm("BSR %q1, %q0 \n" : "=r"(ret) : "rm"(a)); + return ret; +} + +static inline constexpr uint32_t round_up_power_of_two(uint32_t a) { + return a == 1 ? 1 : 1 << (32 - __builtin_clz(a - 1)); +} +} // namespace utils + +} // namespace midas + +#include "impl/slab.ipp" + +#endif // ENABLE_SLAB \ No newline at end of file diff --git a/inc/sync_hashmap.hpp b/inc/sync_hashmap.hpp new file mode 100644 index 0000000..bd7dc8d --- /dev/null +++ b/inc/sync_hashmap.hpp @@ -0,0 +1,56 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include "cache_manager.hpp" +#include "construct_args.hpp" +#include "log.hpp" +#include "object.hpp" +#include "time.hpp" + +namespace midas { + +template , typename Pred = std::equal_to, + typename Alloc = LogAllocator, typename Lock = std::mutex> +class SyncHashMap { +public: + SyncHashMap(); + SyncHashMap(CachePool *pool); + ~SyncHashMap(); + + template std::unique_ptr get(K1 &&key); + template bool get(K1 &&key, Tp &v); + template bool set(const K1 &key, const Tp1 &v); + template bool remove(K1 &&key); + bool clear(); + // std::vector get_all_pairs(); + +private: + struct BucketNode { + uint64_t key_hash; + ObjectPtr pair; + BucketNode *next; + }; + using BNPtr = BucketNode *; + + template + BNPtr create_node(uint64_t key_hash, K1 &&k, Tp1 &&v); + BNPtr delete_node(BNPtr *prev_next, BNPtr node); + template + bool iterate_list(uint64_t key_hash, K1 &&key, BNPtr *&prev_next, + BNPtr &node); + template BNPtr *find(K1 &&key, bool remove = false); + Lock locks_[NBuckets]; + BucketNode *buckets_[NBuckets]; + + CachePool *pool_; +}; + +} // namespace midas + +#include "impl/sync_hashmap.ipp" \ No newline at end of file diff --git a/inc/sync_kv.hpp b/inc/sync_kv.hpp new file mode 100644 index 0000000..6a307d6 --- /dev/null +++ b/inc/sync_kv.hpp @@ -0,0 +1,181 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "robinhood.h" + +#include "cache_manager.hpp" +#include "construct_args.hpp" +#include "log.hpp" +#include "object.hpp" +#include "time.hpp" + +namespace midas { + +namespace kv_types { +// using Key = std::pair; // key location and its length +struct Key { + const void *data; + size_t size; + Key() : data(nullptr), size(0) {} + Key(const void *data_, size_t size_) : data(data_), size(size_) {} +}; +static_assert(sizeof(Key) == sizeof(void *) + sizeof(size_t), + "Key is not correctly aligned!"); + +// using Value = std::pair; // value location and its length +struct Value { + void *data; + size_t size; + Value() : data(nullptr), size(0) {} + Value(void *data_, size_t size_) : data(data_), size(size_) {} +}; +static_assert(sizeof(Value) == sizeof(void *) + sizeof(size_t), + "Value is not correctly aligned!"); + +using CValue = Key; // const value format is the same to key +static_assert(sizeof(CValue) == sizeof(void *) + sizeof(size_t), + "CValue is not correctly aligned!"); + +using ValueWithScore = std::pair; // value and its score +static_assert(sizeof(ValueWithScore) == sizeof(Value) + sizeof(double), + "ValueWithScore is not correctly aligned!"); + +struct BatchPlug { + int16_t hits{0}; + int16_t misses{0}; + int16_t vhits{0}; + int16_t batch_size{0}; + + inline void reset() { hits = misses = vhits = batch_size = 0; } +}; +static_assert(sizeof(BatchPlug) <= sizeof(int64_t), + "BatchPlug is not correctly aligned!"); + +enum RetCode { Failed = 0, Succ = 1, Duplicated = 2 }; +} // namespace kv_types + +namespace kv_utils { +/** Util function to make a Key or [C]Value item. + * WARNING: clearly this declaration is very dirty and error-prune. Always + * use the shortcuts defined below if possible. */ +template T make_(decltype(T::data) data, decltype(T::size) size); +/** Shortcuts to create certain key/value types. */ +constexpr static auto make_key = make_; +constexpr static auto make_value = make_; +constexpr static auto make_cvalue = make_; +} // namespace kv_utils + +template +class SyncKV { +public: + SyncKV(); + SyncKV(CachePool *pool); + ~SyncKV(); + + /** Basic Interfaces */ + void *get(const void *key, size_t klen, size_t *vlen); + kv_types::Value get(const kv_types::Key &key); + template kv_types::Value get(const K &k); + template std::unique_ptr get(const K &k); + bool get(const void *key, size_t klen, void *value, size_t vlen); + + bool set(const void *key, size_t klen, const void *value, size_t vlen); + bool set(const kv_types::Key &key, const kv_types::CValue &value); + template bool set(const K &k, const kv_types::CValue &value); + template bool set(const K &k, const V &v); + + bool remove(const void *key, size_t klen); + bool remove(const kv_types::Key &key); + template bool remove(const K &k); + + /* V should be addable such as [u]int[_64_t]. */ + template + bool inc(const void *key, size_t klen, V offset, V *value); + // TODO: add support for reloaded versions + + /* add only when key doesn't exist. */ + int add(const void *key, size_t klen, const void *value, size_t vlen); + // TODO: add support for reloaded versions + + bool clear(); + // std::vector get_all_pairs(); + + /** Ordered Set Interfaces */ + enum class UpdateType { // mimicing Redis-plus-plus + EXIST, + NOT_EXIST, + ALWAYS + }; + bool zadd(const void *key, size_t klen, const void *value, size_t vlen, + double score, UpdateType type); + bool zrange(const void *key, size_t klen, int64_t start, int64_t end, + std::back_insert_iterator> bi); + bool zrevrange(const void *key, size_t klen, int64_t start, int64_t end, + std::back_insert_iterator> bi); + + /** Batched Interfaces */ + int bget(const std::vector &keys, + std::vector &values); + template + int bget(const std::vector &keys, std::vector &values); + template + int bget(const std::vector &keys, std::vector> &values); + + int bset(const std::vector &keys, + const std::vector &values); + template + int bset(const std::vector &keys, + const std::vector &values); + template + int bset(const std::vector &keys, const std::vector &values); + + int bremove(const std::vector &keys); + template int bremove(const std::vector &keys); + + // User can also mark a section with batch_[stt|end] and manually bget_single + void batch_stt(kv_types::BatchPlug &plug); + int batch_end(kv_types::BatchPlug &plug); + + void *bget_single(const void *key, size_t klen, size_t *vlen, + kv_types::BatchPlug &plug); + kv_types::Value bget_single(const kv_types::Key &key, + kv_types::BatchPlug &plug); + template + kv_types::Value bget_single(const K &k, kv_types::BatchPlug &plug); + template + std::unique_ptr bget_single(const K &k, kv_types::BatchPlug &plug); + +private: + struct BucketNode { + uint64_t key_hash; + ObjectPtr pair; + BucketNode *next; + }; + using BNPtr = BucketNode *; + + static inline uint64_t hash_(const void *key, size_t klen); + void *get_(const void *key, size_t klen, void *value, size_t *vlen, + kv_types::BatchPlug *plug, bool construct); + BNPtr create_node(uint64_t hash, const void *k, size_t kn, const void *v, + size_t vn); + BNPtr delete_node(BNPtr *prev_next, BNPtr node); + bool iterate_list(uint64_t hash, const void *k, size_t kn, size_t *vn, + BNPtr *&prev_next, BNPtr &node); + BNPtr *find(void *k, size_t kn, bool remove = false); + Lock locks_[NBuckets]; + BucketNode *buckets_[NBuckets]; + + CachePool *pool_; +}; + +} // namespace midas + +#include "impl/sync_kv.ipp" \ No newline at end of file diff --git a/inc/sync_list.hpp b/inc/sync_list.hpp new file mode 100644 index 0000000..89231bb --- /dev/null +++ b/inc/sync_list.hpp @@ -0,0 +1,42 @@ +#pragma once + +#include "log.hpp" +#include "object.hpp" +#include "cache_manager.hpp" + +namespace midas { + +template +class SyncList { +public: + SyncList(); + SyncList(CachePool *pool); + + std::unique_ptr pop(); + bool pop(Tp &v); + bool push(const Tp &v); + bool clear(); + + bool empty() const noexcept; + bool size() const noexcept; + +private: + struct ListNode { + ObjectPtr obj; + ListNode *next; + }; + + ListNode *create_node(const Tp &v); + void delete_node(ListNode *node); + + CachePool *pool_; + + Lock lock_; + + ListNode *list_; + std::atomic_int32_t size_; +}; +} // namespace midas + +#include "impl/sync_list.ipp" \ No newline at end of file diff --git a/inc/time.hpp b/inc/time.hpp new file mode 100644 index 0000000..5ae6085 --- /dev/null +++ b/inc/time.hpp @@ -0,0 +1,33 @@ +#pragma once + +#include +#include + +#include "utils.hpp" + +namespace midas { +struct Time { +public: + static inline uint64_t get_us() { return get_us_stt(); }; + static inline uint64_t get_cycles() { return get_cycles_stt(); }; + static inline uint64_t get_us_stt(); + static inline uint64_t get_us_end(); + static inline uint64_t get_cycles_stt(); + static inline uint64_t get_cycles_end(); + +private: + static inline uint64_t rdtsc(); + static inline uint64_t rdtscp(); + + static inline uint64_t cycles_to_us(uint64_t cycles) noexcept; + static inline uint64_t us_to_cycles(uint64_t us) noexcept; +}; + +namespace chrono_utils { +inline std::chrono::steady_clock::time_point now(); +inline double duration(const std::chrono::steady_clock::time_point &stt, + const std::chrono::steady_clock::time_point &end); +} // namespace chrono_utils +} // namespace midas + +#include "impl/time.ipp" \ No newline at end of file diff --git a/inc/transient_ptr.hpp b/inc/transient_ptr.hpp new file mode 100644 index 0000000..b0ee038 --- /dev/null +++ b/inc/transient_ptr.hpp @@ -0,0 +1,61 @@ +#pragma once + +#include +#include + +// #define BOUND_CHECK + +namespace midas { + +class TransientPtr { +public: + TransientPtr(); + TransientPtr(uint64_t addr, size_t size); + bool null() const noexcept; + bool set(uint64_t addr, size_t size); /* point to a transient addr. */ + bool reset() noexcept; + TransientPtr slice(int64_t offset) const; + TransientPtr slice(int64_t offset, size_t size) const; + size_t size() const noexcept; + /** + * Atomic operations + */ + bool cmpxchg(int64_t offset, uint64_t oldval, uint64_t newval); + int64_t atomic_add(int64_t offset, int64_t val); + /** + * Ops with single transient reference (this). + */ + bool copy_from(const void *src, size_t len, int64_t offset = 0); + bool copy_to(void *dst, size_t len, int64_t offset = 0); + /** + * Ops with two transient references (this & src/dst). + */ + bool copy_from(const TransientPtr &src, size_t len, int64_t from_offset = 0, + int64_t to_offset = 0); + bool copy_to(TransientPtr &dst, size_t len, int64_t from_offset = 0, + int64_t to_offset = 0); + /** + * Assign this pointer to dst. The location of dst is checked to make sure the + * correct method is called. + */ + bool assign_to_non_volatile(TransientPtr *dst); + bool assign_to_local_region(TransientPtr *dst); + bool assign_to_foreign_region(TransientPtr *dst); + + /** + * CAUTION! get raw address + */ + uint64_t to_normal_address() const noexcept; + +private: + uint64_t ptr_; + + friend class ObjLocker; +#ifdef BOUND_CHECK + size_t size_; /* accessible range of the pointer */ +#endif // BOUND_CHECK +}; + +} // namespace midas + +#include "impl/transient_ptr.ipp" \ No newline at end of file diff --git a/inc/utils.hpp b/inc/utils.hpp new file mode 100644 index 0000000..0cac5c1 --- /dev/null +++ b/inc/utils.hpp @@ -0,0 +1,67 @@ +#pragma once + +#include +#include + +namespace midas { + +constexpr static uint32_t kCPUFreq = 2095; +constexpr static uint32_t kNumCPUs = 128; +constexpr static uint32_t kSmallObjSizeUnit = sizeof(uint64_t); + +constexpr static uint32_t kShmObjNameLen = 128; +constexpr static uint32_t kPageSize = 4096; // 4KB +constexpr static uint32_t kHugePageSize = 512 * kPageSize; // 2MB == Huge Page +constexpr static uint64_t kVolatileSttAddr = 0x01f'000'000'000; +constexpr static uint64_t kVolatileEndAddr = + kVolatileSttAddr + 0x040'000'000'000; + +/** Fault Handler related */ +constexpr static bool kEnableFaultHandler = true; +/** Log Structured Allocator related */ +constexpr static uint32_t kLogSegmentSize = kHugePageSize; +constexpr static uint64_t kLogSegmentMask = ~(kLogSegmentSize - 1ull); +constexpr static uint32_t kRegionSize = kLogSegmentSize; +constexpr static uint64_t kRegionMask = ~(kRegionSize - 1ull); + +constexpr static int32_t kMaxAliveBytes = std::numeric_limits::max(); +/** Evacuator related */ +constexpr static float kAliveThreshHigh = 0.9; +constexpr static int kNumEvacThds = 12; +constexpr static int kForceReclaimThresh = 512; // #(regions to be reclaimed) +/** High-Level Data Structures & Interfaces related */ +constexpr static bool kEnableConstruct = true; + +#ifndef LIKELY +#define LIKELY(x) __builtin_expect((x), 1) +#endif +#ifndef UNLIKELY +#define UNLIKELY(x) __builtin_expect((x), 0) +#endif + +// align must be power of 2. +#define round_up_to_align(val, align) (((val) + ((align)-1)) & ~((align)-1)) +#define round_to_align(val, align) ((val) & ~((align)-1)) +#define ptr_offset(ptr, offset) \ + reinterpret_cast(reinterpret_cast(ptr) + (offset)) +#define offset_ptrs(ptr1, ptr2) \ + (reinterpret_cast(ptr1) - reinterpret_cast(ptr2)) + +#ifdef DEBUG +#define FORCE_INLINE inline +#else +#define FORCE_INLINE inline __attribute__((always_inline)) +#endif + +#define NOINLINE __attribute__((noinline)) +#define FLATTEN __attribute__((flatten)) +#define SOFT_RESILIENT __attribute__((section("resilient-func"))) + +#define DECL_RESILIENT_FUNC(ret_type, func, ...) \ + ret_type FLATTEN NOINLINE func(__VA_ARGS__) SOFT_RESILIENT; \ + void func##_end() SOFT_RESILIENT; + +#define DELIM_FUNC_IMPL(func) \ + void func##_end() { asm volatile(".byte 0xcc, 0xcc, 0xcc, 0xcc"); } + +} // namespace midas \ No newline at end of file diff --git a/inc/victim_cache.hpp b/inc/victim_cache.hpp new file mode 100644 index 0000000..f7c8943 --- /dev/null +++ b/inc/victim_cache.hpp @@ -0,0 +1,61 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "object.hpp" + +namespace midas { +constexpr static bool kEnableVictimCache = true; + +struct VCEntry { +#pragma pack(push, 1) + ObjectPtr *optr; + size_t size; + void *construct_args; +#pragma pack(pop) + VCEntry(); + VCEntry(ObjectPtr *optr_, void *construct_args_); + void reset(ObjectPtr *optr_, void *construct_args_) noexcept; +}; + +static_assert(sizeof(VCEntry) <= + sizeof(ObjectPtr) + sizeof(size_t) + sizeof(void *), + "VCEntry is not correctly aligned!"); + +class VictimCache { +public: + VictimCache(int64_t size_limit = std::numeric_limits::max(), + int64_t cnt_limit = std::numeric_limits::max()); + ~VictimCache(); + + bool get(ObjectPtr *optr_addr) noexcept; + bool put(ObjectPtr *optr_addr, void *construct_args) noexcept; + bool remove(ObjectPtr *optr_addr) noexcept; + + int64_t size() const noexcept; + int64_t count() const noexcept; + +private: + using VCIter = std::list::iterator; + void pop_back_locked() noexcept; + inline void create_entry(ObjectPtr *optr_addr, void *construct_args) noexcept; + inline void delete_entry(VCEntry *entry) noexcept; + + std::mutex mtx_; + std::list entries_; // LRU list + std::unordered_map map_; // optr * -> construct_args + + int64_t size_limit_; + int64_t cnt_limit_; + int64_t size_; + int64_t cnt_; +}; + +} // namespace midas + +#include "impl/victim_cache.ipp" \ No newline at end of file diff --git a/inc/zipf.hpp b/inc/zipf.hpp new file mode 100644 index 0000000..d666611 --- /dev/null +++ b/inc/zipf.hpp @@ -0,0 +1,59 @@ +/** + * Copied from AIFM. + */ + +#pragma once + +#include +#include +#include + +namespace midas { + +/** + * Example usage: + * + * std::random_device rd; + * std::mt19937 gen(rd()); + * zipf_table_distribution<> zipf(300); + * + * for (int i = 0; i < 100; i++) + * printf("draw %d %d\n", i, zipf(gen)); + */ +template +class zipf_table_distribution { +public: + typedef IntType result_type; + + static_assert(std::numeric_limits::is_integer, ""); + static_assert(!std::numeric_limits::is_integer, ""); + + /// zipf_table_distribution(N, s) + /// Zipf distribution for `N` items, in the range `[1,N]` inclusive. + /// The distribution follows the power-law 1/n^s with exponent `s`. + /// This uses a table-lookup, and thus provides values more + /// quickly than zipf_distribution. However, the table can take + /// up a considerable amount of RAM, and initializing this table + /// can consume significant time. + zipf_table_distribution(const IntType n, const RealType q = 1.0); + void reset(); + IntType operator()(std::mt19937 &rng); + /// Returns the parameter the distribution was constructed with. + RealType s() const; + /// Returns the minimum value potentially generated by the distribution. + result_type min() const; + /// Returns the maximum value potentially generated by the distribution. + result_type max() const; + +private: + std::vector pdf_; ///< Prob. distribution + IntType n_; ///< Number of elements + RealType q_; ///< Exponent + std::discrete_distribution dist_; ///< Draw generator + + /** Initialize the probability mass function */ + IntType init(const IntType n, const RealType q); +}; +} // namespace midas + +#include "impl/zipf.ipp" diff --git a/koord/.gitignore b/koord/.gitignore new file mode 100644 index 0000000..b56ef03 --- /dev/null +++ b/koord/.gitignore @@ -0,0 +1,3 @@ +.clang-format + +build/** \ No newline at end of file diff --git a/koord/Kbuild b/koord/Kbuild new file mode 100644 index 0000000..f79d2da --- /dev/null +++ b/koord/Kbuild @@ -0,0 +1 @@ +obj-m += koord.o diff --git a/koord/Makefile b/koord/Makefile new file mode 100644 index 0000000..b571264 --- /dev/null +++ b/koord/Makefile @@ -0,0 +1,15 @@ +KDIR ?= /lib/modules/$(shell uname -r)/build +BUILD_DIR ?= $(PWD)/build +BUILD_DIR_MAKEFILE ?= $(PWD)/build/Makefile + +default: $(BUILD_DIR_MAKEFILE) + make -C $(KDIR) M=$(BUILD_DIR) src=$(PWD) modules + +$(BUILD_DIR): + mkdir -p "$@" + +$(BUILD_DIR_MAKEFILE): $(BUILD_DIR) + touch "$@" + +clean: + make -C $(KDIR) M=$(BUILD_DIR) src=$(PWD) clean \ No newline at end of file diff --git a/koord/koord.c b/koord/koord.c new file mode 100644 index 0000000..c386b41 --- /dev/null +++ b/koord/koord.c @@ -0,0 +1,287 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "koord.h" + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Yifan Qiao"); +MODULE_DESCRIPTION("Midas Coordinator Module"); + +#ifndef HUGEPAGE_SIZE +#define HUGEPAGE_SIZE (512ul * 4096) // 2MB +#define HUGEPAGE_MASK (HUGEPAGE_SIZE - 1) +#endif // HUGEPAGE_SIZE + +#ifndef HUGEPAGE_SHIFT +#define HUGEPAGE_SHIFT (9 + 12) // 2^21 +#endif // HUGEPAGE_SHIFT + +/* the character device that provides the koord IOCTL interface */ +static struct cdev koord_cdev; + +/** + * ksched_lookup_task - retreives a task from a pid number + * @nr: the pid number + * + * WARNING: must be called inside an RCU read critical section. + * + * Returns a task pointer or NULL if none was found. + */ +static struct task_struct *koord_lookup_task(pid_t nr) +{ + return pid_task(find_vpid(nr), PIDTYPE_PID); +} + +void mm_trace_rss_stat(struct mm_struct *mm, int member, long count) {} + +static int map_zero_page(pte_t *pte, unsigned long addr, void *arg) +{ + struct mm_struct *mm = (struct mm_struct *)arg; + struct page *page = alloc_page(GFP_KERNEL); + pte_t entry; + if (!page) + return -ENOMEM; + entry = mk_pte(page, PAGE_SHARED); + set_pte(pte, entry); + inc_mm_counter(mm, MM_ANONPAGES); + /* on x86 update_mmu_cache is nop */ + // update_mmu_cache(/* vma = */ NULL, addr, pte); + return 0; +} + +static int unmap_page(pte_t *pte, unsigned long addr, void *arg) +{ + struct mm_struct *mm = (struct mm_struct *)arg; + // struct page *page = pte_page(*pte); + // __free_page(page); + free_page((unsigned long)pte); + pte_clear(NULL, 0, pte); + pte_unmap(pte); + dec_mm_counter(mm, MM_ANONPAGES); + return 0; +} + +static int koord_cleanup_pmd(struct mm_struct *mm, unsigned long address) +{ + pgd_t *pgd; + p4d_t *p4d; + pud_t *pud; + pmd_t *pmd; + + pgd = pgd_offset(mm, address); + if (unlikely(pgd_none(*pgd) || pgd_bad(*pgd))) + return -EINVAL; + p4d = p4d_offset(pgd, address); + if (unlikely(p4d_none(*p4d) || p4d_bad(*p4d))) + return -EINVAL; + pud = pud_offset(p4d, address); + if (unlikely(pud_none(*pud)) || pud_bad(*pud)) + return -EINVAL; + pmd = pmd_offset(pud, address); + if (unlikely(pmd_none(*pmd))) + return -EINVAL; + pmd_clear(pmd); + mm_dec_nr_ptes(mm); + // pud_clear(pud); + // mm_dec_nr_ptes(mm); + // p4d_clear(p4d); + // mm_dec_nr_ptes(mm); + return 0; +} + +static int koord_register(pid_t pid) +{ + int ret = 0; + struct task_struct *p; + rcu_read_lock(); + p = koord_lookup_task(pid); + if (!p) + ret = -ESRCH; + rcu_read_unlock(); + return ret; +} + +static int koord_unregister(pid_t pid) +{ + return 0; +} + +static int koord_map_regions(struct koord_region_req __user *ureq) +{ + int ret = 0; + int i; + struct task_struct *t; + struct mm_struct *mm; + struct koord_region_req req; + ret = copy_from_user(&req, ureq, sizeof(req)); + if (unlikely(ret)) + return ret; + + rcu_read_lock(); + t = koord_lookup_task(req.pid); + if (!t) { + rcu_read_unlock(); + return -ESRCH; + } + mm = get_task_mm(t); + rcu_read_unlock(); + if (!mm) + goto fail_mmput; + + for (i = 0; i < req.nr; i++) { + uint64_t addr; + + ret = copy_from_user(&addr, &ureq->addrs[i], sizeof(addr)); + if (unlikely(ret)) + goto fail_mmput; + if (addr & HUGEPAGE_MASK) { + ret = -EINVAL; + goto fail_mmput; + } + ret = apply_to_page_range(mm, addr, req.region_size, + map_zero_page, mm); + if (ret) + goto fail_mmput; + } + +fail_mmput: + mmput(mm); + __flush_tlb_all(); + return ret; +} + +static int koord_unmap_regions(struct koord_region_req __user *ureq) +{ + int ret = 0; + int i; + struct task_struct *t; + struct mm_struct *mm; + struct koord_region_req req; + ret = copy_from_user(&req, ureq, sizeof(req)); + if (unlikely(ret)) + return ret; + + rcu_read_lock(); + t = koord_lookup_task(req.pid); + if (!t) { + rcu_read_unlock(); + return -ESRCH; + } + mm = get_task_mm(t); + if (!mm) + goto fail_mmput; + + for (i = 0; i < req.nr; i++) { + uint64_t addr; + ret = copy_from_user(&addr, &ureq->addrs[i], sizeof(addr)); + if (unlikely(ret)) + goto fail_mmput; + if (addr & HUGEPAGE_MASK) { + ret = -EINVAL; + goto fail_mmput; + } + ret = apply_to_page_range(mm, addr, req.region_size, unmap_page, + mm); + if (unlikely(ret)) + goto fail_mmput; + ret = koord_cleanup_pmd(mm, addr); + if (unlikely(ret)) + goto fail_mmput; + } + +fail_mmput: + mmput(mm); + rcu_read_unlock(); + __flush_tlb_all(); + return ret; +} + +static long koord_ioctl(struct file *file, unsigned int cmd, unsigned long arg) +{ + int ret = 0; + /* validate input */ + if (unlikely(_IOC_TYPE(cmd) != KOORD_MAGIC)) + return -ENOTTY; + if (unlikely(_IOC_NR(cmd) > KOORD_IOC_MAXNR)) + return -ENOTTY; + + switch (cmd) { + case KOORD_REG: + ret = koord_register((pid_t)arg); + break; + case KOORD_UNREG: + ret = koord_unregister((pid_t)arg); + break; + case KOORD_MAP: + ret = koord_map_regions((struct koord_region_req __user *)arg); + break; + case KOORD_UNMAP: + ret = koord_unmap_regions( + (struct koord_region_req __user *)arg); + break; + default: + return -EINVAL; + } + return ret; +} + +static int koord_open(struct inode *inode, struct file *filp) +{ + return 0; +} + +static int koord_release(struct inode *inode, struct file *filp) +{ + return 0; +} + +static struct file_operations koord_ops = { + .owner = THIS_MODULE, + .unlocked_ioctl = koord_ioctl, + .open = koord_open, + .release = koord_release, +}; + +static int __init koord_init(void) +{ + dev_t devno = MKDEV(KOORD_MAJOR, KOORD_MINOR); + int ret; + + ret = register_chrdev_region(devno, 1, "koord"); + if (ret) { + pr_err("Failed to allocate character device region\n"); + return ret; + } + + cdev_init(&koord_cdev, &koord_ops); + ret = cdev_add(&koord_cdev, devno, 1); + if (ret) { + pr_err("Failed to add character device\n"); + unregister_chrdev_region(devno, 1); + return -1; + } + + pr_info("koord: dev ready\n"); + return 0; +} + +static void __exit koord_exit(void) +{ + dev_t devno = MKDEV(KOORD_MAJOR, KOORD_MINOR); + + // Unregister the character device + cdev_del(&koord_cdev); + unregister_chrdev_region(devno, 1); + + pr_info("koord: unloaded\n"); +} + +module_init(koord_init); +module_exit(koord_exit); \ No newline at end of file diff --git a/koord/koord.h b/koord/koord.h new file mode 100644 index 0000000..fc38dbe --- /dev/null +++ b/koord/koord.h @@ -0,0 +1,36 @@ +/* + * koord.h - the UAPI for Midas coordinator and its ioctl's + */ + +#ifndef __KOORD__ +#define __KOORD__ + +#include +#include + +/* + * [Adapted from caladan] NOTE: normally character devices are dynamically + * allocated, but for convenience we can use 280. This value is zoned for + * "experimental and internal use". + */ +#define KOORD_MAJOR 280 +#define KOORD_MINOR 0 + +enum koord_op_t { KOORD_MAP = 0, KOORD_UNMAP = 1 }; + +struct koord_region_req { + pid_t pid; + int region_size; + int nr; + unsigned long addrs[]; +}; + +#define KOORD_MAGIC 0x1F +#define KOORD_IOC_MAXNR 4 + +#define KOORD_REG _IOW(KOORD_MAGIC, 1, pid_t) +#define KOORD_UNREG _IOW(KOORD_MAGIC, 2, pid_t) +#define KOORD_MAP _IOW(KOORD_MAGIC, 3, struct koord_region_req) +#define KOORD_UNMAP _IOW(KOORD_MAGIC, 4, struct koord_region_req) + +#endif // ___KOORD__ \ No newline at end of file diff --git a/koord/setup.sh b/koord/setup.sh new file mode 100755 index 0000000..1e2b117 --- /dev/null +++ b/koord/setup.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +rmmod koord +rm /dev/koord +insmod $(dirname $0)/build/koord.ko +mknod /dev/koord c 280 0 +chmod uga+rwx /dev/koord + diff --git a/koord/user.c b/koord/user.c new file mode 100644 index 0000000..75b315c --- /dev/null +++ b/koord/user.c @@ -0,0 +1,66 @@ +// userspace_program.c +#include +#include +#include +#include +#include +#include + +#define __user +#include "koord.h" + +void touch_region(unsigned long addr) { + volatile int *p = (volatile int *)addr; + p[0] = 1; +} + +int main() { + int fd = open("/dev/koord", O_RDWR); + if (fd < 0) { + perror("Failed to open character device"); + return 1; + } + + unsigned long addr = 0x01f000000000; + int region_size = 512 * 4096; + int nr = 1; + pid_t pid = getpid(); + + struct koord_region_req *req = malloc(sizeof(struct koord_region_req) + + sizeof(unsigned long) * nr); + req->pid = pid; + req->region_size = region_size; + req->nr = nr; + req->addrs[0] = addr; + printf("pid: %d\n", req->pid); + + // Register + if (ioctl(fd, KOORD_REG, pid) < 0) { + perror("Failed to register"); + } + + // Map + if (ioctl(fd, KOORD_MAP, req) < 0) { + perror("Failed to map region"); + } + + touch_region(addr); + + // Unmap + if (ioctl(fd, KOORD_UNMAP, req) < 0) { + perror("Failed to unmap region"); + } + + /* This will trigger seg fault */ + // touch_region(addr); + + // Unregister + if (ioctl(fd, KOORD_UNREG, pid) < 0) { + perror("Failed to unregister"); + } + + close(fd); + free(req); + + return 0; +} \ No newline at end of file diff --git a/scripts/build.sh b/scripts/build.sh new file mode 100755 index 0000000..760167f --- /dev/null +++ b/scripts/build.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +SRC_DIR=$( cd -- $SCRIPT_DIR/.. &> /dev/null && pwd ) + +# Build the Midas static library, the Midas coordinator (daemon), and unit tests. +pushd $SRC_DIR +make -j$(nproc) +popd + +# Build Midas C libraries and bindings +pushd $SRC_DIR/bindings/c +make -j +make install # this will install the lib into bindings/c/lib +popd diff --git a/scripts/gen_compile_commands.sh b/scripts/gen_compile_commands.sh new file mode 100755 index 0000000..adac497 --- /dev/null +++ b/scripts/gen_compile_commands.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +SRC_DIR=$( cd -- $SCRIPT_DIR/.. &> /dev/null && pwd ) + +make --always-make --dry-run \ + | grep -wE 'gcc|g\+\+' \ + | grep -w '\-c' \ + | jq -nR "[inputs|{directory:\"${SRC_DIR}\", command:., file: match(\" [^ ]+$\").string[1:]}]" \ + > compile_commands.json diff --git a/scripts/run_daemon.sh b/scripts/run_daemon.sh new file mode 100755 index 0000000..50e72aa --- /dev/null +++ b/scripts/run_daemon.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +SRC_DIR=$( cd -- $SCRIPT_DIR/.. &> /dev/null && pwd ) + +ulimit -n 1024000 +rm -rf /dev/shm/* +${SRC_DIR}/bin/daemon_main diff --git a/scripts/set_memory_limit.sh b/scripts/set_memory_limit.sh new file mode 100755 index 0000000..87a33fe --- /dev/null +++ b/scripts/set_memory_limit.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +SRC_DIR=$( cd -- $SCRIPT_DIR/.. &> /dev/null && pwd ) + +mem_mb=$1 +mem_limit=$(($mem_mb * 1024 * 1024)) + +echo "Set memory limit for Midas to $mem_mb MB" +echo $mem_limit > ${SRC_DIR}/config/mem.config diff --git a/src/cache_manager.cpp b/src/cache_manager.cpp new file mode 100644 index 0000000..d5881fd --- /dev/null +++ b/src/cache_manager.cpp @@ -0,0 +1,106 @@ +#include + +#include "cache_manager.hpp" +#include "shm_types.hpp" +#include "sig_handler.hpp" +#include "time.hpp" +#include "utils.hpp" + +namespace midas { +void CachePool::profile_stats(StatsMsg *msg) noexcept { + auto curr_ts = Time::get_us_stt(); + auto hit_ratio = static_cast(stats.hits) / (stats.hits + stats.misses); + auto miss_penalty = + stats.miss_bytes + ? (static_cast(stats.miss_cycles) / stats.miss_bytes) + : 0.0; + auto recon_time = + static_cast(stats.miss_cycles) / stats.misses / kCPUFreq; + auto victim_hit_ratio = static_cast(stats.hits + stats.victim_hits) / + (stats.hits + stats.victim_hits + stats.misses); + auto victim_hits = stats.victim_hits.load(); + auto perf_gain = victim_hits * miss_penalty; + + if (msg) { + msg->hits = stats.hits; + msg->misses = stats.misses; + msg->miss_penalty = miss_penalty; + msg->vhits = victim_hits; + } + + if (stats.hits > 0 || stats.misses > 0 || stats.victim_hits > 0) + MIDAS_LOG_PRINTF(kInfo, + "CachePool %s:\n" + "\t Region used: %ld/%ld\n" + "\tCache hit ratio: %.4f\n" + "\t miss penalty: %.2f\n" + "\t construct time: %.2f\n" + "\t hit counts: %lu\n" + "\t miss counts: %lu\n" + "\tVictim hit ratio: %.4f\n" + "\t hit count: %lu\n" + "\t perf gain: %.4f\n" + "\t count: %lu\n" + "\t size: %lu\n", + name_.c_str(), get_rmanager()->NumRegionInUse(), + get_rmanager()->NumRegionLimit(), hit_ratio, miss_penalty, + recon_time, stats.hits.load(), stats.misses.load(), + victim_hit_ratio, victim_hits, perf_gain, vcache_->count(), + vcache_->size()); + + stats.timestamp = curr_ts; + stats.reset(); +} + +inline void CachePool::CacheStats::reset() noexcept { + hits = 0; + misses = 0; + miss_cycles = 0; + miss_bytes = 0; + victim_hits = 0; +} + +CacheManager::CacheManager() : terminated_(false), profiler_(nullptr) { + // assert(create_pool(default_pool_name)); + auto sig_handler = SigHandler::global_sighandler(); + sig_handler->init(); +} + +StatsMsg CacheManager::profile_pools() { + StatsMsg stats{0}; + std::unique_lock ul(mtx_); + for (auto &[_, pool] : pools_) { + uint64_t curr_ts = Time::get_us_stt(); + uint64_t prev_ts = pool->stats.timestamp; + if (pool->stats.hits || pool->stats.misses) { + pool->profile_stats(&stats); + } + pool->stats.reset(); + pool->stats.timestamp = curr_ts; + } + ul.unlock(); + return stats; +} + +bool CacheManager::create_pool(std::string name) { + std::unique_lock ul(mtx_); + if (pools_.find(name) != pools_.cend()) { + MIDAS_LOG(kError) << "CachePool " << name << " has already been created!"; + return false; + } + auto pool = std::make_unique(name); + MIDAS_LOG(kInfo) << "Create cache pool " << name; + pools_[name] = std::move(pool); + return true; +} + +bool CacheManager::delete_pool(std::string name) { + std::unique_lock ul(mtx_); + if (pools_.find(name) == pools_.cend()) { + MIDAS_LOG(kError) << "CachePool " << name << " has already been deleted!"; + return false; + } + pools_.erase(name); + return true; +} +} // namespace midas \ No newline at end of file diff --git a/src/evacuator.cpp b/src/evacuator.cpp new file mode 100644 index 0000000..88b89ff --- /dev/null +++ b/src/evacuator.cpp @@ -0,0 +1,612 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cache_manager.hpp" +#include "evacuator.hpp" +#include "log.hpp" +#include "logging.hpp" +#include "object.hpp" +#include "resource_manager.hpp" +#include "time.hpp" +#include "utils.hpp" + +namespace midas { +void Evacuator::init() { + gc_thd_ = std::make_shared([&]() { + while (!terminated_) { + { + std::unique_lock lk(gc_mtx_); + gc_cv_.wait( + lk, [this] { return terminated_ || rmanager_->reclaim_trigger(); }); + } + auto succ = false; + auto nr_evac_thds = rmanager_->reclaim_nr_thds(); + if (nr_evac_thds == 1) { + succ = serial_gc(); + } else { + for (int i = 0; i < 3; i++) { + succ = parallel_gc(nr_evac_thds); + if (succ) + break; + } + } + + if (!succ) + force_reclaim(); + } + }); +} + +int64_t Evacuator::gc(SegmentList &stash_list) { + if (!rmanager_->reclaim_trigger()) + return 0; + + int nr_skipped = 0; + int nr_scanned = 0; + int nr_evaced = 0; + auto &segments = allocator_->segments_; + + auto stt = chrono_utils::now(); + while (rmanager_->reclaim_trigger()) { + auto segment = segments.pop_front(); + if (!segment) { + nr_skipped++; + if (nr_skipped > rmanager_->NumRegionLimit()) + return -1; + continue; + } + if (!segment->sealed()) { // put in-used segment back to list + segments.push_back(segment); + nr_skipped++; + if (nr_skipped > rmanager_->NumRegionLimit()) { // be in loop for too long + MIDAS_LOG(kDebug) << "Encountered too many unsealed segments during " + "GC, skip GC this round."; + return -1; + } + continue; + } + EvacState ret = scan_segment(segment.get(), true); + nr_scanned++; + if (ret == EvacState::Fault) + continue; + else if (ret == EvacState::Fail) + goto put_back; + // must have ret == EvacState::Succ now + + if (segment->get_alive_ratio() >= kAliveThreshHigh) + goto put_back; + ret = evac_segment(segment.get()); + if (ret == EvacState::Fail) + goto put_back; + else if (ret == EvacState::DelayRelease) + goto stash; + // must have ret == EvacState::Succ or ret == EvacState::Fault now + nr_evaced++; + continue; + put_back: + segments.push_back(segment); + continue; + stash: + stash_list.push_back(segment); + continue; + } + auto end = chrono_utils::now(); + + auto nr_avail = rmanager_->NumRegionAvail(); + // assert(nr_avail > 0); + + if (nr_scanned) + MIDAS_LOG(kDebug) << "GC: " << nr_scanned << " scanned, " << nr_evaced + << " evacuated, " << nr_avail << " available (" + << chrono_utils::duration(stt, end) << "s)."; + + return nr_avail; +} + +bool Evacuator::serial_gc() { + if (!rmanager_->reclaim_trigger()) + return 0; + + int64_t nr_skipped = 0; + int64_t nr_scanned = 0; + int64_t nr_evaced = 0; + auto &segments = allocator_->segments_; + + auto stt = chrono_utils::now(); + while (rmanager_->reclaim_trigger()) { + auto segment = segments.pop_front(); + if (!segment) { + nr_skipped++; + if (nr_skipped > rmanager_->NumRegionLimit()) // be in loop for too long + goto done; + continue; + } + if (!segment->sealed()) { // put in-used segment back to list + segments.push_back(segment); + nr_skipped++; + if (nr_skipped > rmanager_->NumRegionLimit()) { // be in loop for too long + MIDAS_LOG(kDebug) << "Encountered too many unsealed segments during " + "GC, skip GC this round."; + goto done; + } + continue; + } + EvacState ret = scan_segment(segment.get(), true); + nr_scanned++; + if (ret == EvacState::Fault) + continue; + else if (ret == EvacState::Fail) + goto put_back; + // must have ret == EvacState::Succ now + assert(ret == EvacState::Succ); + + if (segment->get_alive_ratio() >= kAliveThreshHigh) + goto put_back; + ret = evac_segment(segment.get()); + if (ret == EvacState::Fail) + goto put_back; + else if (ret == EvacState::DelayRelease) { + // MIDAS_LOG(kWarning); + segment->destroy(); + } + // must have ret == EvacState::Succ or ret == EvacState::Fault now + nr_evaced++; + continue; + put_back: + segments.push_back(segment); + continue; + } + +done: + auto end = chrono_utils::now(); + auto nr_avail = rmanager_->NumRegionAvail(); + + if (nr_scanned) + MIDAS_LOG(kDebug) << "GC: " << nr_scanned << " scanned, " << nr_evaced + << " evacuated, " << nr_avail << " available (" + << chrono_utils::duration(stt, end) << "s)."; + + return nr_avail >= 0; +} + +bool Evacuator::parallel_gc(int nr_workers) { + auto stt = chrono_utils::now(); + rmanager_->prof_reclaim_stt(); + + SegmentList stash_list; + std::atomic_int nr_failed{0}; + std::vector gc_thds; + for (int tid = 0; tid < nr_workers; tid++) { + gc_thds.push_back(std::thread([&, tid = tid]() { + if (gc(stash_list) < 0) + nr_failed++; + })); + } + + for (auto &thd : gc_thds) + thd.join(); + gc_thds.clear(); + + auto &segments = allocator_->segments_; + while (!stash_list.empty()) { + auto segment = stash_list.pop_front(); + // segment->destroy(); + EvacState ret = evac_segment(segment.get()); + if (ret == EvacState::DelayRelease) + segments.push_back(segment); + else if (ret != EvacState::Succ) { + MIDAS_LOG(kError) << (int)ret; + segments.push_back(segment); + } + } + + auto end = chrono_utils::now(); + rmanager_->prof_reclaim_end(nr_workers, chrono_utils::duration(stt, end)); + return rmanager_->NumRegionAvail() >= 0; +} + +int64_t Evacuator::force_reclaim() { + if (!kEnableFaultHandler) + return 0; + + auto stt = chrono_utils::now(); + rmanager_->prof_reclaim_stt(); + auto nr_workers = kNumEvacThds; + + int64_t nr_reclaimed = 0; + std::vector thds; + for (int i = 0; i < nr_workers; i++) { + thds.emplace_back([&] { + auto &segments = allocator_->segments_; + while (rmanager_->NumRegionAvail() <= 0) { + auto segment = segments.pop_front(); + if (!segment) + break; + if (segment.use_count() != 1) { + MIDAS_LOG(kError) << segment << " " << segment.use_count(); + } + assert(segment.use_count() <= 2); + segment->destroy(); + nr_reclaimed++; + } + }); + } + for (auto &thd : thds) + thd.join(); + auto end = chrono_utils::now(); + rmanager_->prof_reclaim_end(nr_workers, chrono_utils::duration(stt, end)); + + if (nr_reclaimed) + MIDAS_LOG(kDebug) << "GC: " << nr_reclaimed << " force reclaimed (" + << chrono_utils::duration(stt, end) << "s). "; + + return nr_reclaimed; +} + +/** util functions */ +inline bool Evacuator::segment_ready(LogSegment *segment) { + return segment->sealed() && !segment->destroyed(); +} + +/** Evacuate a particular segment */ +inline RetCode Evacuator::iterate_segment(LogSegment *segment, uint64_t &pos, + ObjectPtr &optr) { + if (pos + sizeof(MetaObjectHdr) > segment->pos_) + return ObjectPtr::RetCode::Fail; + + auto ret = optr.init_from_soft(TransientPtr(pos, sizeof(LargeObjectHdr))); + if (ret == RetCode::Succ) + pos += optr.obj_size(); // header size is already counted + return ret; +} + +inline EvacState Evacuator::scan_segment(LogSegment *segment, bool deactivate) { + if (!segment_ready(segment)) + return EvacState::Fail; + segment->set_alive_bytes(kMaxAliveBytes); + + int alive_bytes = 0; + // counters + int nr_present = 0; + int nr_deactivated = 0; + int nr_non_present = 0; + int nr_freed = 0; + int nr_small_objs = 0; + int nr_large_objs = 0; + int nr_faulted = 0; + int nr_contd_objs = 0; + + ObjectPtr obj_ptr; + + auto pos = segment->start_addr_; + RetCode ret = RetCode::Fail; + while ((ret = iterate_segment(segment, pos, obj_ptr)) == RetCode::Succ) { + auto lock_id = obj_ptr.lock(); + assert(lock_id != -1 && !obj_ptr.null()); + if (obj_ptr.is_small_obj()) { + nr_small_objs++; + + auto obj_size = obj_ptr.obj_size(); + MetaObjectHdr meta_hdr; + if (!load_hdr(meta_hdr, obj_ptr)) + goto faulted; + else { + if (meta_hdr.is_present()) { + nr_present++; + if (!deactivate) { + alive_bytes += obj_size; + } else if (meta_hdr.is_accessed()) { + meta_hdr.dec_accessed(); + if (!store_hdr(meta_hdr, obj_ptr)) + goto faulted; + nr_deactivated++; + alive_bytes += obj_size; + } else { + auto rref = reinterpret_cast(obj_ptr.get_rref()); + // assert(rref); + // if (!rref) + // MIDAS_LOG(kError) << "null rref detected"; + auto ret = obj_ptr.free(/* locked = */ true); + if (ret == RetCode::FaultLocal) + goto faulted; + // small objs are impossible to fault on other regions + assert(ret != RetCode::FaultOther); + if (rref && !rref->is_victim()) { + auto vcache = pool_->get_vcache(); + vcache->put(rref, nullptr); + } + nr_freed++; + } + } else + nr_non_present++; + } + } else { // large object + nr_large_objs++; + MetaObjectHdr meta_hdr; + if (!load_hdr(meta_hdr, obj_ptr)) + goto faulted; + else { + auto obj_size = obj_ptr.obj_size(); // only partial size here! + if (meta_hdr.is_present()) { + nr_present++; + if (!meta_hdr.is_continue()) { // head segment + if (!deactivate) { + alive_bytes += obj_size; + } else if (meta_hdr.is_accessed()) { + meta_hdr.dec_accessed(); + if (!store_hdr(meta_hdr, obj_ptr)) + goto faulted; + nr_deactivated++; + alive_bytes += obj_size; + } else { + auto rref = reinterpret_cast(obj_ptr.get_rref()); + // assert(rref); + // if (!rref) + // MIDAS_LOG(kError) << "null rref detected"; + // This will free all segments belonging to the same object + auto ret = obj_ptr.free(/* locked = */ true); + if (ret == RetCode::FaultLocal) + goto faulted; + // do nothing when ret == FaultOther and continue scanning + if (rref && !rref->is_victim()) { + auto vcache = pool_->get_vcache(); + vcache->put(rref, nullptr); + } + + nr_freed++; + } + } else { // continued segment + // An inner segment of a large object. Skip it. + LargeObjectHdr lhdr; + if (!load_hdr(lhdr, obj_ptr)) + goto faulted; + auto head = lhdr.get_head(); + MetaObjectHdr head_hdr; + if (head.null() || !load_hdr(head_hdr, head) || + !head_hdr.is_valid() || !head_hdr.is_present()) { + nr_freed++; + } else { + alive_bytes += obj_size; + nr_contd_objs++; + } + } + } else + nr_non_present++; + } + } + obj_ptr.unlock(lock_id); + continue; + faulted: + nr_faulted++; + obj_ptr.unlock(lock_id); + break; + } + + if (!kEnableFaultHandler) + assert(nr_faulted == 0); + if (deactivate) // meaning this is a scanning thread + LogAllocator::count_alive(nr_present); + MIDAS_LOG(kDebug) << "nr_scanned_small_objs: " << nr_small_objs + << ", nr_large_objs: " << nr_large_objs + << ", nr_non_present: " << nr_non_present + << ", nr_deactivated: " << nr_deactivated + << ", nr_freed: " << nr_freed + << ", nr_faulted: " << nr_faulted << ", alive ratio: " + << static_cast(segment->alive_bytes_) / + kLogSegmentSize; + + assert(ret != RetCode::FaultOther); + if (ret == RetCode::FaultLocal || nr_faulted) { + if (!kEnableFaultHandler) + MIDAS_LOG(kError) << "segment is unmapped under the hood"; + segment->destroy(); + return EvacState::Fault; + } + segment->set_alive_bytes(alive_bytes); + return EvacState::Succ; +} + +inline EvacState Evacuator::evac_segment(LogSegment *segment) { + if (!segment_ready(segment)) + return EvacState::Fail; + + // counters + int nr_present = 0; + int nr_freed = 0; + int nr_moved = 0; + int nr_small_objs = 0; + int nr_failed = 0; + int nr_faulted = 0; + int nr_contd_objs = 0; + + ObjectPtr obj_ptr; + auto pos = segment->start_addr_; + RetCode ret = RetCode::Fail; + while ((ret = iterate_segment(segment, pos, obj_ptr)) == RetCode::Succ) { + auto lock_id = obj_ptr.lock(); + assert(lock_id != -1 && !obj_ptr.null()); + MetaObjectHdr meta_hdr; + if (!load_hdr(meta_hdr, obj_ptr)) + goto faulted; + if (!meta_hdr.is_present()) { + nr_freed++; + obj_ptr.unlock(lock_id); + continue; + } + nr_present++; + if (obj_ptr.is_small_obj()) { + nr_small_objs++; + obj_ptr.unlock(lock_id); + auto optptr = allocator_->alloc_(obj_ptr.data_size_in_segment(), true); + lock_id = optptr->lock(); + assert(lock_id != -1 && !obj_ptr.null()); + + if (optptr) { + auto new_ptr = *optptr; + auto ret = new_ptr.move_from(obj_ptr); + if (ret == RetCode::Succ) { + nr_moved++; + } else if (ret == RetCode::Fail) { + // MIDAS_LOG(kError) << "Failed to move the object!"; + nr_failed++; + } else + goto faulted; + } else + nr_failed++; + } else { // large object + if (!meta_hdr.is_continue()) { // the head segment of a large object. + auto opt_data_size = obj_ptr.large_data_size(); + if (!opt_data_size) { + // MIDAS_LOG(kWarning); + nr_freed++; + obj_ptr.unlock(lock_id); + continue; + } + obj_ptr.unlock(lock_id); + auto optptr = allocator_->alloc_(*opt_data_size, true); + lock_id = obj_ptr.lock(); + assert(lock_id != -1 && !obj_ptr.null()); + + if (optptr) { + auto new_ptr = *optptr; + auto ret = new_ptr.move_from(obj_ptr); + if (ret == RetCode::Succ) { + nr_moved++; + } else if (ret == RetCode::Fail) { + // MIDAS_LOG(kError) << "Failed to move the object!"; + nr_failed++; + } else if (ret == RetCode::FaultLocal) { // fault on src (obj_ptr) + if (!kEnableFaultHandler) + MIDAS_LOG(kWarning); + goto faulted; + } + // skip when ret == FaultOther, meaning that fault is on dst (new_ptr) + } else + nr_failed++; + } else { // an inner segment of a large obj + LargeObjectHdr lhdr; + if (!load_hdr(lhdr, obj_ptr)) + goto faulted; + auto head = lhdr.get_head(); + MetaObjectHdr head_hdr; + if (head.null() || !load_hdr(head_hdr, head) || !head_hdr.is_valid() || + !head_hdr.is_present()) { + nr_freed++; + } else { + nr_contd_objs++; + } + } + } + obj_ptr.unlock(lock_id); + continue; + faulted: + nr_faulted++; + obj_ptr.unlock(lock_id); + break; + } + MIDAS_LOG(kDebug) << "nr_present: " << nr_present + << ", nr_moved: " << nr_moved << ", nr_freed: " << nr_freed + << ", nr_failed: " << nr_failed; + + if (!kEnableFaultHandler) + assert(nr_faulted == 0); + assert(ret != RetCode::FaultOther); + if (ret == RetCode::FaultLocal || nr_faulted) { + if (!kEnableFaultHandler) + MIDAS_LOG(kError) << "segment is unmapped under the hood"; + segment->destroy(); + return EvacState::Fault; + } + if (nr_contd_objs) + return EvacState::DelayRelease; + segment->destroy(); + return EvacState::Succ; +} + +inline EvacState Evacuator::free_segment(LogSegment *segment) { + if (!segment_ready(segment)) + return EvacState::Fail; + + // counters + int nr_non_present = 0; + int nr_freed = 0; + int nr_small_objs = 0; + int nr_faulted = 0; + int nr_contd_objs = 0; + + ObjectPtr obj_ptr; + auto pos = segment->start_addr_; + RetCode ret = RetCode::Fail; + while ((ret = iterate_segment(segment, pos, obj_ptr)) == RetCode::Succ) { + auto lock_id = obj_ptr.lock(); + assert(lock_id != -1 && !obj_ptr.null()); + if (obj_ptr.is_small_obj()) { + nr_small_objs++; + + MetaObjectHdr meta_hdr; + if (!load_hdr(meta_hdr, obj_ptr)) + goto faulted; + else { + if (meta_hdr.is_present()) { + auto ret = obj_ptr.free(/* locked = */ true); + if (ret == RetCode::FaultLocal) + goto faulted; + assert(ret != RetCode::FaultOther); + nr_freed++; + } else + nr_non_present++; + } + } else { // large object + MetaObjectHdr meta_hdr; + if (!load_hdr(meta_hdr, obj_ptr)) + goto faulted; + else { + if (!meta_hdr.is_continue()) { // head segment + if (meta_hdr.is_present()) { + auto ret = obj_ptr.free(/* locked = */ true); + if (ret == RetCode::FaultLocal) + goto faulted; + // skip the object and continue when ret == FaultOther + nr_freed++; + } else + nr_non_present++; + } else { // continued segment + nr_contd_objs++; + } + } + } + obj_ptr.unlock(lock_id); + continue; + faulted: + nr_faulted++; + obj_ptr.unlock(lock_id); + break; + } + MIDAS_LOG(kDebug) << "nr_freed: " << nr_freed + << ", nr_non_present: " << nr_non_present + << ", nr_faulted: " << nr_faulted; + + if (!kEnableFaultHandler) + assert(nr_faulted == 0); + assert(ret != RetCode::FaultOther); + if (ret == RetCode::FaultLocal || nr_faulted) { + if (!kEnableFaultHandler) + MIDAS_LOG(kError) << "segment is unmapped under the hood"; + segment->destroy(); + return EvacState::Fault; + } + if (nr_contd_objs) + return EvacState::DelayRelease; + segment->destroy(); + return EvacState::Succ; +} + +} // namespace midas \ No newline at end of file diff --git a/src/fs_shim.cpp b/src/fs_shim.cpp new file mode 100644 index 0000000..f8beb1b --- /dev/null +++ b/src/fs_shim.cpp @@ -0,0 +1,490 @@ +#include "fs_shim.hpp" +#include "logging.hpp" +#include "utils.hpp" + +#ifdef HIJACK_FS_SYSCALLS + +extern "C" { +namespace midas { +/** Intercept fs-related syscalls */ +int open(const char *pathname, int flags, mode_t mode) { + auto shim = FSShim::global_shim(); + assert(shim->open != NULL); + + int fd = shim->open(pathname, flags, mode); + if (fd == -1) + return fd; + if (is_shm_file(pathname)) { + shim->exclude_interpose(fd); + return fd; + } + MIDAS_LOG_PRINTF(kDebug, "open(pathname=%s, flags=0x%x, mode=0%o) = %d\n", + pathname, flags, mode, fd); + assert(shim->on_open(fd)); + return fd; +} + +int open64(const char *pathname, int flags, mode_t mode) { + auto shim = FSShim::global_shim(); + assert(shim->open64 != NULL); + + int fd = shim->open64(pathname, flags, mode); + if (fd == -1) + return fd; + if (is_shm_file(pathname)) { + shim->exclude_interpose(fd); + return fd; + } + MIDAS_LOG_PRINTF(kDebug, "open64(pathname=%s, flags=0x%x, mode=0%o) = %d\n", + pathname, flags, mode, fd); + assert(shim->on_open(fd)); + return fd; +} + +int creat(const char *pathname, int flags, mode_t mode) { + auto shim = FSShim::global_shim(); + assert(shim->creat != NULL); + + int fd = shim->creat(pathname, flags, mode); + if (fd == -1) + return fd; + if (is_shm_file(pathname)) { + shim->exclude_interpose(fd); + return fd; + } + MIDAS_LOG_PRINTF(kDebug, "creat(pathname=%s, flags=0x%x, mode=0%o) = %d\n", + pathname, flags, mode, fd); + return fd; +} + +int creat64(const char *pathname, int flags, mode_t mode) { + auto shim = FSShim::global_shim(); + assert(shim->creat64 != NULL); + + int fd = shim->creat64(pathname, flags, mode); + assert(fd != -1); + MIDAS_LOG_PRINTF(kDebug, "creat64(pathname=%s, flags=0x%x, mode=0%o) = %d\n", + pathname, flags, mode, fd); + return fd; +} + +int openat(int dirfd, const char *pathname, int flags, mode_t mode) { + auto shim = FSShim::global_shim(); + assert(shim->openat != NULL); + + int fd = shim->openat(dirfd, pathname, flags, mode); + if (fd == -1) + return fd; + if (is_shm_file(pathname)) { + shim->exclude_interpose(fd); + return fd; + } + MIDAS_LOG_PRINTF(kDebug, + "openat(dirfd=%d, pathname=%s, flags=0x%x, mode=0%o) = %d\n", + dirfd, pathname, flags, mode, fd); + assert(shim->on_open(fd)); + return fd; +} + +int openat64(int dirfd, const char *pathname, int flags, mode_t mode) { + auto shim = FSShim::global_shim(); + assert(shim->openat64 != NULL); + + int fd = shim->openat64(dirfd, pathname, flags, mode); + if (fd == -1) + return fd; + if (is_shm_file(pathname)) { + shim->exclude_interpose(fd); + return fd; + } + MIDAS_LOG_PRINTF( + kDebug, "openat64(dirfd=%d, pathname=%s, flags=0x%x, mode=0%o) = %d\n", + dirfd, pathname, flags, mode, fd); + assert(shim->on_open(fd)); + return fd; +} + +int dup(int oldfd) { + auto shim = FSShim::global_shim(); + assert(shim->dup != NULL); + + int fd = shim->dup(oldfd); + if (fd == -1) + return fd; + MIDAS_LOG_PRINTF(kDebug, "dup(oldfd=%d) = %d\n", oldfd, fd); + return fd; +} + +int dup2(int oldfd, int newfd) { + auto shim = FSShim::global_shim(); + assert(shim->dup2 != NULL); + + /* if newfd is already opened, the kernel will close it directly + * once dup2 is invoked. So now is the last chance to mark the + * pages as "DONTNEED" */ + // if (valid_fd(newfd)) + // free_unclaimed_pages(newfd, true); + + int ret = shim->dup2(oldfd, newfd); + if (ret == -1) + return ret; + MIDAS_LOG_PRINTF(kDebug, "dup2(oldfd=%d, newfd=%d) = %d\n", oldfd, newfd, + ret); + return ret; +} + +int close(int fd) { + auto shim = FSShim::global_shim(); + assert(shim->close != NULL); + + if (shim->is_excluded(fd)) + shim->reset_interpose(fd); + else if (fd > 0) + assert(shim->on_close(fd)); + int ret = shim->close(fd); + if (ret == -1) + return ret; + MIDAS_LOG_PRINTF(kDebug, "close(%d) = %d\n", fd, ret); + return ret; +} + +FILE *fopen(const char *path, const char *mode) { + auto shim = FSShim::global_shim(); + assert(shim->fopen != NULL); + + FILE *fp = shim->fopen(path, mode); + if (fp == nullptr) + return fp; + int fd = fileno(fp); + if (fd == -1) + return fp; + if (is_shm_file(path)) { + shim->exclude_interpose(fd); + return fp; + } + MIDAS_LOG_PRINTF(kDebug, "fopen(path=%s, mode=%s) = %p\n", path, mode, fp); + assert(shim->on_open(fd)); + + return fp; +} + +FILE *fopen64(const char *path, const char *mode) { + auto shim = FSShim::global_shim(); + assert(shim->fopen64 != NULL); + + FILE *fp = shim->fopen64(path, mode); + if (fp == nullptr) + return fp; + int fd = fileno(fp); + if (fd == -1) + return fp; + if (is_shm_file(path)) { + shim->exclude_interpose(fd); + return fp; + } + MIDAS_LOG_PRINTF(kDebug, "fopen64(path=%s, mode=%s) = %p\n", path, mode, fp); + assert(shim->on_open(fd)); + + return fp; +} + +int fclose(FILE *fp) { + auto shim = FSShim::global_shim(); + assert(shim->close != NULL); + + int fd = fileno(fp); + if (shim->is_excluded(fd)) + shim->reset_interpose(fd); + else if (fd > 0) + assert(shim->on_close(fd)); + int ret = shim->fclose(fp); + if (ret != 0) + return ret; + MIDAS_LOG_PRINTF(kDebug, "fclose(fp=%p) = %d\n", fp, ret); + + return ret; +} + +ssize_t read(int fd, void *buf, size_t count) { + auto shim = FSShim::global_shim(); + assert(shim->read != NULL); + if (shim->is_excluded(fd)) + return shim->read(fd, buf, count); + + ssize_t ret_count = shim->on_read(fd, buf, count, true, 0); + // assert(ret_count >= 0); + if (ret_count < 0) + ret_count = shim->read(fd, buf, count); + MIDAS_LOG_PRINTF(kDebug, "read(fd=%d, buf=%p, count=%ld) = %ld\n", fd, buf, + count, ret_count); + return ret_count; +} + +ssize_t write(int fd, const void *buf, size_t count) { + auto shim = FSShim::global_shim(); + assert(shim->write != NULL); + if (shim->is_excluded(fd)) + return shim->write(fd, buf, count); + + ssize_t ret_count = shim->on_write(fd, buf, count, true, 0); + // assert(ret_count >= 0); + if (ret_count < 0) + ret_count = shim->write(fd, buf, count); + MIDAS_LOG_PRINTF(kDebug, "write(fd=%d, buf=%p, count=%ld) = %ld\n", fd, buf, + count, ret_count); + return ret_count; +} + +ssize_t pread(int fd, void *buf, size_t count, off_t offset) { + auto shim = FSShim::global_shim(); + assert(shim->pread != NULL); + if (shim->is_excluded(fd)) + return shim->pread(fd, buf, count, offset); + + ssize_t ret_count = shim->on_read(fd, buf, count, false, offset); + // assert(ret_count >= 0); + if (ret_count < 0) + ret_count = shim->pread(fd, buf, count, offset); + MIDAS_LOG_PRINTF(kInfo, "pread(fd=%d, buf=%p, count=%ld, offset=%lu) = %ld\n", + fd, buf, count, offset, ret_count); + return ret_count; +} + +ssize_t pwrite(int fd, const void *buf, size_t count, off_t offset) { + auto shim = FSShim::global_shim(); + assert(shim->pwrite != NULL); + if (shim->is_excluded(fd)) + return shim->pwrite(fd, buf, count, offset); + + ssize_t ret_count = shim->on_write(fd, buf, count, false, offset); + // assert(ret_count >= 0); + if (ret_count < 0) + ret_count = shim->pwrite(fd, buf, count, offset); + MIDAS_LOG_PRINTF(kInfo, + "pwrite(fd=%d, buf=%p, count=%ld, offset=%lu) = %ld\n", fd, + buf, count, offset, ret_count); + return ret_count; +} + +size_t fread(void *ptr, size_t size, size_t nmemb, FILE *stream) { + auto shim = FSShim::global_shim(); + assert(shim->fread != NULL); + int fd = fileno(stream); + if (shim->is_excluded(fd)) + return shim->fread(ptr, size, nmemb, stream); + + // on_read() returns #(eles) rather than total len + ssize_t ret_nmemb = shim->on_read(fd, ptr, size * nmemb, true, 0) / size; + assert(ret_nmemb >= 0); + if (ret_nmemb < 0) + ret_nmemb = shim->fread(ptr, size, nmemb, stream); + MIDAS_LOG_PRINTF(kDebug, + "fread(ptr=%p, size=%lu, nmemb=%lu, stream=%p) = %lu\n", ptr, + size, nmemb, stream, ret_nmemb); + return ret_nmemb; +} + +size_t fwrite(const void *ptr, size_t size, size_t nmemb, FILE *stream) { + auto shim = FSShim::global_shim(); + assert(shim->fwrite != NULL); + int fd = fileno(stream); + if (shim->is_excluded(fd)) + return shim->fwrite(ptr, size, nmemb, stream); + + // on_write() returns #(eles) rather than total len + ssize_t ret_nmemb = shim->on_write(fd, ptr, size * nmemb, true, 0) / size; + assert(ret_nmemb >= 0); + if (ret_nmemb < 0) + ret_nmemb = shim->fwrite(ptr, size, nmemb, stream); + MIDAS_LOG_PRINTF(kDebug, + "fwrite(ptr=%p, size=%lu, nmemb=%lu, stream=%p) = %lu\n", + ptr, size, nmemb, stream, ret_nmemb); + return ret_nmemb; +} + +off_t lseek(int fd, off_t offset, int whence) { + auto shim = FSShim::global_shim(); + assert(shim->lseek != NULL); + if (shim->is_excluded(fd)) + return shim->lseek(fd, offset, whence); + + off_t abs_offset = shim->lseek(fd, offset, whence); + shim->on_lseek(fd, abs_offset); + MIDAS_LOG_PRINTF(kDebug, "lseek(fd=%d, offset=%lu, whence=%d) = %lu\n", fd, + offset, whence, abs_offset); + return abs_offset; +} + +/** Implementation of page cache for files */ +inline bool FSShim::on_open(int fd) { + ino_t inode = get_inode(fd); + if (invalid_inode(inode)) + return false; + + std::unique_lock ul(mtx_); + init_cache(); + auto iter = files_.find(inode); + if (iter != files_.cend()) { + iter->second.offset = 0; + return true; + } + files_[inode] = File(pool_); + MIDAS_LOG(kDebug) << "create cache for file " << inode; + return true; +} + +inline bool FSShim::on_close(int fd) { + ino_t inode = get_inode(fd); + if (invalid_inode(inode)) + return false; + + std::unique_lock ul(mtx_); + init_cache(); + auto iter = files_.find(inode); + if (iter == files_.cend()) { + return false; + } + auto &file = iter->second; + file.offset = 0; + return true; +} + +inline ssize_t FSShim::on_read(int fd, void *buf, size_t count, bool upd_offset, + off_t offset) { + ino_t inode = get_inode(fd); + if (invalid_inode(inode)) + return -1; + + ssize_t ret_count = 0; + std::unique_lock ul(mtx_); + init_cache(); + auto iter = files_.find(inode); + if (iter == files_.cend()) { + MIDAS_LOG_PRINTF(kWarning, "file %lu doesn't exist!\n", inode); + return -1; + } + ul.unlock(); + auto &file = iter->second; // page cache for this file + const off_t prev_off = this->lseek(fd, 0, SEEK_CUR); + if (file.offset != prev_off) { // FIX: figure out why mismatch can happen + MIDAS_LOG(kError) << file.offset << " != " << prev_off; + file.offset = prev_off; + } + + char *dst_cursor = reinterpret_cast(buf); + size_t remaining_cnt = count; + auto curr_off = prev_off + offset; + int32_t off_in_pg = curr_off % kPageSize; // may > 0 for the first page + while (remaining_cnt > 0) { + pfn_t pg = curr_off / kPageSize; + ssize_t cnt_rd = 0; + ssize_t cnt_in_pg = + std::min(off_in_pg + remaining_cnt, kPageSize) - off_in_pg; + auto page = file.cache->get(pg); + if (page) { + auto src_cursor = reinterpret_cast(page.get()) + off_in_pg; + std::memcpy(dst_cursor, src_cursor, cnt_in_pg); + cnt_rd = cnt_in_pg; + } else { // cache miss path + auto stt = Time::get_cycles_stt(); + cnt_rd = this->pread(fd, dst_cursor, cnt_in_pg, curr_off); + + assert(off_in_pg + cnt_rd <= kPageSize); + if (off_in_pg + cnt_rd == kPageSize) { // only cache full pages + Page page; + this->pread(fd, page, kPageSize, curr_off - off_in_pg); + assert((curr_off - off_in_pg) % kPageSize == 0); + file.cache->set(pg, page); + auto end = Time::get_cycles_end(); + pool_->record_miss_penalty(end - stt, kPageSize); + } + } + ret_count += cnt_rd; + curr_off += cnt_rd; + dst_cursor += cnt_rd; + remaining_cnt -= cnt_rd; + off_in_pg = 0; // reset off_in_pg after the first page + if (cnt_rd == -1) // failed to read + goto failed; + if (cnt_rd < cnt_in_pg) + break; + } + if (upd_offset) { // adjust file offset + assert(offset == 0); + off_t file_off = this->lseek(fd, ret_count, SEEK_CUR); + assert(file_off == curr_off); + file.offset = curr_off; + } + return ret_count; + +failed: + return -1; +} + +inline ssize_t FSShim::on_write(int fd, const void *buf, size_t count, + bool upd_offset, off_t offset) { + ino_t inode = get_inode(fd); + if (invalid_inode(inode)) + return -1; + + std::unique_lock ul(mtx_); + auto iter = files_.find(inode); + if (iter == files_.cend()) { + MIDAS_LOG_PRINTF(kWarning, "file %lu doesn't exist!\n", inode); + return false; + } + auto &file = iter->second; + const off_t prev_off = this->lseek(fd, 0, SEEK_CUR); + if (file.offset != prev_off) { // FIX: figure out why mismatch can happen + MIDAS_LOG(kError) << file.offset << prev_off; + file.offset = prev_off; + } + + auto curr_off = prev_off + offset; + ssize_t ret_count = this->pwrite(fd, buf, count, curr_off); + if (ret_count == -1) // failed to write + goto failed; + + // Evict all pages from cache for now. Ideally we should cache dirty + // pages and evict them in fsync(). + for (pfn_t pg = curr_off / kPageSize; + pg <= (curr_off + ret_count) / kPageSize; pg++) + file.cache->remove(pg); + + if (upd_offset) { + assert(offset == 0); + auto file_off = this->lseek(fd, ret_count, SEEK_CUR); + curr_off += ret_count; + if (curr_off != file_off) { + MIDAS_LOG(kError) << curr_off << " " << file_off << " " << prev_off << " " + << ret_count; + } + assert(file_off == curr_off); + file.offset = curr_off; + } + return ret_count; + +failed: + return -1; +} + +inline bool FSShim::on_lseek(int fd, off_t abs_offset) { + ino_t inode = get_inode(fd); + if (invalid_inode(inode)) + return false; + + std::unique_lock ul(mtx_); + auto iter = files_.find(inode); + if (iter == files_.cend()) { + MIDAS_LOG_PRINTF(kWarning, "file %lu doesn't exist!\n", inode); + return false; + } + auto &file = iter->second; + file.offset = abs_offset; + return true; +} + +} // namespace midas +} // extern "C" + +#endif // HIJACK_FS_SYSCALLS \ No newline at end of file diff --git a/src/log.cpp b/src/log.cpp new file mode 100644 index 0000000..8499a1b --- /dev/null +++ b/src/log.cpp @@ -0,0 +1,260 @@ +#include +#include +#include +#include +#include +#include +#include + +#include "cache_manager.hpp" +#include "evacuator.hpp" +#include "log.hpp" +#include "logging.hpp" +#include "object.hpp" +#include "resource_manager.hpp" +#include "transient_ptr.hpp" +#include "utils.hpp" + +namespace midas { + +using RetCode = ObjectPtr::RetCode; + +/** LogSegment */ +inline std::optional LogSegment::alloc_small(size_t size) { + if (sealed_ || destroyed_) + return std::nullopt; + auto obj_size = ObjectPtr::obj_size(size); + if (pos_ - start_addr_ + obj_size > kLogSegmentSize) { // this segment is full + seal(); + return std::nullopt; + } + ObjectPtr obj_ptr; + if (obj_ptr.init_small(pos_, size) != RetCode::Succ) { + if (!kEnableFaultHandler) + MIDAS_LOG(kError); + seal(); + return std::nullopt; + } + pos_ += obj_size; + return obj_ptr; +} + +inline std::optional> +LogSegment::alloc_large(size_t size, const TransientPtr head_tptr, + TransientPtr prev_tptr) { + if (sealed_ || destroyed_) + return std::nullopt; + if (pos_ - start_addr_ + sizeof(LargeObjectHdr) >= kLogSegmentSize) { + seal(); + return std::nullopt; + } + + ObjectPtr obj_ptr; + size_t trunced_size = std::min( + kLogSegmentSize - (pos_ - start_addr_) - sizeof(LargeObjectHdr), size); + TransientPtr addr(pos_, sizeof(LargeObjectHdr) + trunced_size); + const bool is_head = head_tptr.null(); + if (obj_ptr.init_large(pos_, trunced_size, is_head, head_tptr, + TransientPtr()) != RetCode::Succ) + return std::nullopt; + if (!prev_tptr.null()) { + LargeObjectHdr lhdr; + if (!load_hdr(lhdr, prev_tptr)) + return std::nullopt; + lhdr.set_next(addr); + if (!store_hdr(lhdr, prev_tptr)) + return std::nullopt; + } + + pos_ += sizeof(LargeObjectHdr) + trunced_size; + // if (pos_ - start_addr_ == kLogSegmentSize) + // seal(); + return std::make_pair(addr, trunced_size); +} + +inline bool LogSegment::free(ObjectPtr &ptr) { + return ptr.free() == RetCode::Succ; +} + +void LogSegment::destroy() noexcept { + destroyed_ = true; + auto *rmanager = owner_->pool_->get_rmanager(); + rmanager->FreeRegion(region_id_); + alive_bytes_ = kMaxAliveBytes; +} + +/** LogAllocator */ +// alloc a new segment +inline std::shared_ptr LogAllocator::allocSegment(bool overcommit) { + auto *rmanager = pool_->get_rmanager(); + int rid = rmanager->AllocRegion(overcommit); + if (rid == -1) + return nullptr; + VRange range = rmanager->GetRegion(rid); + + return std::make_shared( + this, rid, reinterpret_cast(range.stt_addr)); +} + +std::optional LogAllocator::alloc_(size_t size, bool overcommit) { + size = round_up_to_align(size, kSmallObjSizeUnit); + if (size >= kSmallObjThreshold) { // large obj + return alloc_large(size, overcommit); + } + + if (pcab_.local_seg && pcab_.local_seg->owner_ != this) { + pcab_.local_seg->owner_->stashed_pcabs_.push_back(pcab_.local_seg); + pcab_.local_seg.reset(); + } + if (!pcab_.local_seg) + pcab_.local_seg = stashed_pcabs_.pop_front(); + while (LIKELY(pcab_.local_seg.get() != nullptr)) { + auto ret = pcab_.local_seg->alloc_small(size); + if (ret) + return ret; + assert(pcab_.local_seg->sealed()); + // put pcab into segments_ and drop the reference so segments_ will be the + // only owner. + segments_.push_back(pcab_.local_seg); + pcab_.local_seg = stashed_pcabs_.pop_front(); + } + // slowpath + auto segment = allocSegment(overcommit); + if (!segment) + return std::nullopt; + + // since pcab hold a reference to segment here, don't put segment into + // segments_ for now. Instead, putting it back to segments_ when pcab is + // sealed and guaranteed not be used anymore. + pcab_.local_seg = segment; + auto ret = pcab_.local_seg->alloc_small(size); + assert(ret); + return ret; +} + +// Large objects +std::optional LogAllocator::alloc_large(size_t size, + bool overcommit) { + assert(size >= kSmallObjThreshold); + + ObjectPtr obj_ptr; + int64_t remaining_size = size; + TransientPtr head_tptr, prev_tptr; + size_t alloced_size = 0; + + if (pcab_.local_seg && pcab_.local_seg->owner_ != this) { + pcab_.local_seg->owner_->stashed_pcabs_.push_back(pcab_.local_seg); + pcab_.local_seg.reset(); + } + if (!pcab_.local_seg) + pcab_.local_seg = stashed_pcabs_.pop_front(); + while (LIKELY(pcab_.local_seg.get() != nullptr)) { + auto option = pcab_.local_seg->alloc_large(size, TransientPtr(), TransientPtr()); + if (option) { + head_tptr = option->first; + alloced_size = option->second; + // do not seal pcab at this time as allocation may not finish. Instead, + // seal pcab at the end of allocation. + assert(alloced_size > 0); + break; + } + // could be seg fault caused failure. The fault must happen + // on this segment as there is no prev/next segment at + // this point. + if (!pcab_.local_seg->sealed()) + pcab_.local_seg->seal(); + segments_.push_back(pcab_.local_seg); + + pcab_.local_seg = stashed_pcabs_.pop_front(); + } + remaining_size -= alloced_size; + if (remaining_size <= 0) { // common path. + assert(remaining_size == 0); + if (obj_ptr.init_from_soft(head_tptr) != RetCode::Succ) + return std::nullopt; + return obj_ptr; + } + + std::vector> alloced_segs; + std::vector alloced_ptrs; + if (head_tptr.null()) { + auto segment = allocSegment(overcommit); + if (!segment) + goto failed; + alloced_segs.push_back(segment); + assert(remaining_size == size); + auto option = + segment->alloc_large(remaining_size, TransientPtr(), TransientPtr()); + if (segment->full()) + segment->seal(); + if (!option) // out of memory during allocation + goto failed; + head_tptr = option->first; + alloced_size = option->second; + remaining_size -= alloced_size; + } + if (head_tptr.null()) + goto failed; + prev_tptr = head_tptr; + while (remaining_size > 0) { + alloced_ptrs.emplace_back(prev_tptr); + auto segment = allocSegment(overcommit); + if (!segment) + goto failed; + alloced_segs.push_back(segment); + auto option = segment->alloc_large(remaining_size, head_tptr, prev_tptr); + if (segment->full()) + segment->seal(); + if (!option) // out of memory during allocation + goto failed; + prev_tptr = option->first; + alloced_size = option->second; + remaining_size -= alloced_size; + } + if (obj_ptr.init_from_soft(head_tptr) != RetCode::Succ) + goto failed; + + assert(!pcab_.local_seg || pcab_.local_seg->full()); + if (pcab_.local_seg && pcab_.local_seg->full()) { + pcab_.local_seg->seal(); + segments_.push_back(pcab_.local_seg); + pcab_.local_seg.reset(); + } + assert(!pcab_.local_seg); + if (!alloced_segs.empty()) { + auto segment = alloced_segs.back(); + if (LIKELY(!segment->sealed())) { + // If the last segment is not full, use it as pcab so skip putting it into + // segments_ here. + pcab_.local_seg = segment; + alloced_segs.pop_back(); + } + } + for (auto &segment : alloced_segs) + segments_.push_back(segment); + return obj_ptr; + +failed: + if (!kEnableFaultHandler) + MIDAS_LOG(kDebug) << "allocation failed!"; + for (auto &tptr : alloced_ptrs) { + MetaObjectHdr mhdr; + if (!load_hdr(mhdr, tptr)) + continue; + mhdr.clr_present(); + if (!store_hdr(mhdr, tptr)) + continue; + } + for (auto &segment : alloced_segs) + segment->destroy(); + return std::nullopt; +} + +// Define PCAB +thread_local LogAllocator::PCAB LogAllocator::pcab_; +thread_local int32_t LogAllocator::access_cnt_ = 0; +thread_local int32_t LogAllocator::alive_cnt_ = 0; +std::atomic_int64_t LogAllocator::total_access_cnt_{0}; +std::atomic_int64_t LogAllocator::total_alive_cnt_{0}; + +} // namespace midas \ No newline at end of file diff --git a/src/object.cpp b/src/object.cpp new file mode 100644 index 0000000..0eb9a86 --- /dev/null +++ b/src/object.cpp @@ -0,0 +1,297 @@ +#include "object.hpp" +#include "logging.hpp" +#include "obj_locker.hpp" +#include "utils.hpp" + +namespace midas { +LockID ObjectPtr::lock() { + if (null()) + return INV_LOCK_ID; + auto locker = ObjLocker::global_objlocker(); + if (is_small_obj() || is_head_obj()) + return locker->lock(obj_); + else { // always lock the head segment even this is a continued segment. + LargeObjectHdr lhdr; + if (!load_hdr(lhdr, *this)) + return INV_LOCK_ID; + return locker->lock(lhdr.get_head()); + } +} + +void ObjectPtr::unlock(LockID id) { + assert(id != INV_LOCK_ID); + auto locker = ObjLocker::global_objlocker(); + locker->unlock(id); +} + +RetCode ObjectPtr::free(bool locked) noexcept { + if (locked) + return is_small_obj() ? free_small() : free_large(); + + auto ret = RetCode::Fail; + LockID lock_id = lock(); + if (lock_id == INV_LOCK_ID) // lock failed as obj_ has just been reset. + return RetCode::Fail; + if (!null()) + ret = is_small_obj() ? free_small() : free_large(); + unlock(lock_id); + return ret; +} + +bool ObjectPtr::copy_from_small(const void *src, size_t len, int64_t offset) { + auto ret = false; + if (null()) + return false; + auto lock_id = lock(); + if (lock_id == INV_LOCK_ID) // lock failed as obj_ has just been reset. + return false; + if (!null()) { + MetaObjectHdr meta_hdr; + if (!load_hdr(meta_hdr, *this)) + goto done; + if (!meta_hdr.is_present()) + goto done; + meta_hdr.inc_accessed(); + if (!store_hdr(meta_hdr, *this)) + goto done; + + ret = obj_.copy_from(src, len, hdr_size() + offset); + } +done: + unlock(lock_id); + return ret; +} + +bool ObjectPtr::copy_to_small(void *dst, size_t len, int64_t offset) { + auto ret = false; + if (null()) + return false; + auto lock_id = lock(); + if (lock_id == INV_LOCK_ID) // lock failed as obj_ has just been reset. + return false; + if (!null()) { + MetaObjectHdr meta_hdr; + if (!load_hdr(meta_hdr, *this)) + goto done; + if (!meta_hdr.is_present()) + goto done; + meta_hdr.inc_accessed(); + if (!store_hdr(meta_hdr, *this)) + goto done; + + ret = obj_.copy_to(dst, len, hdr_size() + offset); + } +done: + unlock(lock_id); + return ret; +} + +bool ObjectPtr::copy_from_large(const void *src, size_t len, int64_t offset) { + auto ret = false; + if (null()) + return false; + auto lock_id = lock(); + if (lock_id == INV_LOCK_ID) + return false; + if (!null()) { + MetaObjectHdr meta_hdr; + if (!load_hdr(meta_hdr, *this)) + goto done; + if (meta_hdr.is_continue() || !meta_hdr.is_present()) // invalid head chunk + goto done; + meta_hdr.inc_accessed(); + if (!store_hdr(meta_hdr, *this)) + goto done; + + int64_t remaining_offset = offset; + ObjectPtr optr = *this; + while (remaining_offset > 0) { + if (optr.null()) + goto fail_free; + if (remaining_offset < optr.data_size_in_segment()) + break; + remaining_offset -= optr.data_size_in_segment(); + + if (iter_large(optr) != RetCode::Succ) + goto fail_free; + } + assert(remaining_offset < optr.data_size_in_segment()); + // Now optr is pointing to the first part for copy + int64_t remaining_len = len; + while (remaining_len > 0) { + const auto copy_len = std::min( + remaining_len, optr.data_size_in_segment() - remaining_offset); + if (!optr.obj_.copy_from(src, copy_len, + sizeof(LargeObjectHdr) + remaining_offset)) + goto fail_free; + remaining_offset = 0; // copy from the beginning for the following parts + remaining_len -= copy_len; + if (remaining_len <= 0) + break; + src = reinterpret_cast(reinterpret_cast(src) + + copy_len); + + if (iter_large(optr) != RetCode::Succ) + goto fail_free; + } + ret = true; + } + +done: + unlock(lock_id); + return ret; + +fail_free: + free_large(); + unlock(lock_id); + return ret; +} + +bool ObjectPtr::copy_to_large(void *dst, size_t len, int64_t offset) { + auto ret = false; + if (null()) + return false; + auto lock_id = lock(); + if (lock_id == INV_LOCK_ID) + return false; + if (!null()) { + MetaObjectHdr meta_hdr; + if (!load_hdr(meta_hdr, *this)) + goto done; + if (meta_hdr.is_continue() || !meta_hdr.is_present()) + goto done; + meta_hdr.inc_accessed(); + if (!store_hdr(meta_hdr, *this)) + goto done; + + int64_t remaining_offset = offset; + ObjectPtr optr = *this; + while (remaining_offset > 0) { + if (optr.null()) + goto fail_free; + if (remaining_offset < optr.data_size_in_segment()) + break; + remaining_offset -= optr.data_size_in_segment(); + + if (iter_large(optr) != RetCode::Succ) + goto fail_free; + } + // Now optr is pointing to the first part for copy + int64_t remaining_len = len; + while (remaining_len > 0) { + const auto copy_len = std::min( + remaining_len, optr.data_size_in_segment() - remaining_offset); + if (!optr.obj_.copy_to(dst, copy_len, + sizeof(LargeObjectHdr) + remaining_offset)) + goto fail_free; + remaining_offset = 0; // copy from the beginning for the following parts + remaining_len -= copy_len; + if (remaining_len <= 0) + break; + dst = + reinterpret_cast(reinterpret_cast(dst) + copy_len); + + if (iter_large(optr) != RetCode::Succ) + goto fail_free; + } + ret = true; + } + +done: + unlock(lock_id); + return ret; + +fail_free: + free_large(); + unlock(lock_id); + return ret; +} + +// For evacuator only. Must have src locked +RetCode ObjectPtr::copy_from_large(const TransientPtr &src, size_t len, + int64_t from_offset, int64_t to_offset) { + if (null()) + return RetCode::Fail; + + int64_t remaining_offset = to_offset; + ObjectPtr optr = *this; + while (remaining_offset > 0) { + if (optr.null()) + return RetCode::Fail; + if (remaining_offset < optr.data_size_in_segment()) + break; + remaining_offset -= optr.data_size_in_segment(); + + LargeObjectHdr lhdr; + if (!load_hdr(lhdr, optr)) + return RetCode::FaultOther; // dst (this) is considered as other + auto next = lhdr.get_next(); + if (next.null() || optr.init_from_soft(next) != RetCode::Succ) + return RetCode::FaultOther; // dst (this) is considered as other + } + // Now optr is pointing to the first part for copy + auto src_tptr = src.slice(from_offset); + int64_t remaining_len = len; + while (remaining_len > 0) { + const auto copy_len = std::min( + remaining_len, optr.data_size_in_segment() - remaining_offset); + if (!optr.obj_.copy_from(src_tptr, copy_len, 0, + sizeof(LargeObjectHdr) + remaining_offset)) { + // TODO (YIFAN): so far we cannot tell whether fault on src or dst. so we + // make a conservative assumption and always return FaultLocal to ensure + // correctness and simplicity. This may negatively impact the performance + // as FaultLocal will unmap the current segment entirely, but hopefully + // the impact is not significant as this is a rare case. + if (!kEnableFaultHandler) + MIDAS_LOG(kError); + return RetCode::FaultLocal; + } + remaining_offset = 0; // copy from the beginning for non-head parts + remaining_len -= copy_len; + if (remaining_len <= 0) + break; + src_tptr = src_tptr.slice(copy_len); + + if (iter_large(optr) != RetCode::Succ) + return RetCode::FaultOther; // dst (this) is considered as other + } + return RetCode::Succ; +} + +RetCode ObjectPtr::move_large(ObjectPtr &src) noexcept { + MetaObjectHdr mhdr; + if (!load_hdr(mhdr, *this)) + return RetCode::FaultOther; // dst (this) is considered as other + if (!mhdr.is_present()) + return RetCode::Fail; + + assert(!is_small_obj() && is_head_obj()); + assert(!src.is_small_obj() && src.is_head_obj()); + + auto opt_size = src.large_data_size(); + if (!opt_size) + return RetCode::FaultLocal; + size_t remaining_len = *opt_size; + size_t dst_offset = 0; + ObjectPtr optr = src; + while (!optr.null()) { + auto ret = RetCode::Fail; + assert(optr.hdr_size() == sizeof(LargeObjectHdr)); + ret = copy_from_large(optr.obj_, optr.data_size_in_segment(), + optr.hdr_size(), dst_offset); + if (ret != RetCode::Succ) + return ret; + dst_offset += optr.data_size_in_segment(); + remaining_len -= optr.data_size_in_segment(); + ret = iter_large(optr); + if (ret != RetCode::Succ) { + if (ret == RetCode::Fail) + return RetCode::Fail; + return ret; + } + } + + assert(remaining_len == 0); + return RetCode::Succ; +} +} // namespace midas \ No newline at end of file diff --git a/src/perf.cpp b/src/perf.cpp new file mode 100644 index 0000000..ccda86a --- /dev/null +++ b/src/perf.cpp @@ -0,0 +1,304 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "logging.hpp" +#include "perf.hpp" +#include "time.hpp" + +namespace midas { + +Trace::Trace() : absl_start_us(0), start_us(0), duration_us(0) {} +Trace::Trace(uint64_t absl_start_us_, uint64_t start_us_, uint64_t duration_us_) + : absl_start_us(absl_start_us_), start_us(start_us_), + duration_us(duration_us_) {} + +Perf::Perf(PerfAdapter &adapter) + : adapter_(adapter), trace_format_(kUnsorted), real_kops_(0), succ_ops(0) {} + +void Perf::reset() { + traces_.clear(); + trace_format_ = kUnsorted; + real_kops_ = 0; + tputs_.clear(); +} + +uint64_t Perf::gen_reqs(std::vector *all_reqs, + uint32_t num_threads, double target_kops, + uint64_t duration_us, uint64_t start_us) { + std::vector threads; + + for (uint32_t i = 0; i < num_threads; i++) { + threads.emplace_back([&, &reqs = all_reqs[i], tid = i] { + std::random_device rd; + std::mt19937 gen(rd()); + std::exponential_distribution d(target_kops / 1000 / num_threads); + uint64_t cur_us = start_us; + uint64_t end_us = cur_us + duration_us; + + while (cur_us < end_us) { + auto interval = std::max(1l, std::lround(d(gen))); + PerfRequestWithTime req_with_time; + req_with_time.start_us = cur_us; + req_with_time.req = adapter_.gen_req(tid); + reqs.emplace_back(std::move(req_with_time)); + cur_us += interval; + } + }); + } + + for (auto &thread : threads) { + thread.join(); + } + return start_us + duration_us; +} + +uint64_t Perf::gen_phased_reqs(std::vector *all_reqs, + uint32_t num_threads, + std::vector &target_kops_vec, + std::vector &duration_us_vec, + std::vector &transition_us_vec, + uint64_t start_us) { + const auto nr_phases = target_kops_vec.size(); + auto cur_us = start_us; + for (uint32_t phase = 0; phase < nr_phases; phase++) { + auto target_kops = target_kops_vec[phase]; + auto duration_us = duration_us_vec[phase]; + cur_us = gen_reqs(all_reqs, num_threads, target_kops_vec[phase], + duration_us, cur_us); + if (phase == nr_phases - 1) + break; + // transition phase + const auto trans_dur_step_us = transition_us_vec[phase] / kTransSteps; + const auto next_target_kops = target_kops_vec[phase + 1]; + const auto trans_step_kops = (next_target_kops - target_kops) / kTransSteps; + auto transition_kops = target_kops; + for (uint32_t step = 0; step < kTransSteps; step++) { + cur_us = gen_reqs(all_reqs, num_threads, transition_kops, + trans_dur_step_us, cur_us); + transition_kops += trans_step_kops; + } + } + + return cur_us; +} + +std::vector Perf::benchmark(std::vector *all_reqs, + uint32_t num_threads, + uint64_t miss_ddl_thresh_us) { + std::vector threads; + std::vector all_traces[num_threads]; + + for (uint32_t i = 0; i < num_threads; i++) { + all_traces[i].reserve(all_reqs[i].size()); + } + + for (uint32_t i = 0; i < num_threads; i++) { + threads.emplace_back( + [&, &reqs = all_reqs[i], &traces = all_traces[i], tid = i] { + int nr_succ = 0; + auto start_us = Time::get_us(); + + for (const auto &req : reqs) { + auto relative_us = Time::get_us() - start_us; + if (req.start_us > relative_us) { + std::this_thread::sleep_for( + std::chrono::microseconds(req.start_us - relative_us)); + } else if (req.start_us + miss_ddl_thresh_us < relative_us) { + continue; + } + Trace trace; + trace.absl_start_us = Time::get_us(); + trace.start_us = trace.absl_start_us - start_us; + bool ok = adapter_.serve_req(tid, req.req.get()); + trace.duration_us = Time::get_us() - start_us - trace.start_us; + if (ok) { + traces.push_back(trace); + nr_succ++; + if (nr_succ % kReportBatch == 0) { + succ_ops += nr_succ; + nr_succ = 0; + } + } + } + }); + } + + for (auto &thread : threads) { + thread.join(); + } + + std::vector gathered_traces; + for (uint32_t i = 0; i < num_threads; i++) { + gathered_traces.insert(gathered_traces.end(), all_traces[i].begin(), + all_traces[i].end()); + } + return gathered_traces; +} + +void Perf::run(uint32_t num_threads, double target_kops, uint64_t duration_us, + uint64_t warmup_us, uint64_t miss_ddl_thresh_us) { + std::vector all_warmup_reqs[num_threads]; + std::vector all_perf_reqs[num_threads]; + MIDAS_LOG(kInfo) << "Start generating requests..."; + gen_reqs(all_warmup_reqs, num_threads, target_kops, warmup_us); + MIDAS_LOG(kInfo) << "Finish generating warmup requests..."; + gen_reqs(all_perf_reqs, num_threads, target_kops, duration_us); + MIDAS_LOG(kInfo) << "Finish generating perf requests..."; + benchmark(all_warmup_reqs, num_threads, miss_ddl_thresh_us); + + bool stop = false; + std::unique_ptr report_thd; + if (kEnableReporter) { + report_thd = std::make_unique([&] { + while (!stop) { + uint64_t duration_us = kReportInterval * to_us; + std::this_thread::sleep_for(std::chrono::microseconds(duration_us)); + report_tput(duration_us); + } + dump_tput(); + }); + } + + traces_ = + std::move(benchmark(all_perf_reqs, num_threads, miss_ddl_thresh_us)); + + stop = true; + if (report_thd) + report_thd->join(); + + auto real_duration_us = + std::accumulate(traces_.begin(), traces_.end(), static_cast(0), + [](uint64_t ret, Trace t) { + return std::max(ret, t.start_us + t.duration_us); + }); + real_kops_ = static_cast(traces_.size()) / (real_duration_us / 1000); +} + +void Perf::run_phased(uint32_t num_threads, std::vector target_kops_vec, + std::vector &duration_us_vec, + std::vector &transition_us_vec, + double warmup_kops, uint64_t warmup_us, + uint64_t miss_ddl_thresh_us) { + std::vector all_warmup_reqs[num_threads]; + std::vector all_perf_reqs[num_threads]; + auto cur_us = gen_reqs(all_warmup_reqs, num_threads, warmup_kops, warmup_us); + gen_phased_reqs(all_perf_reqs, num_threads, target_kops_vec, duration_us_vec, + transition_us_vec, cur_us); + + benchmark(all_warmup_reqs, num_threads, miss_ddl_thresh_us); + + bool stop = false; + std::unique_ptr report_thd; + if (kEnableReporter) { + report_thd = std::make_unique([&] { + while (!stop) { + uint64_t duration_us = kReportInterval * to_us; + std::this_thread::sleep_for(std::chrono::microseconds(duration_us)); + report_tput(duration_us); + } + dump_tput(); + }); + } + + traces_ = + std::move(benchmark(all_perf_reqs, num_threads, miss_ddl_thresh_us)); + + stop = true; + if (report_thd) + report_thd->join(); + + auto real_duration_us = + std::accumulate(traces_.begin(), traces_.end(), static_cast(0), + [](uint64_t ret, Trace t) { + return std::max(ret, t.start_us + t.duration_us); + }); + real_kops_ = static_cast(traces_.size()) / (real_duration_us / 1000); +} + +uint64_t Perf::get_average_lat() { + if (trace_format_ != kSortedByDuration) { + std::sort(traces_.begin(), traces_.end(), + [](const Trace &x, const Trace &y) { + return x.duration_us < y.duration_us; + }); + trace_format_ = kSortedByDuration; + } + + auto sum = std::accumulate( + std::next(traces_.begin()), traces_.end(), 0ULL, + [](uint64_t sum, const Trace &t) { return sum + t.duration_us; }); + return sum / traces_.size(); +} + +uint64_t Perf::get_nth_lat(double nth) { + if (trace_format_ != kSortedByDuration) { + std::sort(traces_.begin(), traces_.end(), + [](const Trace &x, const Trace &y) { + return x.duration_us < y.duration_us; + }); + trace_format_ = kSortedByDuration; + } + + size_t idx = nth / 100.0 * traces_.size(); + return traces_[idx].duration_us; +} + +std::vector Perf::get_timeseries_nth_lats(uint64_t interval_us, + double nth) { + std::vector timeseries; + if (trace_format_ != kSortedByStart) { + std::sort( + traces_.begin(), traces_.end(), + [](const Trace &x, const Trace &y) { return x.start_us < y.start_us; }); + trace_format_ = kSortedByStart; + } + + auto cur_win_us = traces_.front().start_us; + auto absl_cur_win_us = traces_.front().absl_start_us; + std::vector win_durations; + for (auto &trace : traces_) { + if (cur_win_us + interval_us < trace.start_us) { + std::sort(win_durations.begin(), win_durations.end()); + if (win_durations.size() >= 100) { + size_t idx = nth / 100.0 * win_durations.size(); + timeseries.emplace_back(absl_cur_win_us, cur_win_us, + win_durations[idx]); + } + cur_win_us += interval_us; + absl_cur_win_us += interval_us; + win_durations.clear(); + } + win_durations.push_back(trace.duration_us); + } + + return timeseries; +} + +double Perf::get_real_kops() const { return real_kops_; } + +const std::vector &Perf::get_traces() const { return traces_; } + +void Perf::report_tput(uint64_t duration_us) { + float tput = succ_ops * 1000.0 / duration_us; + MIDAS_LOG(kInfo) << "Tput: " << tput << " Kops"; + tputs_.emplace_back(tput); + succ_ops = 0; +} + +void Perf::dump_tput() { + std::ofstream tput_file("tput.txt"); + if (!tput_file.good()) + return; + for (auto tput : tputs_) + tput_file << tput << "\t"; + tput_file << std::endl; + tput_file.close(); +} +} // namespace midas \ No newline at end of file diff --git a/src/resilient_func.cpp b/src/resilient_func.cpp new file mode 100644 index 0000000..fdb366a --- /dev/null +++ b/src/resilient_func.cpp @@ -0,0 +1,130 @@ +#include +#include +#include + +#include "resilient_func.hpp" +#include "utils.hpp" + +namespace midas { + +// Must have len < 16 bytes. Manually unroll instructions for data < 16 bytes. +FORCE_INLINE void rmemcpy_tiny(uint8_t *dst, const uint8_t *src, size_t len) { + // assert(len < 16); // should not assert in soft resilient functions + if (len & 8) { + *(reinterpret_cast(dst)) = + *(reinterpret_cast(src)); + dst += 8; + src += 8; + } + if (len & 4) { + *(reinterpret_cast(dst)) = + *(reinterpret_cast(src)); + dst += 4; + src += 4; + } + if (len & 2) { + *(reinterpret_cast(dst)) = + *(reinterpret_cast(src)); + dst += 2; + src += 2; + } + if (len & 1) + *dst = *src; +} + +// NOTE: used when len <= 32 bytes (256 bits) +FORCE_INLINE void rmemcpy_small(void *dst, const void *src, size_t len) { + auto *dst_ = reinterpret_cast(dst); + const auto *src_ = reinterpret_cast(src); + auto qwords = len >> 3; + len -= qwords << 3; // remaining len + while (qwords-- > 0) { + *(dst_++) = *(src_++); + } + if (UNLIKELY(len)) + rmemcpy_tiny(reinterpret_cast(dst_), + reinterpret_cast(src_), len); +} + +/** YIFAN: not perform well with small size (< 16 bytes) */ +FORCE_INLINE void rmemcpy_ermsb(void *dst, const void *src, size_t len) { + asm volatile("rep movsb" : "+D"(dst), "+S"(src), "+c"(len)::"memory"); +} + +FORCE_INLINE void rmemcpy_avx128(void *dst, const void *src, size_t len) { + /* dst, src -> 16 bytes addresses + * len -> divided into multiple of 16 */ + auto *dst_vec = reinterpret_cast<__m128i *>(dst); + const auto *src_vec = reinterpret_cast(src); + + size_t nr_vwords = len / sizeof(__m128i); + len -= nr_vwords * sizeof(__m128i); + const bool dst_aligned = !((uint64_t)dst_vec & (0x80 - 1)); + const bool src_aligned = !((uint64_t)src_vec & (0x80 - 1)); + if (dst_aligned && src_aligned) + for (; nr_vwords > 0; nr_vwords--, src_vec++, dst_vec++) + _mm_store_si128(dst_vec, _mm_load_si128(src_vec)); + else + for (; nr_vwords > 0; nr_vwords--, src_vec++, dst_vec++) + _mm_storeu_si128(dst_vec, _mm_lddqu_si128(src_vec)); + + if (len) + rmemcpy_small(dst_vec, src_vec, len); +} + +/** YIFAN: In my experience it is not faster than avx128 for most cases. */ +FORCE_INLINE void rmemcpy_avx256(void *dst, const void *src, size_t len) { + /* dst, src -> 32 bytes addresses + * len -> divided into multiple of 32 */ + auto *dst_vec = reinterpret_cast<__m256i *>(dst); + const auto *src_vec = reinterpret_cast(src); + + size_t nr_vwords = len / sizeof(__m256i); + len -= nr_vwords * sizeof(__m256i); + for (; nr_vwords > 0; nr_vwords--, src_vec++, dst_vec++) + _mm256_storeu_si256(dst_vec, _mm256_lddqu_si256(src_vec)); + + if (len) + rmemcpy_small(dst_vec, src_vec, len); +} + +/** YIFAN: In my experience it is not faster than avx128 for most cases. */ +FORCE_INLINE void rmemcpy_avx_unroll(void *dst, const void *src, size_t len) { + /* dst, src -> 256 byte aligned + * len -> multiple of 256 */ + auto *dst_vec = reinterpret_cast<__m512i *>(dst); + const auto *src_vec = reinterpret_cast(src); + size_t nr_vwords = len / sizeof(__m512i); + len -= nr_vwords * sizeof(__m512i); + for (; nr_vwords > 0; nr_vwords -= 4, src_vec += 4, dst_vec += 4) { + _mm512_storeu_si512(dst_vec + 0, _mm512_loadu_si512(src_vec)); + _mm512_storeu_si512(dst_vec + 1, _mm512_loadu_si512(src_vec + 1)); + _mm512_storeu_si512(dst_vec + 2, _mm512_loadu_si512(src_vec + 2)); + _mm512_storeu_si512(dst_vec + 3, _mm512_loadu_si512(src_vec + 3)); + } + while (nr_vwords-- > 0) { + _mm512_storeu_si512(dst_vec++, _mm512_loadu_si512(src_vec++)); + } + if (len) { + rmemcpy_small(dst_vec, src_vec, len); + } +} + +/** + * YIFAN: within 20% overhead compared to std::memcpy for small data (< 16 + * bytes). ~30% faster than std::memcpy for large data (>= 512 bytes). + */ +bool rmemcpy(void *dst, const void *src, size_t len) { + if (src == dst || len == 0) + return true; + if (UNLIKELY(len < sizeof(__m128i))) { // sizeof(__m128i) == 16 + rmemcpy_tiny(reinterpret_cast(dst), + reinterpret_cast(src), len); + return true; + } + rmemcpy_avx128(dst, src, len); + return true; +} +DELIM_FUNC_IMPL(rmemcpy) + +} // namespace midas \ No newline at end of file diff --git a/src/resource_manager.cpp b/src/resource_manager.cpp new file mode 100644 index 0000000..6785908 --- /dev/null +++ b/src/resource_manager.cpp @@ -0,0 +1,527 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cache_manager.hpp" +#include "evacuator.hpp" +#include "logging.hpp" +#include "qpair.hpp" +#include "resource_manager.hpp" +#include "shm_types.hpp" +#include "time.hpp" +#include "utils.hpp" + +namespace midas { +constexpr static int32_t kMaxAllocRetry = 5; +constexpr static int32_t kReclaimRepeat = 20; +constexpr static auto kReclaimTimeout = + std::chrono::milliseconds(100); // milliseconds +constexpr static auto kAllocRetryDelay = + std::chrono::microseconds(100); // microseconds +constexpr static int32_t kMonitorTimeout = 1; // seconds +constexpr static int32_t kDisconnTimeout = 3; // seconds +constexpr static bool kEnableFreeList = true; +constexpr static int32_t kFreeListSize = 512; + +std::atomic_int64_t Region::global_mapped_rid_{0}; + +Region::Region(uint64_t pid, uint64_t region_id) noexcept + : pid_(pid), prid_(region_id), vrid_(INVALID_VRID) { + map(); +} + +void Region::map() noexcept { + assert(vrid_ == INVALID_VRID); + const auto rwmode = boost::interprocess::read_write; + const std::string shm_name_ = utils::get_region_name(pid_, prid_); + SharedMemObj shm_obj(boost::interprocess::open_only, shm_name_.c_str(), + rwmode); + shm_obj.get_size(size_); + vrid_ = global_mapped_rid_.fetch_add(1); + void *addr = reinterpret_cast(kVolatileSttAddr + vrid_ * kRegionSize); + shm_region_ = std::make_unique(shm_obj, rwmode, 0, size_, addr); +} + +void Region::unmap() noexcept { + shm_region_.reset(); + vrid_ = INVALID_VRID; +} + +Region::~Region() noexcept { + unmap(); + free(); +} + +void Region::free() noexcept { + SharedMemObj::remove(utils::get_region_name(pid_, prid_).c_str()); +} + +ResourceManager::ResourceManager(CachePool *cpool, + const std::string &daemon_name) noexcept + : cpool_(cpool), id_(get_unique_id()), region_limit_(0), + txqp_(std::make_shared(utils::get_sq_name(daemon_name, false), + false), + std::make_shared(utils::get_ackq_name(daemon_name, id_), + true)), + rxqp_(std::to_string(id_), true), stop_(false), nr_pending_(0), stats_() { + handler_thd_ = std::make_shared([&]() { pressure_handler(); }); + if (!cpool_) + cpool_ = CachePool::global_cache_pool(); + assert(cpool_); + connect(daemon_name); +} + +ResourceManager::~ResourceManager() noexcept { + stop_ = true; + handler_thd_->join(); + + disconnect(); + rxqp_.destroy(); + txqp_.RecvQ().destroy(); +} + +std::shared_ptr +ResourceManager::global_manager_shared_ptr() noexcept { + return CachePool::global_cache_pool()->rmanager_; +} + +/** trigger evacuation */ +bool ResourceManager::reclaim_trigger() noexcept { + return reclaim_target() > 0; +} + +int64_t ResourceManager::reclaim_target() noexcept { + constexpr static float kAvailRatioThresh = 0.01; + int64_t nr_avail = NumRegionAvail(); + int64_t nr_limit = NumRegionLimit(); + if (nr_limit <= 1) + return 0; + int64_t target_avail = nr_limit * kAvailRatioThresh; + int64_t nr_to_reclaim = nr_pending_; + auto headroom = std::max(reclaim_headroom(), target_avail); + if (nr_avail <= headroom) + nr_to_reclaim += std::max(headroom - nr_avail, 2l); + nr_to_reclaim = std::min(nr_to_reclaim, nr_limit); + return nr_to_reclaim; +} + +int32_t ResourceManager::reclaim_headroom() noexcept { + auto scale_factor = 5; + if (stats_.alloc_tput > 1000 || stats_.alloc_tput > 8 * stats_.reclaim_tput) + scale_factor = 20; + else if (stats_.alloc_tput > 500 || stats_.alloc_tput > 6 * stats_.reclaim_tput) + scale_factor = 15; + else if (stats_.alloc_tput > 300) + scale_factor = 10; + auto headroom = std::min( + region_limit_ * 0.5, + std::max(32, + scale_factor * stats_.reclaim_dur * stats_.alloc_tput)); + stats_.headroom = headroom; + // MIDAS_LOG(kInfo) << "headroom: " << headroom; + return headroom; +} + +int32_t ResourceManager::reclaim_nr_thds() noexcept { + int32_t nr_evac_thds = kNumEvacThds; + if (stats_.reclaim_tput > 1e-6) { // having non-zero reclaim_tput + nr_evac_thds = stats_.alloc_tput / stats_.reclaim_tput; + if (nr_evac_thds < 4) + nr_evac_thds++; + else + nr_evac_thds *= 2; + nr_evac_thds = std::min(kNumEvacThds, nr_evac_thds); + nr_evac_thds = std::max(1, nr_evac_thds); + } + return nr_evac_thds; +} + + +/** Profiling for stats */ +inline void ResourceManager::prof_alloc_tput() { + auto time = Time::get_us(); + if (stats_.prev_time == 0) { // init + stats_.prev_time = time; + stats_.prev_alloced = stats_.nr_alloced; + } else { + auto dur_us = time - stats_.prev_time; + auto alloc_tput = (stats_.nr_alloced - stats_.prev_alloced) * 1e6 / dur_us; + stats_.alloc_tput = alloc_tput; + stats_.prev_time = time; + stats_.prev_alloced = stats_.nr_alloced; + MIDAS_LOG(kDebug) << "Allocation Tput: " << alloc_tput; + } + + if (stats_.accum_evac_dur > 1e-6 && stats_.accum_nr_reclaimed >= 1) { // > 1us && reclaimed > 1 segment + stats_.reclaim_tput = stats_.accum_nr_reclaimed / stats_.accum_evac_dur; + stats_.reclaim_dur = stats_.accum_evac_dur / stats_.evac_cnt; + MIDAS_LOG(kDebug) << "Reclamation Tput: " << stats_.reclaim_tput + << ", Duration: " << stats_.reclaim_dur; + // reset accumulated counters + stats_.evac_cnt = 0; + stats_.accum_nr_reclaimed = 0; + stats_.accum_evac_dur = 0; + } + MIDAS_LOG(kDebug) << "Evacuator params: headroom = " << reclaim_headroom() + << ", nr_thds = " << reclaim_nr_thds(); +} + +void ResourceManager::prof_reclaim_stt() { + stats_.prev_evac_alloced = stats_.nr_evac_alloced; + stats_.prev_freed = stats_.nr_freed; +} + +void ResourceManager::prof_reclaim_end(int nr_thds, double dur_s) { + auto nr_freed = stats_.nr_freed - stats_.prev_freed; + auto nr_evac_alloced = stats_.nr_evac_alloced - stats_.prev_evac_alloced; + stats_.accum_nr_reclaimed += + static_cast(nr_freed - nr_evac_alloced) / nr_thds; + stats_.accum_evac_dur += static_cast(dur_s); // to us + stats_.evac_cnt++; +} + +/** interacting with the global daemon */ +int ResourceManager::connect(const std::string &daemon_name) noexcept { + std::unique_lock lk(mtx_); + try { + unsigned int prio = 0; + CtrlMsg msg{.id = id_, .op = CtrlOpCode::CONNECT}; + + txqp_.send(&msg, sizeof(CtrlMsg)); + int ret = txqp_.recv(&msg, sizeof(CtrlMsg)); + if (ret) { + return -1; + } + if (msg.op == CtrlOpCode::CONNECT && msg.ret == CtrlRetCode::CONN_SUCC) + MIDAS_LOG(kInfo) << "Connection established."; + else { + MIDAS_LOG(kError) << "Connection failed."; + abort(); + } + region_limit_ = msg.mmsg.size / kRegionSize; + } catch (boost::interprocess::interprocess_exception &e) { + MIDAS_LOG(kError) << e.what(); + } + + return 0; +} + +int ResourceManager::disconnect() noexcept { + std::unique_lock lk(mtx_); + try { + unsigned int prio = 0; + CtrlMsg msg{.id = id_, .op = CtrlOpCode::DISCONNECT}; + + txqp_.send(&msg, sizeof(CtrlMsg)); + int ret = txqp_.timed_recv(&msg, sizeof(CtrlMsg), kDisconnTimeout); + if (ret) + return -1; + if (msg.op == CtrlOpCode::DISCONNECT && msg.ret == CtrlRetCode::CONN_SUCC) + MIDAS_LOG(kInfo) << "Connection destroyed."; + else { + MIDAS_LOG(kError) << "Disconnection failed."; + return -1; + } + } catch (boost::interprocess::interprocess_exception &e) { + MIDAS_LOG(kError) << e.what(); + } + + return 0; +} + +void ResourceManager::pressure_handler() { + MIDAS_LOG(kError) << "pressure handler thd is running..."; + + while (!stop_) { + CtrlMsg msg; + if (rxqp_.timed_recv(&msg, sizeof(msg), kMonitorTimeout) != 0) + continue; + + MIDAS_LOG(kDebug) << "PressureHandler recved msg " << msg.op; + switch (msg.op) { + case UPDLIMIT: + do_update_limit(msg); + break; + case PROF_STATS: + do_profile_stats(msg); + break; + case FORCE_RECLAIM: + do_force_reclaim(msg); + break; + case DISCONNECT: + do_disconnect(msg); + default: + MIDAS_LOG(kError) << "Recved unknown message: " << msg.op; + } + } +} + +void ResourceManager::do_update_limit(CtrlMsg &msg) { + assert(msg.mmsg.size != 0); + auto new_region_limit = msg.mmsg.size; + MIDAS_LOG(kError) << "Client " << id_ << " update limit: " << region_limit_ + << "->" << new_region_limit; + region_limit_ = new_region_limit; + + CtrlMsg ack{.op = CtrlOpCode::UPDLIMIT, .ret = CtrlRetCode::MEM_SUCC}; + if (NumRegionAvail() < 0) { // under memory pressure + auto before_usage = NumRegionInUse(); + MIDAS_LOG_PRINTF(kInfo, "Memory shrinkage: %ld to reclaim (%ld->%ld).\n", + -NumRegionAvail(), NumRegionInUse(), NumRegionLimit()); + if (!reclaim()) // failed to reclaim enough memory + ack.ret = CtrlRetCode::MEM_FAIL; + auto after_usage = NumRegionInUse(); + auto nr_reclaimed = before_usage - after_usage; + MIDAS_LOG_PRINTF(kError, "Memory shrinkage: %ld reclaimed (%ld/%ld).\n", + nr_reclaimed, NumRegionInUse(), NumRegionLimit()); + } else if (reclaim_trigger()) { // concurrent GC if needed + cpool_->get_evacuator()->signal_gc(); + } + ack.mmsg.size = region_map_.size() + freelist_.size(); + rxqp_.send(&ack, sizeof(ack)); +} + +void ResourceManager::do_force_reclaim(CtrlMsg &msg) { + assert(msg.mmsg.size != 0); + auto new_region_limit = msg.mmsg.size; + MIDAS_LOG(kError) << "Client " << id_ << " update limit: " << region_limit_ + << "->" << new_region_limit; + region_limit_ = new_region_limit; + + force_reclaim(); + + CtrlMsg ack{.op = CtrlOpCode::UPDLIMIT, .ret = CtrlRetCode::MEM_SUCC}; + ack.mmsg.size = region_map_.size() + freelist_.size(); + rxqp_.send(&ack, sizeof(ack)); +} + +void ResourceManager::do_profile_stats(CtrlMsg &msg) { + StatsMsg stats{0}; + cpool_->profile_stats(&stats); + prof_alloc_tput(); + stats.headroom = stats_.headroom; + rxqp_.send(&stats, sizeof(stats)); +} + +void ResourceManager::do_disconnect(CtrlMsg &msg) { + stop_ = true; + handler_thd_->join(); + MIDAS_LOG(kError) << "Client " << id_ << " Disconnected!"; + exit(-1); +} + +bool ResourceManager::reclaim() { + if (NumRegionInUse() > NumRegionLimit() + kForceReclaimThresh) + return force_reclaim(); + if (NumRegionInUse() < NumRegionLimit()) + return true; + + nr_pending_++; + cpool_->get_evacuator()->signal_gc(); + for (int rep = 0; rep < kReclaimRepeat; rep++) { + { + std::unique_lock ul(mtx_); + while (!freelist_.empty()) { + auto region = freelist_.back(); + freelist_.pop_back(); + free_region(region, true); + } + cv_.wait_for(ul, kReclaimTimeout, [&] { return NumRegionAvail() > 0; }); + } + if (NumRegionAvail() > 0) + break; + cpool_->get_evacuator()->signal_gc(); + } + + nr_pending_--; + return NumRegionAvail() > 0; +} + +bool ResourceManager::force_reclaim() { + if (NumRegionInUse() < NumRegionLimit()) + return true; + + nr_pending_++; + { + std::unique_lock ul(mtx_); + while (!freelist_.empty()) { + auto region = freelist_.back(); + freelist_.pop_back(); + free_region(region, true); + } + } + + while (NumRegionAvail() <= 0) + cpool_->get_evacuator()->force_reclaim(); + nr_pending_--; + return NumRegionAvail() > 0; +} + +void ResourceManager::SetWeight(float weight) noexcept { + CtrlMsg msg{.id = id_, + .op = CtrlOpCode::SET_WEIGHT, + .mmsg = {.weight = weight}}; + txqp_.send(&msg, sizeof(msg)); +} + +void ResourceManager::SetLatCritical(bool value) noexcept { + CtrlMsg msg{.id = id_, + .op = CtrlOpCode::SET_LAT_CRITICAL, + .mmsg = {.lat_critical = value}}; + txqp_.send(&msg, sizeof(msg)); +} + +void ResourceManager::UpdateLimit(size_t size) noexcept { + CtrlMsg msg{ + .id = id_, .op = CtrlOpCode::UPDLIMIT_REQ, .mmsg = {.size = size}}; + txqp_.send(&msg, sizeof(msg)); +} + +int64_t ResourceManager::AllocRegion(bool overcommit) noexcept { + int retry_cnt = 0; +retry: + if (retry_cnt >= kMaxAllocRetry) { + MIDAS_LOG(kDebug) << "Cannot allocate new region after " << retry_cnt + << " retires!"; + return -1; + } + retry_cnt++; + if (!overcommit && reclaim_trigger()) { + if (NumRegionAvail() <= 0) { // block waiting for reclamation + if (kEnableFaultHandler && retry_cnt >= kMaxAllocRetry / 2) + force_reclaim(); + else + reclaim(); + } else + cpool_->get_evacuator()->signal_gc(); + } + + // 1) Fast path. Allocate from freelist + std::unique_lock lk(mtx_); + if (!freelist_.empty()) { + auto region = freelist_.back(); + freelist_.pop_back(); + region->map(); + int64_t region_id = region->ID(); + region_map_[region_id] = region; + overcommit ? stats_.nr_evac_alloced++ : stats_.nr_alloced++; + return region_id; + } + // 2) Local alloc path. Do reclamation and try local allocation again + if (!overcommit && NumRegionAvail() <= 0) { + lk.unlock(); + std::this_thread::sleep_for(kAllocRetryDelay); + goto retry; + } + // 3) Remote alloc path. Comm with daemon and try to alloc + CtrlMsg msg{.id = id_, + .op = overcommit ? CtrlOpCode::OVERCOMMIT : CtrlOpCode::ALLOC, + .mmsg = {.size = kRegionSize}}; + txqp_.send(&msg, sizeof(msg)); + + unsigned prio; + CtrlMsg ret_msg; + int ret = txqp_.recv(&ret_msg, sizeof(ret_msg)); + if (ret) { + MIDAS_LOG(kError) << "Allocation error: " << ret; + return -1; + } + if (ret_msg.ret != CtrlRetCode::MEM_SUCC) { + lk.unlock(); + goto retry; + } + + int64_t region_id = ret_msg.mmsg.region_id; + assert(region_map_.find(region_id) == region_map_.cend()); + + auto region = std::make_shared(id_, region_id); + region_map_[region_id] = region; + assert(region->Size() == ret_msg.mmsg.size); + assert((reinterpret_cast(region->Addr()) & (~kRegionMask)) == 0); + + MIDAS_LOG(kDebug) << "Allocated region: " << region->Addr() << " [" + << region->Size() << "]"; + overcommit ? stats_.nr_evac_alloced++ : stats_.nr_alloced++; + return region_id; +} + +void ResourceManager::FreeRegion(int64_t rid) noexcept { + stats_.nr_freed++; + std::unique_lock lk(mtx_); + auto region_iter = region_map_.find(rid); + if (region_iter == region_map_.cend()) { + MIDAS_LOG(kError) << "Invalid region_id " << rid; + return; + } + int64_t freed_bytes = free_region(region_iter->second, false); + if (freed_bytes == -1) { + MIDAS_LOG(kError) << "Failed to free region " << rid; + } +} + +void ResourceManager::FreeRegions(size_t size) noexcept { + std::unique_lock lk(mtx_); + size_t total_freed = 0; + int nr_freed_regions = 0; + while (!region_map_.empty()) { + auto region = region_map_.begin()->second; + int64_t freed_bytes = free_region(region, false); + if (freed_bytes == -1) { + MIDAS_LOG(kError) << "Failed to free region " << region->ID(); + continue; + } + total_freed += freed_bytes; + nr_freed_regions++; + if (total_freed >= size) + break; + } + MIDAS_LOG(kInfo) << "Freed " << nr_freed_regions << " regions (" + << total_freed << "bytes)"; +} + +/** This function is supposed to be called inside a locked section */ +inline size_t ResourceManager::free_region(std::shared_ptr region, + bool enforce) noexcept { + int64_t rid = region->ID(); + uint64_t rsize = region->Size(); + /* Only stash freed region into the free list when: + * (1) not enforce free (happen in reclamation); + * (2) still have avail memory budget (or else we will need to return extra + memory allocated by the evacuator); + * (3) freelist is not full. + */ + if (kEnableFreeList && !enforce && NumRegionAvail() > 0 && + freelist_.size() < kFreeListSize) { + region->unmap(); + freelist_.emplace_back(region); + } else { + CtrlMsg msg{.id = id_, + .op = CtrlOpCode::FREE, + .mmsg = {.region_id = rid, .size = rsize}}; + txqp_.send(&msg, sizeof(msg)); + + CtrlMsg ack; + unsigned prio; + int ret = txqp_.recv(&ack, sizeof(ack)); + assert(ret == 0); + if (ack.op != CtrlOpCode::FREE || ack.ret != CtrlRetCode::MEM_SUCC) + return -1; + MIDAS_LOG(kDebug) << "Free region " << rid << " @ " << region->Addr(); + } + + region_map_.erase(rid); + if (NumRegionAvail() > 0) + cv_.notify_all(); + MIDAS_LOG(kDebug) << "region_map size: " << region_map_.size(); + return rsize; +} + +} // namespace midas diff --git a/src/sig_handler.cpp b/src/sig_handler.cpp new file mode 100644 index 0000000..3bd3ef6 --- /dev/null +++ b/src/sig_handler.cpp @@ -0,0 +1,100 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "logging.hpp" +#include "sig_handler.hpp" + +extern "C" void print_callstack(siginfo_t *info, ucontext_t *ctx); + +namespace midas { +void SigHandler::init() { + // register rmemcpy + register_func(reinterpret_cast(&rmemcpy), + reinterpret_cast(&rmemcpy_end)); +} + +void SigHandler::register_func(uint64_t stt_ip, uint64_t end_ip) { + assert(stt_ip <= end_ip); + funcs.emplace_back(stt_ip, end_ip); +} + +ResilientFunc *SigHandler::dispatcher(uint64_t ip) { + ResilientFunc *ret = nullptr; + for (auto &f : funcs) { + if (f.contain(ip)) { + ret = &f; + break; + } + } + return ret; +} + +bool SigHandler::softfault_handler(siginfo_t *info, ucontext_t *ctx) { + if (!in_volatile_range((uint64_t)info->si_addr)) + return false; + + void *ip = (void *)ctx->uc_mcontext.gregs[REG_RIP]; + uint64_t *bp = (uint64_t *)ctx->uc_mcontext.gregs[REG_RBP]; + MIDAS_LOG_PRINTF(kDebug, "fault @ %p, ip = %p, bp = %p\n", info->si_addr, ip, + bp); + // return false; + + auto func = dispatcher(ctx->uc_mcontext.gregs[REG_RIP]); + if (!func) + return false; + + // fault handling + if (func->omitted_frame_pointer) { // jump to ret directly + ctx->uc_mcontext.gregs[REG_RIP] = func->fail_entry; + ctx->uc_mcontext.gregs[REG_RAX] = 0; // return value + } else { // return to the upper level stack + ctx->uc_mcontext.gregs[REG_RIP] = bp[1]; + ctx->uc_mcontext.gregs[REG_RBP] = bp[0]; + ctx->uc_mcontext.gregs[REG_RSP] = reinterpret_cast(bp); + ctx->uc_mcontext.gregs[REG_RAX] = 0; // return value + } + + MIDAS_LOG_PRINTF(kDebug, "return to ip = %p, rbp = %p, rsp = %p\n", + (void *)ctx->uc_mcontext.gregs[REG_RIP], + (void *)ctx->uc_mcontext.gregs[REG_RBP], + (void *)ctx->uc_mcontext.gregs[REG_RSP]); + return true; +} + +static inline bool softfault_handler(siginfo_t *info, ucontext_t *ptr) { + return SigHandler::global_sighandler()->softfault_handler(info, ptr); +} + +static void signal_segv(int signum, siginfo_t *info, void *ptr) { + ucontext_t *ctx = reinterpret_cast(ptr); + MIDAS_LOG(kDebug) << "Segmentation Fault!"; + if (softfault_handler(info, ctx)) + return; + MIDAS_LOG(kError) << "Cannot handle error @ " << ptr; + // print_callstack(info, ctx); + std::cerr << "Stack Trace:\n" << boost::stacktrace::stacktrace() << std::endl; + + exit(-1); +} + +void setup_sigsegv() { + struct sigaction action; + memset(&action, 0, sizeof(action)); + action.sa_sigaction = signal_segv; + action.sa_flags = SA_SIGINFO; + if (sigaction(SIGSEGV, &action, NULL) < 0) + MIDAS_LOG(kError) << "sigaction failed!"; +} + +SigHandler::SigHandler() { setup_sigsegv(); } + +} // namespace midas \ No newline at end of file diff --git a/src/slab.cpp b/src/slab.cpp new file mode 100644 index 0000000..c58a290 --- /dev/null +++ b/src/slab.cpp @@ -0,0 +1,118 @@ +#ifdef ENABLE_SLAB + +#include +#include + +#include "logging.hpp" +#include "resource_manager.hpp" +#include "slab.hpp" +#include "utils.hpp" + +namespace midas { + +static inline SlabHeader *slab_header(const void *ptr) noexcept { + uint64_t addr = reinterpret_cast(ptr); + return reinterpret_cast(addr & kRegionMask); +} + +uint32_t SlabRegion::init() { + // init header + SlabHeader *hdr = slab_header(stt_addr); + hdr->slab_id = this->slab_id; + hdr->slab_size = this->slab_size; + hdr->region = this; + // MIDAS_LOG(kDebug) << this; + // init freelist + uint32_t i; + char *ptr = reinterpret_cast(stt_addr); + ptr += slab_size; + for (i = 0; i < capacity; i++, ptr += slab_size) { + push(ptr); + } + + return i; +} + +inline void SlabRegion::push(void *addr) { + assert(nr_alloced > 0); + /* In place construct a FreeSlot */ + FreeSlot *slot = reinterpret_cast(addr); + /* YIFAN: this is actually unnecessary for the in-place slot list. Leave it + * here in case later we change the design */ + slot->addr = addr; + + /* Push the new slot into the front */ + ret_mtx->lock(); + slot->next = ret_slots; + ret_slots = slot; + nr_freed++; + ret_mtx->unlock(); + + // MIDAS_LOG(kDebug) << "push " << addr; +} + +inline void *SlabRegion::pop() { + if (UNLIKELY(!slots || nr_alloced == capacity)) { + if (UNLIKELY(!ret_slots)) + return nullptr; + else { // slow path + ret_mtx->lock(); + slots = ret_slots; + ret_slots = nullptr; + nr_alloced -= nr_freed; + nr_freed = 0; + ret_mtx->unlock(); + } + } + + FreeSlot *slot = slots; + slots = slot->next; + + /* Manual memset *slot to 0 */ + slot->addr = 0; + slot->next = 0; + + nr_alloced++; + return slot; +} + +void *SlabAllocator::_alloc(uint32_t size) { + uint32_t idx = get_slab_idx(size); + uint32_t slab_size = get_slab_size(idx); + assert(idx < kNumSlabClasses); + + // MIDAS_LOG(kDebug) << slab_size << " " << idx; + + for (auto ®ion : slab_regions[idx]) { + // MIDAS_LOG(kDebug) << region.full(); + if (!region->full()) + return region->pop(); + } + + /* Slow path: allocate a new region */ + auto *rmanager = ResourceManager::global_manager(); + int rid = rmanager->AllocRegion(); + if (rid == -1) + return nullptr; + VRange range = rmanager->GetRegion(rid); + slab_regions[idx].push_back( + std::make_shared(idx, slab_size, range.stt_addr, range.size)); + + return slab_regions[idx].back()->pop(); +} + +void SlabAllocator::free(void *addr) { + /* cannot equal since the first slot is the slab header */ + assert(reinterpret_cast(addr) > kVolatileSttAddr); + SlabHeader *hdr = slab_header(addr); + auto *region = hdr->region; + // MIDAS_LOG(kDebug) << region; + region->push(addr); +} + +thread_local std::vector> + SlabAllocator::slab_regions[kNumSlabClasses]; + +} // namespace midas + +#endif // ENABLE_SLAB \ No newline at end of file diff --git a/src/stacktrace.cpp b/src/stacktrace.cpp new file mode 100644 index 0000000..3813cb7 --- /dev/null +++ b/src/stacktrace.cpp @@ -0,0 +1,70 @@ +extern "C" { +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#ifndef NO_CPP_DEMANGLE +#include +#ifdef __cplusplus +using __cxxabiv1::__cxa_demangle; +#endif +#endif + +void print_callstack(siginfo_t *info, ucontext_t *ctx) { + const bool verbose = false; + static const char *si_codes[3] = {"", "SEGV_MAPERR", "SEGV_ACCERR"}; + + fprintf(stderr, "info.si_signo = %d\n", info->si_signo); + fprintf(stderr, "info.si_errno = %d\n", info->si_errno); + fprintf(stderr, "info.si_code = %d (%s)\n", info->si_code, + si_codes[info->si_code]); + fprintf(stderr, "info.si_addr = %p\n", info->si_addr); + if (verbose) + for (int i = 0; i < NGREG; i++) + fprintf(stderr, "reg[%02d] = 0x%llx\n", i, + ctx->uc_mcontext.gregs[i]); + + int f = 0; + Dl_info dlinfo; + void *ip = (void *)ctx->uc_mcontext.gregs[REG_RIP]; + void **bp = (void **)ctx->uc_mcontext.gregs[REG_RBP]; + fprintf(stderr, "ip = %p,\tbp = %p\n", ip, bp); + + fprintf(stderr, "Stack trace:\n"); + while (bp && ip) { + if (!dladdr(ip, &dlinfo)) + break; + + const char *symname = dlinfo.dli_sname; + +#ifndef NO_CPP_DEMANGLE + int status; + char *tmp = __cxa_demangle(symname, NULL, 0, &status); + + if (status == 0 && tmp) + symname = tmp; +#endif + + fprintf(stderr, "% 2d: %p <%s+%lu> (%s)\n", ++f, ip, symname, + (unsigned long)ip - (unsigned long)dlinfo.dli_saddr, + dlinfo.dli_fname); + +#ifndef NO_CPP_DEMANGLE + if (tmp) + free(tmp); +#endif + + if (dlinfo.dli_sname && !strcmp(dlinfo.dli_sname, "main")) + break; + + ip = bp[1]; + bp = (void **)bp[0]; + } +} +} \ No newline at end of file diff --git a/test/boost_shm_apis.cpp b/test/boost_shm_apis.cpp new file mode 100644 index 0000000..c9f4462 --- /dev/null +++ b/test/boost_shm_apis.cpp @@ -0,0 +1,76 @@ + +int ipc() { + try { + // { + // message_queue t_mq(create_only, + // "mq" // name + // , 20, sizeof(int) + // ); + // } + + // opening the message queue whose name is mq + message_queue mq(open_only, // only open + "mq" // name + ); + size_t recvd_size; + unsigned int priority = 0; + + for (int i = 0; i < 20; ++i) { + mq.send((void *)&i, sizeof(int), priority); + } + + // now send the messages to the queue + for (int i = 0; i < 20; ++i) { + int buffer; + mq.receive((void *)&buffer, sizeof(int), recvd_size, priority); + if (recvd_size != sizeof(int)) + ; // do the error handling + std::cout << buffer << " " << recvd_size << " " << priority << std::endl; + } + } catch (interprocess_exception &e) { + std::cout << e.what() << std::endl; + } + + return 0; +} + +int shm(int argc, char *argv[]) { + if (argc == 1) { // Parent process + // Remove shared memory on construction and destruction + struct shm_remove { + shm_remove() { shared_memory_object::remove("MySharedMemory"); } + ~shm_remove() { shared_memory_object::remove("MySharedMemory"); } + } remover; + + // Create a shared memory object. + shared_memory_object shm(create_only, "MySharedMemory", read_write); + + // Set size + shm.truncate(1000); + + // Map the whole shared memory in this process + mapped_region region(shm, read_write); + + // Write all the memory to 1 + std::memset(region.get_address(), 1, region.get_size()); + + // Launch child process + std::string s(argv[0]); + s += " child "; + if (0 != std::system(s.c_str())) + return 1; + } else { + // Open already created shared memory object. + shared_memory_object shm(open_only, "MySharedMemory", read_only); + + // Map the whole shared memory in this process + mapped_region region(shm, read_only); + + // Check that memory was initialized to 1 + char *mem = static_cast(region.get_address()); + for (std::size_t i = 0; i < region.get_size(); ++i) + if (*mem++ != 1) + return 1; // Error checking memory + } + return 0; +} \ No newline at end of file diff --git a/test/test_batched_kv.cpp b/test/test_batched_kv.cpp new file mode 100644 index 0000000..3ae5f2c --- /dev/null +++ b/test/test_batched_kv.cpp @@ -0,0 +1,283 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "resource_manager.hpp" +#include "sync_kv.hpp" + +#define TEST_OBJECT 1 +#define TEST_LARGE 1 + +constexpr static size_t kCacheSize = 1024ull * 1024 * 2000; +constexpr static int kNBuckets = (1 << 20); +constexpr static int kNumInsertThds = 10; +constexpr static int kNumRemoveThds = 10; +constexpr static int kBatchSize = 10; + +#if TEST_LARGE +constexpr static int kNumObjs = 10240; +constexpr static int kKLen = 32; +constexpr static int kVLen = 8192; +#else // !TEST_LARGE +constexpr static int kNumObjs = 102400; +constexpr static int kKLen = 61; +constexpr static int kVLen = 10; +#endif // TEST_LARGE +static_assert(kNumObjs % kBatchSize == 0, + "#(Objects) doesn't align with batch size!"); + +template struct Object; + +#if TEST_OBJECT +using K = Object; +using V = Object; +#else // !TEST_OBJECT +using K = int; +using V = int; +#endif // TEST_OBJECT + +/** YIFAN: string is not supported! Its reference will be lost after data copy + */ +// using K = std::string; +// using V = std::string; + +template struct Object { + char data[Len]; + bool operator==(const Object &other) const { + return strncmp(data, other.data, Len) == 0; + } + bool operator!=(const Object &other) const { + return strncmp(data, other.data, Len) != 0; + } + bool operator<(const Object &other) const { + return strncmp(data, other.data, Len) < 0; + } + void random_fill() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution dist('A', 'z'); + + for (uint32_t i = 0; i < Len - 1; i++) + data[i] = dist(mt); + data[Len - 1] = 0; + } +}; + +template K get_K() { return K(); } +template V get_V() { return V(); } + +template <> std::string get_K() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution dist('A', 'z'); + + std::string str = ""; + str.reserve(kKLen); + for (uint32_t i = 0; i < kKLen - 1; i++) + str += dist(mt); + str += '\0'; + return str; +} + +template <> std::string get_V() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution dist('A', 'z'); + + std::string str = ""; + str.reserve(kVLen); +#if TEST_LARGE == 1 + constexpr static int kFillStride = 2000; + for (uint32_t i = 0; i < kVLen - 1; i++) + if (i % kFillStride == 0) + str += dist(mt); + else + str += static_cast(i % ('z' - 'A')) + 'A'; +#else // !TEST_LARGE + for (uint32_t i = 0; i < kVLen - 1; i++) + str += dist(mt); +#endif + str += '\0'; + return str; +} + +template <> int get_K() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution dist(0, 1 << 30); + return dist(mt); +} + +template <> int get_V() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution dist(0, 1 << 30); + return dist(mt); +} + +template <> Object get_K() { + Object k; + k.random_fill(); + return k; +} + +template <> Object get_V() { + Object v; + v.random_fill(); + return v; +} +namespace std { +template <> struct hash> { + size_t operator()(const Object &k) const { + return std::hash()( + std::string_view(k.data, strlen(k.data))); + } +}; +} // namespace std + +struct Op { + enum OpCode { Set, Get, Remove } opcode; + K key; + V val; +}; +std::vector ks[kNumInsertThds]; +std::vector vs[kNumInsertThds]; +std::vector ops[kNumInsertThds]; + +void gen_workload() { + std::vector thds; + for (int tid = 0; tid < kNumInsertThds; tid++) { + for (int o = 0; o < kNumObjs; o++) { + K k = get_K(); + V v = get_V(); + ks[tid].push_back(k); + vs[tid].push_back(v); + Op op{.opcode = Op::Set, .key = k, .val = v}; + ops[tid].push_back(op); + } + } + for (auto &thd : thds) + thd.join(); + std::cout << "Finish generate workload." << std::endl; +} + +int main(int argc, char *argv[]) { + auto *rmanager = midas::ResourceManager::global_manager(); + rmanager->UpdateLimit(kCacheSize); + + auto *kvstore = new midas::SyncKV(); + + std::atomic_int32_t nr_succ{0}; + std::atomic_int32_t nr_err{0}; + + gen_workload(); + + std::vector thds; + for (int tid = 0; tid < kNumInsertThds; tid++) { + thds.push_back(std::thread([&, tid = tid]() { + for (int i = 0; i < kNumObjs; i += kBatchSize) { + std::vector keys; + std::vector values; + for (int j = i; j < i + kBatchSize; j++) { + auto &k = ks[tid][j]; + auto &v = vs[tid][j]; + keys.emplace_back(midas::kv_utils::make_key(&k, sizeof(k))); + values.emplace_back(midas::kv_utils::make_cvalue(&v, sizeof(v))); + } + int ret = kvstore->bset(keys, values); + nr_succ += ret; + nr_err += kBatchSize - ret; + } + })); + } + for (auto &thd : thds) + thd.join(); + thds.clear(); + + if (nr_err == 0) + std::cout << "Set test passed!" << std::endl; + else + std::cout << "Set test failed! " << nr_succ << " passed, " << nr_err + << " failed." << std::endl; + + std::atomic_int32_t nr_equal{0}; + std::atomic_int32_t nr_nequal{0}; + nr_succ = nr_err = 0; + for (int tid = 0; tid < kNumInsertThds; tid++) { + thds.push_back(std::thread([&, tid = tid]() { + for (int i = 0; i < kNumObjs; i += kBatchSize) { + std::vector keys; + std::vector values; + for (int j = i; j < i + kBatchSize; j++) { + auto &k = ks[tid][j]; + keys.emplace_back(midas::kv_utils::make_key(&k, sizeof(k))); + } + int succ = kvstore->bget(keys, values); + nr_succ += succ; + nr_err += kBatchSize - succ; + assert(values.size() == kBatchSize); + for (int j = 0; j < kBatchSize; j++) { + auto &v = vs[tid][i + j]; + V *v2 = reinterpret_cast(values[j].data); + size_t stored_vn = values[j].size; + if (v2) { + assert(stored_vn == sizeof(V)); + if (v == *v2) { + nr_equal++; + } else { + nr_nequal++; + } + free(v2); + } + } + } + })); + } + for (auto &thd : thds) + thd.join(); + thds.clear(); + + if (nr_nequal == 0) + std::cout << "Get test passed! " << nr_succ << " passed, " << nr_err + << " failed. " << nr_equal << "/" << nr_equal + nr_nequal + << " pairs are equal" << std::endl; + else + std::cout << "Get test failed! " << nr_succ << " passed, " << nr_err + << " failed. " << nr_equal << "/" << nr_equal + nr_nequal + << " pairs are equal" << std::endl + << "NOTE: a small amount of failures are expected if only " + "std_map is protected by lock, as keys can conflict in our " + "sync_hash_map and the result of races are uncertain." + << std::endl; + + nr_succ = nr_err = 0; + for (int tid = 0; tid < kNumRemoveThds; tid++) { + thds.push_back(std::thread([&, tid = tid]() { + for (int i = 0; i < kNumObjs; i += kBatchSize) { + std::vector keys; + for (int j = i; j < i + kBatchSize; j++) { + auto &k = ks[tid][j]; + keys.emplace_back(midas::kv_utils::make_key(&k, sizeof(k))); + } + int succ = kvstore->bremove(keys); + nr_succ += succ; + nr_err += kBatchSize - succ; + } + })); + } + for (auto &thd : thds) + thd.join(); + + if (nr_err == 0) + std::cout << "Remove test passed!" << std::endl; + else + std::cout << "Remove test failed! " << nr_succ << " passed, " << nr_err + << " failed." << std::endl; + return 0; +} \ No newline at end of file diff --git a/test/test_cache_manager.cpp b/test/test_cache_manager.cpp new file mode 100644 index 0000000..83d143c --- /dev/null +++ b/test/test_cache_manager.cpp @@ -0,0 +1,34 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cache_manager.hpp" +#include "log.hpp" +#include "object.hpp" +#include "resource_manager.hpp" + +constexpr static int kNumPools = 100; + +int main() { + auto resource_manager = midas::ResourceManager::global_manager(); + std::this_thread::sleep_for(std::chrono::milliseconds(1000)); // to establish connection + auto cache_manager = midas::CacheManager::global_cache_manager(); + for (int i = 0; i < kNumPools; i++) { + std::string pool_name = "test" + std::to_string(i); + cache_manager->create_pool(pool_name); + std::cout << "Num pools: " << cache_manager->num_pools() << std::endl; + } + + for (int i = 0; i < kNumPools; i++) { + std::string pool_name = "test" + std::to_string(i); + cache_manager->delete_pool(pool_name); + std::cout << "Num pools: " << cache_manager->num_pools() << std::endl; + } + + return 0; +} \ No newline at end of file diff --git a/test/test_concurrent_evacuator.cpp b/test/test_concurrent_evacuator.cpp new file mode 100644 index 0000000..54a174b --- /dev/null +++ b/test/test_concurrent_evacuator.cpp @@ -0,0 +1,117 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "evacuator.hpp" +#include "log.hpp" +#include "object.hpp" + +constexpr int kNumGCThds = 2; +constexpr int kNumThds = 10; +constexpr int kNumRepeat = 100; +constexpr int kNumObjs = 40960; + +constexpr int kObjSize = 111; + +struct Object { + char data[kObjSize]; + + void random_fill() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution dist('A', 'z'); + + for (uint32_t i = 0; i < kObjSize; i++) { + data[i] = dist(mt); + } + } + + bool equal(Object &other) { + return (strncmp(data, other.data, kObjSize) == 0); + } +}; + +int main(int argc, char *argv[]) { + std::vector threads; + + std::atomic_int nr_errs(0); + std::vector> ptrs[kNumThds]; + std::vector objs[kNumThds]; + + for (int tid = 0; tid < kNumThds; tid++) { + threads.push_back(std::thread([&, tid = tid]() { + auto *allocator = midas::LogAllocator::global_allocator(); + for (int i = 0; i < kNumObjs; i++) { + Object obj; + obj.random_fill(); + auto objptr = std::make_shared(); + + if (!allocator->alloc_to(sizeof(Object), objptr.get()) || + !objptr->copy_from(&obj, sizeof(Object))) { + nr_errs++; + continue; + } + ptrs[tid].push_back(objptr); + objs[tid].push_back(obj); + } + + for (int i = 0; i < ptrs[tid].size(); i++) { + bool ret = false; + auto ptr = ptrs[tid][i]; + Object stored_o; + if (!ptr->copy_to(&stored_o, sizeof(Object)) || + !objs[tid][i].equal(stored_o)) + nr_errs++; + } + })); + } + + for (auto &thd : threads) + thd.join(); + threads.clear(); + + bool stop_evac = false; + std::thread evac_thd([&]() { + midas::Evacuator evacuator; + while (!stop_evac) { + evacuator.gc(); + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + }); + + for (int tid = 0; tid < kNumThds; tid++) { + threads.push_back(std::thread([&, tid = tid]() { + for (int j = 0; j < kNumRepeat; j++) { + auto nr_ptrs = ptrs[tid].size(); + for (int i = 0; i < nr_ptrs; i++) { + auto ptr = ptrs[tid][i]; + Object stored_o; + if (!ptr || !ptr->copy_to(&stored_o, sizeof(Object)) || + !objs[tid][i].equal(stored_o)) + nr_errs++; + } + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + })); + } + for (auto &thd : threads) + thd.join(); + threads.clear(); + std::this_thread::sleep_for(std::chrono::milliseconds(5000)); + stop_evac = true; + evac_thd.join(); + + if (nr_errs == 0) + std::cout << "Test passed!" << std::endl; + else + std::cout << "Test failed, nr_errs: " << nr_errs << std::endl + << "Note: errors are expected when evacute period is short, and " + "objects are evicted." + << std::endl; + return 0; +} \ No newline at end of file diff --git a/test/test_concurrent_evacuator2.cpp b/test/test_concurrent_evacuator2.cpp new file mode 100644 index 0000000..8ffd473 --- /dev/null +++ b/test/test_concurrent_evacuator2.cpp @@ -0,0 +1,118 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "evacuator.hpp" +#include "log.hpp" +#include "object.hpp" + +constexpr int kNumGCThds = 2; +constexpr int kNumThds = 10; +constexpr int kNumRepeat = 100; +constexpr int kNumObjs = 40960; + +constexpr int kObjSize = 111; + +struct Object { + char data[kObjSize]; + + void random_fill() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution dist('A', 'z'); + + for (uint32_t i = 0; i < kObjSize; i++) { + data[i] = dist(mt); + } + } + + bool equal(Object &other) { + return (strncmp(data, other.data, kObjSize) == 0); + } +}; + +int main(int argc, char *argv[]) { + std::vector threads; + + bool stop_evac = false; + std::thread evac_thd([&]() { + midas::Evacuator evacuator; + while (!stop_evac) { + evacuator.gc(); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + }); + + std::atomic_int nr_errs(0); + std::vector> ptrs[kNumThds]; + std::vector objs[kNumThds]; + + for (int tid = 0; tid < kNumThds; tid++) { + threads.push_back(std::thread([&, tid = tid]() { + for (int i = 0; i < kNumObjs; i++) { + auto *allocator = midas::LogAllocator::global_allocator(); + Object obj; + obj.random_fill(); + auto objptr = std::make_shared(); + + if (!allocator->alloc_to(sizeof(Object), objptr.get()) || + !objptr->copy_from(&obj, sizeof(Object))) { + nr_errs++; + continue; + } + ptrs[tid].push_back(objptr); + objs[tid].push_back(obj); + } + + for (int i = 0; i < ptrs[tid].size(); i++) { + bool ret = false; + auto ptr = ptrs[tid][i]; + Object stored_o; + if (!ptr->copy_to(&stored_o, sizeof(Object)) || + !objs[tid][i].equal(stored_o)) + nr_errs++; + } + })); + } + + for (auto &thd : threads) + thd.join(); + threads.clear(); + + for (int tid = 0; tid < kNumThds; tid++) { + threads.push_back(std::thread([&, tid = tid]() { + for (int j = 0; j < kNumRepeat; j++) { + auto nr_ptrs = ptrs[tid].size(); + for (int i = 0; i < nr_ptrs; i++) { + auto ptr = ptrs[tid][i]; + Object stored_o; + if (!ptr || !ptr->copy_to(&stored_o, sizeof(Object)) || + !objs[tid][i].equal(stored_o)) + nr_errs++; + } + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + })); + } + for (auto &thd : threads) + thd.join(); + threads.clear(); + std::this_thread::sleep_for(std::chrono::milliseconds(5000)); + stop_evac = true; + evac_thd.join(); + + if (nr_errs == 0) + std::cout << "Test passed!" << std::endl; + else + std::cout << "Test failed, nr_errs: " << nr_errs << std::endl + << "Note: errors are expected when evacute period is short, and " + "objects are evicted." + << std::endl; + + return 0; +} \ No newline at end of file diff --git a/test/test_concurrent_evacuator3.cpp b/test/test_concurrent_evacuator3.cpp new file mode 100644 index 0000000..9bda460 --- /dev/null +++ b/test/test_concurrent_evacuator3.cpp @@ -0,0 +1,173 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "evacuator.hpp" +#include "log.hpp" +#include "resource_manager.hpp" +#include "slab.hpp" +#include "sync_hashmap.hpp" + +constexpr int kNBuckets = (1 << 20); +constexpr int kNumMutatorThds = 20; +constexpr int kNumGCThds = 3; +constexpr int kNumOps = 409600; +constexpr int kKLen = 18; +constexpr int kVLen = 31; + +constexpr float kSetRatio = 70; +constexpr float kGetRatio = 20; +constexpr float kRmvRatio = 10; + +template struct Object; + +using K = Object; +using V = Object; + +/** Define Object */ +template struct Object { + char data[Len]; + bool operator==(const Object &other) const { + return strncmp(data, other.data, Len) == 0; + } + bool operator!=(const Object &other) const { + return strncmp(data, other.data, Len) != 0; + } + bool operator<(const Object &other) const { + return strncmp(data, other.data, Len) < 0; + } + void random_fill() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution dist('A', 'z'); + + for (uint32_t i = 0; i < Len - 1; i++) + data[i] = dist(mt); + data[Len - 1] = 0; + } +}; + +Object get_K() { + Object k; + k.random_fill(); + return k; +} +Object get_V() { + Object v; + v.random_fill(); + return v; +} +namespace std { +template <> struct hash> { + size_t operator()(const Object &k) const { + return std::hash()( + std::string_view(k.data, strlen(k.data))); + } +}; +} // namespace std +/** Define Object [End] */ + +/** Generate requests */ +struct Op { + enum OpCode { Set, Get, Remove } opcode; + K key; + V val; +}; +std::vector ks[kNumMutatorThds]; +std::vector vs[kNumMutatorThds]; +std::vector ops[kNumMutatorThds]; + +void gen_workload() { + for (int tid = 0; tid < kNumMutatorThds; tid++) { + for (int o = 0; o < kNumOps; o++) { + K k = get_K(); + V v = get_V(); + ks[tid].push_back(k); + vs[tid].push_back(v); + Op op{.opcode = Op::Set, .key = k, .val = v}; + ops[tid].push_back(op); + op.opcode = Op::Get; + ops[tid].push_back(op); + // op.opcode = Op::Remove; + // ops[tid].push_back(op); + } + + for (int o = 0; o < kNumOps; o++) { + Op op{.opcode = Op::Get, .key = ks[tid][o], .val = vs[tid][o]}; + ops[tid].push_back(op); + } + + // for (int o = 0; o < kNumOps; o++) { + // Op op { .opcode = Op::Remove, .key = ks[tid][o], .val = vs[tid][o] }; + // ops[tid].push_back(op); + // } + } + std::cout << "Finish generate workload." << std::endl; +} + +int main(int argc, char *argv[]) { + auto *rmanager = midas::ResourceManager::global_manager(); + std::vector thds; + + gen_workload(); + bool stop = false; + std::thread evac_thd([&]() { + midas::Evacuator evacuator; + while (!stop) { + evacuator.gc(); + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + } + }); + + auto *hashmap = new midas::SyncHashMap(); + std::unordered_map std_maps[kNumMutatorThds]; + + std::atomic_int32_t nr_succ = 0; + std::atomic_int32_t nr_err = 0; + + for (int tid = 0; tid < kNumMutatorThds; tid++) { + thds.push_back(std::thread([&, tid = tid]() { + for (auto &op : ops[tid]) { + auto k = op.key; + auto v = op.val; + bool ret = true; + if (op.opcode == Op::Set) + ret = hashmap->set(k, v); + else if (op.opcode == Op::Get) { + auto v = hashmap->get(k); + } else if (op.opcode == Op::Remove) { + hashmap->remove(k); + } else { + std::cerr << "should not reach here!" << std::endl; + } + + if (!ret) + nr_err++; + else + nr_succ++; + } + })); + } + for (auto &thd : thds) + thd.join(); + thds.clear(); + + std::this_thread::sleep_for(std::chrono::milliseconds(4000)); + stop = true; + evac_thd.join(); + + if (nr_err == 0) + std::cout << "Test passed!" << std::endl; + else + std::cout << "Tet test failed! " << nr_succ << " passed, " << nr_err + << " failed." << std::endl; + + return 0; +} \ No newline at end of file diff --git a/test/test_feat_extractor.cpp b/test/test_feat_extractor.cpp new file mode 100644 index 0000000..67aeec4 --- /dev/null +++ b/test/test_feat_extractor.cpp @@ -0,0 +1,534 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cache_manager.hpp" +#include "sync_hashmap.hpp" +#include "time.hpp" +#include "zipf.hpp" + +constexpr static int kFeatDim = 2048; +constexpr static int kMD5Len = 32; + +constexpr static int kMissPenalty = 1; // ms +constexpr static int kNrThd = 24; +constexpr static int KPerThdLoad = 100000; +constexpr static int kNumBuckets = 1 << 20; + +constexpr static bool kSkewedDist = true; // false for uniform distribution +constexpr static double kSkewness = 0.9; // zipf + +constexpr static bool kSimulate = true; +constexpr static int kSimuNumImgs = 1000 * 1000; + +constexpr static int kStatInterval = 2; // seconds + +const static std::string data_dir = + "/mnt/ssd/yifan/code/cachebank/apps/FeatureExtraction/data/"; +const static std::string md5_filename = data_dir + "md5.txt"; +const static std::string img_filename = data_dir + "val_img_names.txt"; +const static std::string feat_filename = data_dir + "enb5_feat_vec.data"; + +const static std::string cachepool_name = "feats"; +float cache_ratio = 1.0; +size_t cache_size = (kSimulate ? kSimuNumImgs : 41620ull) * (80 + 8192); + +struct MD5Key { + char data[kMD5Len]; + + const std::string to_string() { + char str[kMD5Len + 1]; + std::memcpy(str, data, kMD5Len); + str[kMD5Len] = '\0'; + return str; + } +}; + +struct Feature { + float data[kFeatDim]; +}; +static_assert(sizeof(Feature) == sizeof(float) * kFeatDim, + "Feature struct size incorrect"); + +inline void md5_from_file(MD5Key &md5_result, const std::string &filename) { + std::ifstream file(filename, std::ifstream::binary); + MD5_CTX md5Context; + MD5_Init(&md5Context); + char buf[1024 * 16]; + while (file.good()) { + file.read(buf, sizeof(buf)); + MD5_Update(&md5Context, buf, file.gcount()); + } + unsigned char result[MD5_DIGEST_LENGTH]; + MD5_Final(result, &md5Context); + + std::stringstream md5str_stream; + md5str_stream << std::hex << std::uppercase << std::setfill('0'); + for (const auto &byte : result) + md5str_stream << std::setw(2) << (int)byte; + md5str_stream << "\0"; + std::memcpy(md5_result.data, md5str_stream.str().c_str(), kMD5Len); +} + +namespace std { +template <> struct hash { + size_t operator()(const MD5Key &k) const { + return std::hash()(std::string_view(k.data, kMD5Len)); + } +}; + +template <> struct equal_to { + size_t operator()(const MD5Key &k1, const MD5Key &k2) const { + return std::memcmp(k1.data, k2.data, kMD5Len) == 0; + } +}; +} // namespace std + +class FakeBackend { +public: + FakeBackend() : _arrival_req_id(-1), _processed_req_id(-1), _alive(true) { + int kProcessors = 2; + for (int i = 0; i < kProcessors; i++) { + processor_thds.push_back(std::thread([&]() { processor(); })); + } + } + ~FakeBackend() { + _alive = false; + { + std::unique_lock lk(_p_mtx); + _p_cv.notify_all(); + } + for (auto &thd : processor_thds) + thd.join(); + } + Feature *serve_req() { + int req_id = 0; + { + std::unique_lock plk(_p_mtx); + req_id = _arrival_req_id.fetch_add(1) + 1; + } + _p_cv.notify_all(); + while (_processed_req_id.load() < req_id) + std::this_thread::sleep_for(std::chrono::microseconds(100)); + auto ret = new Feature(); + return ret; + } + +private: + int processor() { + while (_alive) { + std::unique_lock plk(_p_mtx); + _p_cv.wait(plk, [&] { + return !_alive || _arrival_req_id.load() > _processed_req_id.load(); + }); + + while (_arrival_req_id.load() > _processed_req_id.load()) { + std::this_thread::sleep_for(std::chrono::milliseconds(kMissPenalty)); + _processed_req_id.fetch_add(1); + } + } + return 0; + } + std::mutex _p_mtx; + std::condition_variable _p_cv; + + std::atomic _arrival_req_id; + std::atomic _processed_req_id; + + bool _alive; + std::vector processor_thds; +}; + +struct FeatReq { + int tid; + int rid; + std::string filename; + Feature *feat; + uint64_t start_us; +}; + +struct Trace { + uint64_t absl_start_us; + uint64_t start_us; + uint64_t duration; +}; + +class FeatExtractor { +public: + FeatExtractor(); + ~FeatExtractor(); + int warmup_cache(); + int simu_warmup_cache(); + void perf(uint64_t miss_ddl_us = 10ul * 1000 * 1000); // 10s + void gen_load(); + +private: + size_t load_imgs(); + size_t load_feats(); + + bool serve_req(const FeatReq &img_req); + // To re-construct cache-miss objects + int construct_callback(void *arg); + + FakeBackend fakeGPUBackend; + + size_t nr_imgs; + std::vector imgs; + char *raw_feats; + std::vector> feats; + + struct { + int nr_hit = 0; + int nr_miss = 0; + } perthd_cnts[kNrThd]; + void report_hit_rate(); + std::vector reqs[kNrThd]; + std::shared_ptr gens[kNrThd]; + + midas::CachePool *cpool; + std::shared_ptr> feat_map; +}; + +/** initialization & utils */ +FeatExtractor::FeatExtractor() : raw_feats(nullptr), nr_imgs(0) { + cpool = midas::CacheManager::global_cache_manager()->get_pool(cachepool_name); + if (!cpool) { + std::cerr << "Failed to get cache pool!" << std::endl; + exit(-1); + } + feat_map = + std::make_unique>(cpool); + load_imgs(); + load_feats(); + nr_imgs = kSimulate ? kSimuNumImgs : imgs.size(); + + cpool->set_construct_func( + [&](void *args) { return construct_callback(args); }); + + for (int i = 0; i < kNrThd; i++) { + std::random_device rd; + gens[i] = std::make_shared(rd()); + } + // gen_load(); +} + +FeatExtractor::~FeatExtractor() { + if (raw_feats) + delete[] raw_feats; +} + +int FeatExtractor::construct_callback(void *arg) { + auto args_ = reinterpret_cast(arg); + auto md5 = reinterpret_cast(args_->key); + auto feat_buf = reinterpret_cast(args_->value); + assert(args_->key_len == sizeof(MD5Key)); + + auto feat = fakeGPUBackend.serve_req(); + if (feat_buf) { + assert(args_->value_len == sizeof(Feature)); + std::memcpy(feat_buf, feat, sizeof(Feature)); + delete feat; + } else { + args_->value = feat; + args_->value_len = sizeof(Feature); + } + + return 0; +} + +void FeatExtractor::gen_load() { + midas::zipf_table_distribution<> zipf_dist(nr_imgs, kSkewness); + std::uniform_int_distribution<> uni_dist(0, nr_imgs - 1); + + int nr_tests = 10; + std::vector target_kopss; + std::vector durations; + for (int i = 0; i < nr_tests; i++) { + target_kopss.emplace_back(i + 1); + durations.emplace_back(10); // seconds + } + const uint64_t us = 1000 * 1000; + uint64_t transit_dur = 10; // seconds + int transit_stages = 10; + uint64_t stage_us = transit_dur * us / transit_stages; + + std::vector thds; + for (int tid = 0; tid < kNrThd; tid++) { + thds.emplace_back([&, tid = tid] { + reqs[tid].clear(); + uint64_t cur_us = 0; + for (int i = 0; i < nr_tests; i++) { + auto target_kops = target_kopss[i]; + auto duration_us = cur_us + durations[i] * us; // seconds -> us + std::exponential_distribution ed(target_kops / 1000 / kNrThd); + while (cur_us < duration_us) { + auto interval = std::max(1l, std::lround(ed(*gens[tid]))); + int id = kSkewedDist ? zipf_dist(*gens[tid]) : uni_dist(*gens[tid]); + id = nr_imgs - 1 - id; + FeatReq req{ + .tid = tid, .rid = id, .filename = imgs.at(id % imgs.size()), + .feat = feats.at(id % feats.size()).get(), + .start_us = cur_us}; + reqs[tid].emplace_back(req); + cur_us += interval; + } + if (i == nr_tests - 1) // transition + break; + const auto transit_kops_step = + (target_kopss[i + 1] - target_kopss[i]) / transit_stages; + for (int j = 0; j < transit_stages; j++) { + auto transit_kops = target_kops + transit_kops_step * j; + std::exponential_distribution ed(transit_kops / 1000 / + kNrThd); + auto duration_us = cur_us + stage_us; + while (cur_us < duration_us) { + auto interval = std::max(1l, std::lround(ed(*gens[tid]))); + int id = kSkewedDist ? zipf_dist(*gens[tid]) : uni_dist(*gens[tid]); + id = nr_imgs - 1 - id; + FeatReq req{.tid = tid, + .rid = id, + .filename = imgs.at(id % imgs.size()), + .feat = feats.at(id % feats.size()).get(), + .start_us = cur_us}; + reqs[tid].emplace_back(req); + cur_us += interval; + } + } + } + }); + } + for (auto &thd : thds) + thd.join(); + std::cout << "Finish load generation." << std::endl; +} + +bool FeatExtractor::serve_req(const FeatReq &req) { + MD5Key md5; + md5_from_file(md5, req.filename); + if (kSimulate) { + std::ostringstream oss; + oss << std::setw(32) << std::setfill('0') << req.rid; + std::string md5_str = oss.str(); + md5_str.copy(md5.data, kMD5Len); + } + auto feat_opt = feat_map->get(md5); + if (feat_opt) { + perthd_cnts[req.tid].nr_hit++; + return true; + } + + // Cache miss + auto stt = midas::Time::get_cycles_stt(); + perthd_cnts[req.tid].nr_miss++; + midas::ConstructArgs args{.key = &md5, + .key_len = sizeof(md5), + .value = req.feat, + .value_len = sizeof(Feature)}; + assert(cpool->construct(&args) == 0); + feat_map->set(md5, req.feat); + auto end = midas::Time::get_cycles_end(); + + cpool->record_miss_penalty(end - stt, sizeof(*req.feat)); + return true; +} + +size_t FeatExtractor::load_imgs() { + std::ifstream img_file(img_filename, std::ifstream::in); + if (!img_file.good()) { + std::cerr << "cannot open img_file " << img_filename << std::endl; + return -1; + } + + while (img_file.good()) { + std::string name; + std::getline(img_file, name); + if (!name.empty()) + imgs.push_back(name); + // std::cout << name << " " << imgs.at(imgs.size() - 1) << std::endl; + } + + size_t nr_imgs = imgs.size(); + MD5Key md5; + md5_from_file(md5, imgs[0]); + std::cout << "Load " << nr_imgs << " images, MD5 of " << imgs[0] << ": " + << md5.to_string() << std::endl; + + return nr_imgs; +} + +size_t FeatExtractor::load_feats() { + size_t nr_imgs = imgs.size(); + raw_feats = new char[nr_imgs * kFeatDim * sizeof(float)]; + size_t nr_feat_vecs = 0; + + std::ifstream feat_file(feat_filename, std::ifstream::binary); + if (feat_file.good()) { + feat_file.read(raw_feats, nr_imgs * sizeof(float) * kFeatDim); + } + + char *ptr = raw_feats; + for (int i = 0; i < nr_imgs; i++) { + auto new_feat = std::make_shared(); + feats.emplace_back(new_feat); + std::memcpy(new_feat->data, ptr, sizeof(float) * kFeatDim); + ptr += sizeof(float) * kFeatDim; + } + + return feats.size(); +} + +int FeatExtractor::warmup_cache() { + std::ifstream md5_file(md5_filename); + + size_t nr_imgs = imgs.size(); + std::cout << nr_imgs << " " << feats.size() << std::endl; + // for (int i = 0; i < nr_imgs * cache_ratio; i++) { + for (int i = 0; i < nr_imgs; i++) { + MD5Key md5; + std::string md5_str; + md5_file >> md5_str; + md5_str.copy(md5.data, kMD5Len); + + // md5_from_file(md5, imgs.at(i)); + // std::cout << imgs.at(i) << " " << md5 << std::endl; + feat_map->set(md5, *feats[i]); + } + std::cout << "Done warm up cache" << std::endl; + return 0; +} + +int FeatExtractor::simu_warmup_cache() { + std::cout << "Warming up cache with synthetic data..." << std::endl; + std::vector thds; + for (int tid = 0; tid < kNrThd; tid++) { + thds.push_back(std::thread([&, tid = tid] { + const auto chunk = (nr_imgs + kNrThd - 1) / kNrThd; + auto stt = chunk * tid; + auto end = std::min(stt + chunk, nr_imgs); + for (int i = stt; i < end; i++) { + MD5Key md5; + std::ostringstream oss; + oss << std::setw(32) << std::setfill('0') << i; + std::string md5_str = oss.str(); + md5_str.copy(md5.data, kMD5Len); + + feat_map->set(md5, *feats[i % feats.size()]); + } + })); + } + for (auto &thd : thds) + thd.join(); + std::cout << "Done warm up cache" << std::endl; + return 0; +} + +void FeatExtractor::perf(uint64_t miss_ddl_us) { + gen_load(); + + std::atomic_int_fast32_t nr_succ{0}; + bool stop = false; + auto stt = std::chrono::high_resolution_clock::now(); + std::thread perf_thd([&] { + while (!stop) { + std::this_thread::sleep_for(std::chrono::seconds(kStatInterval)); + std::cout << "Tput " << nr_succ / 1000.0 / kStatInterval << " Kops" + << std::endl; + nr_succ = 0; + } + }); + + std::vector all_traces[kNrThd]; + std::vector worker_thds; + for (int tid = 0; tid < kNrThd; tid++) { + worker_thds.emplace_back(std::thread([&, tid = tid] { + auto start_us = midas::Time::get_us_stt(); + auto &thd_reqs = reqs[tid]; + auto &thd_traces = all_traces[tid]; + int cnt = 0; + for (auto &req : thd_reqs) { + auto relative_us = midas::Time::get_us_stt() - start_us; + if (req.start_us > relative_us) { + std::this_thread::sleep_for( + std::chrono::microseconds(req.start_us - relative_us)); + } else if (req.start_us + miss_ddl_us < relative_us) { + continue; + } + Trace trace; + trace.absl_start_us = midas::Time::get_us_stt(); + trace.start_us = trace.absl_start_us - start_us; + serve_req(req); + trace.duration = + midas::Time::get_us_stt() - start_us - trace.start_us; + thd_traces.emplace_back(trace); + cnt++; + if (cnt % 100 == 0) { + nr_succ += 100; + cnt = 0; + } + } + })); + } + + for (auto &thd : worker_thds) { + thd.join(); + } + stop = true; + perf_thd.join(); + + auto end = std::chrono::high_resolution_clock::now(); + auto duration = + std::chrono::duration_cast(end - stt).count(); + auto tput = static_cast(kNrThd * KPerThdLoad) / duration; + std::cout << "Perf done. Duration: " << duration + << " ms, Throughput: " << tput << " Kops" << std::endl; + report_hit_rate(); + + std::vector gathered_traces; + for (int i = 0; i < kNrThd; i++) { + gathered_traces.insert(gathered_traces.end(), all_traces[i].begin(), + all_traces[i].end()); + } + std::cout << gathered_traces.size() << std::endl; +} + +void FeatExtractor::report_hit_rate() { + int nr_hit = 0; + int nr_miss = 0; + for (int i = 0; i < kNrThd; i++) { + nr_hit += perthd_cnts[i].nr_hit; + nr_miss += perthd_cnts[i].nr_miss; + } + std::cout << "Cache hit ratio = " << nr_hit << "/" << nr_hit + nr_miss + << " = " << 1.0 * nr_hit / (nr_hit + nr_miss) << std::endl; +} + +int main(int argc, char *argv[]) { + if (argc <= 1) { + std::cout << "Usage: ./" << argv[0] << " " << std::endl; + exit(-1); + } + cache_ratio = std::stof(argv[1]); + midas::CacheManager::global_cache_manager()->create_pool(cachepool_name); + auto pool = midas::CacheManager::global_cache_manager()->get_pool(cachepool_name); + pool->update_limit(cache_size * cache_ratio); + + FeatExtractor client; + if (kSimulate) + client.simu_warmup_cache(); + else + client.warmup_cache(); + client.perf(); + std::cout << "Test passed!" << std::endl; + return 0; +} \ No newline at end of file diff --git a/test/test_feat_extractor_kv.cpp b/test/test_feat_extractor_kv.cpp new file mode 100644 index 0000000..b0e73ca --- /dev/null +++ b/test/test_feat_extractor_kv.cpp @@ -0,0 +1,428 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cache_manager.hpp" +#include "sync_kv.hpp" +#include "time.hpp" +#include "zipf.hpp" + +constexpr static int kFeatDim = 2048; +constexpr static int kMD5Len = 32; + +constexpr static int kMissPenalty = 1; // ms +constexpr static int kNrThd = 24; +constexpr static int KPerThdLoad = 5000; +constexpr static int kNumBuckets = 1 << 20; + +constexpr static bool kSkewedDist = true; // false for uniform distribution +constexpr static double kSkewness = 0.9; // zipf + +constexpr static bool kSimulate = true; +constexpr static int kSimuNumImgs = 1000 * 1000; + +const static std::string data_dir = + "/mnt/ssd/yifan/code/cachebank/apps/FeatureExtraction/data/"; +const static std::string md5_filename = data_dir + "md5.txt"; +const static std::string img_filename = data_dir + "val_img_names.txt"; +const static std::string feat_filename = data_dir + "enb5_feat_vec.data"; + +float cache_ratio = 1.0; +size_t cache_size = (kSimulate ? kSimuNumImgs : 41620ull) * (80 + 8192); + +struct MD5Key { + char data[kMD5Len]; + + const std::string to_string() { + char str[kMD5Len + 1]; + std::memcpy(str, data, kMD5Len); + str[kMD5Len] = '\0'; + return str; + } +}; + +struct Feature { + float data[kFeatDim]; +}; +static_assert(sizeof(Feature) == sizeof(float) * kFeatDim, + "Feature struct size incorrect"); + +inline void md5_from_file(MD5Key &md5_result, const std::string &filename) { + std::ifstream file(filename, std::ifstream::binary); + MD5_CTX md5Context; + MD5_Init(&md5Context); + char buf[1024 * 16]; + while (file.good()) { + file.read(buf, sizeof(buf)); + MD5_Update(&md5Context, buf, file.gcount()); + } + unsigned char result[MD5_DIGEST_LENGTH]; + MD5_Final(result, &md5Context); + + std::stringstream md5str_stream; + md5str_stream << std::hex << std::uppercase << std::setfill('0'); + for (const auto &byte : result) + md5str_stream << std::setw(2) << (int)byte; + md5str_stream << "\0"; + std::memcpy(md5_result.data, md5str_stream.str().c_str(), kMD5Len); +} + +namespace std { +template <> struct hash { + size_t operator()(const MD5Key &k) const { + return std::hash()(std::string_view(k.data, kMD5Len)); + } +}; + +template <> struct equal_to { + size_t operator()(const MD5Key &k1, const MD5Key &k2) const { + return std::memcmp(k1.data, k2.data, kMD5Len) == 0; + } +}; +} // namespace std + +class FakeBackend { +public: + FakeBackend() : _arrival_req_id(-1), _processed_req_id(-1), _alive(true) { + int kProcessors = 2; + for (int i = 0; i < kProcessors; i++) { + processor_thds.push_back(std::thread([&]() { processor(); })); + } + } + ~FakeBackend() { + _alive = false; + { + std::unique_lock lk(_p_mtx); + _p_cv.notify_all(); + } + for (auto &thd : processor_thds) + thd.join(); + } + Feature *serve_req() { + int req_id = 0; + { + std::unique_lock plk(_p_mtx); + req_id = _arrival_req_id.fetch_add(1) + 1; + } + _p_cv.notify_all(); + while (_processed_req_id.load() < req_id) + std::this_thread::sleep_for(std::chrono::microseconds(100)); + auto ret = new Feature(); + return ret; + } + +private: + int processor() { + while (_alive) { + std::unique_lock plk(_p_mtx); + _p_cv.wait(plk, [&] { + return !_alive || _arrival_req_id.load() > _processed_req_id.load(); + }); + + while (_arrival_req_id.load() > _processed_req_id.load()) { + std::this_thread::sleep_for(std::chrono::milliseconds(kMissPenalty)); + _processed_req_id.fetch_add(1); + } + } + return 0; + } + std::mutex _p_mtx; + std::condition_variable _p_cv; + + std::atomic _arrival_req_id; + std::atomic _processed_req_id; + + bool _alive; + std::vector processor_thds; +}; + +struct FeatReq { + int tid; + int rid; + std::string filename; + Feature *feat; +}; + +class FeatExtractor { +public: + FeatExtractor(); + ~FeatExtractor(); + int warmup_cache(); + int simu_warmup_cache(); + void perf(); + +private: + size_t load_imgs(); + size_t load_feats(); + + void gen_load(); + bool serve_req(const FeatReq &img_req); + // To re-construct cache-miss objects + int construct_callback(void *arg); + + FakeBackend fakeGPUBackend; + + size_t nr_imgs; + std::vector imgs; + char *raw_feats; + std::vector> feats; + + struct { + int nr_hit = 0; + int nr_miss = 0; + } perthd_cnts[kNrThd]; + void report_hit_rate(); + std::vector reqs[kNrThd]; + std::shared_ptr gens[kNrThd]; + + std::shared_ptr> feat_map; +}; + +/** initialization & utils */ +FeatExtractor::FeatExtractor() : raw_feats(nullptr), nr_imgs(0) { + feat_map = + std::make_unique>(); + load_imgs(); + load_feats(); + nr_imgs = kSimulate ? kSimuNumImgs : imgs.size(); + + auto cpool = midas::CachePool::global_cache_pool(); + cpool->set_construct_func( + [&](void *args) { return construct_callback(args); }); + + for (int i = 0; i < kNrThd; i++) { + std::random_device rd; + gens[i] = std::make_shared(rd()); + } + gen_load(); +} + +FeatExtractor::~FeatExtractor() { + if (raw_feats) + delete[] raw_feats; +} + +int FeatExtractor::construct_callback(void *arg) { + auto args_ = reinterpret_cast(arg); + auto md5 = reinterpret_cast(args_->key); + auto feat_buf = reinterpret_cast(args_->value); + assert(args_->key_len == sizeof(MD5Key)); + + auto feat = fakeGPUBackend.serve_req(); + if (feat_buf) { + assert(args_->value_len == sizeof(Feature)); + std::memcpy(feat_buf, feat, sizeof(Feature)); + delete feat; + } else { + args_->value = feat; + args_->value_len = sizeof(Feature); + } + + return 0; +} + +void FeatExtractor::gen_load() { + midas::zipf_table_distribution<> zipf_dist(nr_imgs, kSkewness); + std::uniform_int_distribution<> uni_dist(0, nr_imgs - 1); + + std::vector thds; + for (int tid = 0; tid < kNrThd; tid++) { + reqs[tid].clear(); + for (int o = 0; o < KPerThdLoad; o++) { + int id = kSkewedDist ? zipf_dist(*gens[tid]) : uni_dist(*gens[tid]); + id = nr_imgs - 1 - id; + FeatReq req{.tid = tid, + .rid = id, + .filename = imgs.at(id % imgs.size()), + .feat = feats.at(id % feats.size()).get()}; + reqs[tid].push_back(req); + } + } + std::cout << "Finish load generation." << std::endl; +} + +bool FeatExtractor::serve_req(const FeatReq &req) { + auto cpool = midas::CachePool::global_cache_pool(); + + MD5Key md5; + md5_from_file(md5, req.filename); + if (kSimulate) { + std::ostringstream oss; + oss << std::setw(32) << std::setfill('0') << req.rid; + std::string md5_str = oss.str(); + md5_str.copy(md5.data, kMD5Len); + } + size_t feat_size = 0; + if (feat_map->get(&md5, sizeof(md5), req.feat, sizeof(Feature))) { + perthd_cnts[req.tid].nr_hit++; + return true; + } + + // Cache miss + auto stt = midas::Time::get_cycles_stt(); + perthd_cnts[req.tid].nr_miss++; + midas::ConstructArgs args{.key = &md5, + .key_len = sizeof(md5), + .value = req.feat, + .value_len = sizeof(Feature)}; + assert(cpool->construct(&args) == 0); + feat_map->set(&md5, sizeof(md5), req.feat, sizeof(Feature)); + auto end = midas::Time::get_cycles_end(); + + cpool->record_miss_penalty(end - stt, sizeof(*req.feat)); + return true; +} + +size_t FeatExtractor::load_imgs() { + std::ifstream img_file(img_filename, std::ifstream::in); + if (!img_file.good()) { + std::cerr << "cannot open img_file " << img_filename << std::endl; + return -1; + } + + while (img_file.good()) { + std::string name; + std::getline(img_file, name); + if (!name.empty()) + imgs.push_back(name); + // std::cout << name << " " << imgs.at(imgs.size() - 1) << std::endl; + } + + size_t nr_imgs = imgs.size(); + MD5Key md5; + md5_from_file(md5, imgs[0]); + std::cout << "Load " << nr_imgs << " images, MD5 of " << imgs[0] << ": " + << md5.to_string() << std::endl; + + return nr_imgs; +} + +size_t FeatExtractor::load_feats() { + size_t nr_imgs = imgs.size(); + raw_feats = new char[nr_imgs * kFeatDim * sizeof(float)]; + size_t nr_feat_vecs = 0; + + std::ifstream feat_file(feat_filename, std::ifstream::binary); + if (feat_file.good()) { + feat_file.read(raw_feats, nr_imgs * sizeof(float) * kFeatDim); + } + + char *ptr = raw_feats; + for (int i = 0; i < nr_imgs; i++) { + auto new_feat = std::make_shared(); + feats.emplace_back(new_feat); + std::memcpy(new_feat->data, ptr, sizeof(float) * kFeatDim); + ptr += sizeof(float) * kFeatDim; + } + + return feats.size(); +} + +int FeatExtractor::warmup_cache() { + std::ifstream md5_file(md5_filename); + + size_t nr_imgs = imgs.size(); + std::cout << nr_imgs << " " << feats.size() << std::endl; + // for (int i = 0; i < nr_imgs * cache_ratio; i++) { + for (int i = 0; i < nr_imgs; i++) { + MD5Key md5; + std::string md5_str; + md5_file >> md5_str; + md5_str.copy(md5.data, kMD5Len); + + // md5_from_file(md5, imgs.at(i)); + // std::cout << imgs.at(i) << " " << md5 << std::endl; + feat_map->set(&md5, sizeof(md5), feats[i].get(), sizeof(Feature)); + } + std::cout << "Done warm up cache" << std::endl; + return 0; +} + +int FeatExtractor::simu_warmup_cache() { + std::cout << "Warming up cache with synthetic data..." << std::endl; + std::vector thds; + for (int tid = 0; tid < kNrThd; tid++) { + thds.push_back(std::thread([&, tid = tid] { + const auto chunk = (nr_imgs + kNrThd - 1) / kNrThd; + auto stt = chunk * tid; + auto end = std::min(stt + chunk, nr_imgs); + for (int i = stt; i < end; i++) { + MD5Key md5; + std::ostringstream oss; + oss << std::setw(32) << std::setfill('0') << i; + std::string md5_str = oss.str(); + md5_str.copy(md5.data, kMD5Len); + + feat_map->set(&md5, sizeof(md5), feats[i % feats.size()].get(), + sizeof(Feature)); + } + })); + } + for (auto &thd : thds) + thd.join(); + std::cout << "Done warm up cache" << std::endl; + return 0; +} + +void FeatExtractor::perf() { + auto stt = std::chrono::high_resolution_clock::now(); + std::vector worker_thds; + for (int tid = 0; tid < kNrThd; tid++) { + worker_thds.push_back(std::thread([&, tid = tid]() { + for (int i = 0; i < KPerThdLoad; i++) { + // auto req = gen_req(tid); + serve_req(reqs[tid][i]); + } + })); + } + + for (auto &thd : worker_thds) { + thd.join(); + } + + auto end = std::chrono::high_resolution_clock::now(); + auto duration = + std::chrono::duration_cast(end - stt).count(); + auto tput = static_cast(kNrThd * KPerThdLoad) / duration; + std::cout << "Perf done. Duration: " << duration + << " ms, Throughput: " << tput << " Kops" << std::endl; + report_hit_rate(); +} + +void FeatExtractor::report_hit_rate() { + int nr_hit = 0; + int nr_miss = 0; + for (int i = 0; i < kNrThd; i++) { + nr_hit += perthd_cnts[i].nr_hit; + nr_miss += perthd_cnts[i].nr_miss; + } + std::cout << "Cache hit ratio = " << nr_hit << "/" << nr_hit + nr_miss + << " = " << 1.0 * nr_hit / (nr_hit + nr_miss) << std::endl; +} + +int main(int argc, char *argv[]) { + if (argc <= 1) { + std::cout << "Usage: ./" << argv[0] << " " << std::endl; + exit(-1); + } + cache_ratio = std::stof(argv[1]); + midas::CachePool::global_cache_pool()->update_limit(cache_size * cache_ratio); + + FeatExtractor client; + if (kSimulate) + client.simu_warmup_cache(); + else + client.warmup_cache(); + client.perf(); + std::cout << "Test passed!" << std::endl; + return 0; +} \ No newline at end of file diff --git a/test/test_fs_shim.cpp b/test/test_fs_shim.cpp new file mode 100644 index 0000000..f759207 --- /dev/null +++ b/test/test_fs_shim.cpp @@ -0,0 +1,109 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +// #include "fs_shim.hpp" + +constexpr static int32_t kNumEles = 100 * 1024 * 1024; +constexpr static int32_t kStride = 100; +constexpr static int32_t kRepeat = 4; +constexpr static char kFileName[] = "/tmp/test_fs_shim.data"; + +int *src_arr; +int *dst_arr; + +std::atomic_int32_t nr_succ{0}; +std::atomic_int32_t nr_fail{0}; + +int init() { + src_arr = new int[kNumEles]; + dst_arr = new int[kNumEles]; + return 0; +} + +int destory() { + delete[] src_arr; + delete[] dst_arr; + return 0; +} + +int prep() { + std::memset(src_arr, 0, sizeof(int) * kNumEles); + std::memset(src_arr, 0, sizeof(int) * kNumEles); + static std::mt19937 mt(rand()); + static std::uniform_int_distribution<> dist(std::numeric_limits::max()); + + std::cout << "Prep..." << std::endl; + for (int i = 0; i < kNumEles; i+= kStride) + src_arr[i] = dist(mt); + std::cout << "Prep done!" << std::endl; + + return 0; +} + +int write() { + std::ofstream outfile; + outfile.open(kFileName); + for(int i = 0; i < kNumEles; i++) { + outfile << src_arr[i] << "\n"; + } + outfile.close(); + + std::cout << "Write done!" << std::endl; + + return 0; +} + +int read() { + std::ifstream infile; + infile.open(kFileName); + int cnt = 0; + while(infile >> dst_arr[cnt]) { + cnt++; + } + infile.close(); + + if (cnt != kNumEles) { + std::cerr << "Read failed! " << cnt << " out of " << kNumEles + << " were read" << std::endl; + nr_fail++; + } + + for (int i = 0; i < cnt; i++) { + if (dst_arr[i] != src_arr[i]) { + std::cerr << "Check failed @ ele[" << i << "], " << dst_arr[i] + << " != " << src_arr[i] << std::endl; + nr_fail++; + break; + } + } + std::cout << "Read done!" << std::endl; + + return 0; +} + +int main(int argc, char *argv[]) { + init(); + for (int i = 0; i < kRepeat; i++) { + prep(); + write(); + read(); + read(); + read(); + } + destory(); + if (nr_fail == 0) + std::cout << "Test passed!" << std::endl; + else + std::cout << "Test failed!" << std::endl; + return 0; +} \ No newline at end of file diff --git a/test/test_hashmap_clear.cpp b/test/test_hashmap_clear.cpp new file mode 100644 index 0000000..13e0ca9 --- /dev/null +++ b/test/test_hashmap_clear.cpp @@ -0,0 +1,270 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "resource_manager.hpp" +#include "slab.hpp" +#include "sync_hashmap.hpp" + +constexpr int kNBuckets = (1 << 16); +constexpr int kNumInsertThds = 40; +constexpr int kNumObjs = 102400; +constexpr int kKLen = 61; +constexpr int kVLen = 10; + +template struct Object; + +// using K = int; +// using V = int; +using K = Object; +using V = Object; + +/** YIFAN: string is not supported! Its reference will be lost after data copy + */ +// using K = std::string; +// using V = std::string; + +template struct Object { + char data[Len]; + bool operator==(const Object &other) const { + return strncmp(data, other.data, Len) == 0; + } + bool operator!=(const Object &other) const { + return strncmp(data, other.data, Len) != 0; + } + bool operator<(const Object &other) const { + return strncmp(data, other.data, Len) < 0; + } + void random_fill() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution dist('A', 'z'); + + for (uint32_t i = 0; i < Len - 1; i++) + data[i] = dist(mt); + data[Len - 1] = 0; + } +}; + +template K get_K() { return K(); } +template V get_V() { return V(); } + +template <> std::string get_K() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution dist('A', 'z'); + + std::string str = ""; + for (uint32_t i = 0; i < kKLen - 1; i++) + str += dist(mt); + str += '\0'; + return str; +} + +template <> std::string get_V() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution dist('A', 'z'); + + std::string str = ""; + for (uint32_t i = 0; i < kVLen - 1; i++) + str += dist(mt); + str += '\0'; + return str; +} + +template <> int get_K() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution dist(0, 1 << 30); + return dist(mt); +} + +template <> int get_V() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution dist(0, 1 << 30); + return dist(mt); +} + +template <> Object get_K() { + Object k; + k.random_fill(); + return k; +} + +template <> Object get_V() { + Object v; + v.random_fill(); + return v; +} +namespace std { +template <> struct hash> { + size_t operator()(const Object &k) const { + return std::hash()( + std::string_view(k.data, strlen(k.data))); + } +}; +} // namespace std + +struct Op { + enum OpCode { Set, Get, Remove } opcode; + K key; + V val; +}; +std::vector ks[kNumInsertThds]; +std::vector vs[kNumInsertThds]; +std::vector ops[kNumInsertThds]; + +void gen_workload() { + std::vector thds; + for (int tid = 0; tid < kNumInsertThds; tid++) { + for (int o = 0; o < kNumObjs; o++) { + K k = get_K(); + V v = get_V(); + ks[tid].push_back(k); + vs[tid].push_back(v); + Op op{.opcode = Op::Set, .key = k, .val = v}; + ops[tid].push_back(op); + } + } + for (auto &thd : thds) + thd.join(); + std::cout << "Finish generate workload." << std::endl; +} + +int main(int argc, char *argv[]) { + auto *rmanager = midas::ResourceManager::global_manager(); + std::vector thds; + + auto *hashmap = new midas::SyncHashMap(); + std::mutex std_map_lock; + std::unordered_map std_map; + + std::atomic_int32_t nr_succ = 0; + std::atomic_int32_t nr_err = 0; + + gen_workload(); + + hashmap->clear(); + for (int tid = 0; tid < kNumInsertThds; tid++) { + thds.push_back(std::thread([&, tid = tid]() { + for (int i = 0; i < kNumObjs; i++) { + auto &k = ks[tid][i]; + auto &v = vs[tid][i]; + std_map_lock.lock(); + std_map[k] = v; + std_map_lock.unlock(); + bool ret = hashmap->set(k, v); + if (!ret) + nr_err++; + else + nr_succ++; + } + })); + } + for (auto &thd : thds) + thd.join(); + thds.clear(); + + if (nr_err == 0) + std::cout << "Set test passed!" << std::endl; + else + std::cout << "Set test failed! " << nr_succ << " passed, " << nr_err + << " failed." << std::endl; + + nr_succ = nr_err = 0; + for (int tid = 0; tid < kNumInsertThds; tid++) { + thds.push_back(std::thread([&]() { + for (auto &pair : std_map) { + const K &k = pair.first; + V &v = pair.second; + auto v2 = hashmap->get(k); + + if (!v2 || v != *v2) { + nr_err++; + } else + nr_succ++; + } + })); + } + for (auto &thd : thds) + thd.join(); + thds.clear(); + + if (nr_err == 0) + std::cout << "Get test passed!" << std::endl; + else + std::cout << "Get test failed! " << nr_succ << " passed, " << nr_err + << " failed." << std::endl + << "NOTE: a small amount of failures are expected if only " + "std_map is protected by lock, as keys can conflict in our " + "sync_hash_map and the result of races are uncertain." + << std::endl; + + if (hashmap->clear()) { + std::cout << "Clear test passed!" << std::endl; + } else { + std::cout << "Clear test failed!" << std::endl; + } + + for (int tid = 0; tid < kNumInsertThds; tid++) { + thds.push_back(std::thread([&, tid = tid]() { + for (int i = 0; i < kNumObjs; i++) { + auto &k = ks[tid][i]; + auto &v = vs[tid][i]; + bool ret = hashmap->set(k, v); + if (!ret) + nr_err++; + else + nr_succ++; + } + })); + } + for (auto &thd : thds) + thd.join(); + thds.clear(); + + if (nr_err == 0) + std::cout << "2nd round Set test passed!" << std::endl; + else + std::cout << "2nd round Set test failed! " << nr_succ << " passed, " << nr_err + << " failed." << std::endl; + + nr_succ = nr_err = 0; + for (int tid = 0; tid < kNumInsertThds; tid++) { + thds.push_back(std::thread([&]() { + for (auto &pair : std_map) { + const K &k = pair.first; + V &v = pair.second; + auto v2 = hashmap->get(k); + + if (!v2 || v != *v2) { + nr_err++; + } else + nr_succ++; + } + })); + } + for (auto &thd : thds) + thd.join(); + thds.clear(); + + if (nr_err == 0) + std::cout << "2nd round Get test passed!" << std::endl; + else + std::cout << "2nd round Get test failed! " << nr_succ << " passed, " << nr_err + << " failed." << std::endl + << "NOTE: a small amount of failures are expected if only " + "std_map is protected by lock, as keys can conflict in our " + "sync_hash_map and the result of races are uncertain." + << std::endl; + + return 0; +} \ No newline at end of file diff --git a/test/test_large_alloc.cpp b/test/test_large_alloc.cpp new file mode 100644 index 0000000..49e877e --- /dev/null +++ b/test/test_large_alloc.cpp @@ -0,0 +1,135 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "log.hpp" +#include "object.hpp" + +constexpr static int kNumThds = 10; +constexpr static int kNumObjs = 10240; + +constexpr static int kObjMinSize = 8 * 1000; +constexpr static int kObjMaxSize = 9 * 1000; + +int random_obj_size() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution dist(kObjMinSize, kObjMaxSize); + + return dist(mt); +} + +struct Object { + int obj_size; + char *data; + + Object() : Object(random_obj_size()) {} + Object(int obj_size_) : obj_size(obj_size_) { + data = new char[obj_size]; + } + ~Object() { + delete[] data; + } + + void random_fill() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution dist('A', 'z'); + + constexpr static int kFillStride = 10; + for (uint32_t i = 0; i < obj_size / kFillStride; i++) { + data[i * kFillStride] = dist(mt); + } + // memcpy(data, &obj_size, sizeof(int)); + } + + bool equal(Object &other) { + return (strncmp(data, other.data, obj_size) == 0); + } +}; + +std::vector> objs[kNumThds]; +void gen_workload() { + std::cout << "Generating workload..." << std::endl; + for (int tid = 0; tid < kNumThds; tid++) { + for (int i = 0; i < kNumObjs; i++) { + auto obj = std::make_shared(); + obj->random_fill(); + objs[tid].push_back(obj); + } + } + std::cout << "Finish generating workload!" << std::endl; +} + +int main(int argc, char *argv[]) { + auto *allocator = midas::LogAllocator::global_allocator(); + + gen_workload(); + + std::atomic_int nr_errs(0); + std::atomic_int nr_eqs(0); + std::atomic_int nr_neqs(0); + std::vector> ptrs[kNumThds]; + + std::vector threads; + for (int tid = 0; tid < kNumThds; tid++) { + threads.push_back(std::thread([&, tid = tid]() { + for (int i = 0; i < kNumObjs; i++) { + const auto &obj = objs[tid][i]; + const auto size = obj->obj_size; + + bool ret = false; + + auto objptr = std::make_shared(); + ptrs[tid].push_back(objptr); + if (!allocator->alloc_to(size, objptr.get()) || + !(ret = objptr->copy_from(obj->data, size))) { + nr_errs++; + } + } + + for (int i = 0; i < ptrs[tid].size(); i++) { + bool ret = false; + const auto &obj = objs[tid][i]; + const auto size = obj->obj_size; + + auto ptr = ptrs[tid][i]; + Object stored_o(size); + if (!ptr->copy_to(stored_o.data, size)) { + nr_errs++; + continue; + } + if (!objs[tid][i]->equal(stored_o)) { + nr_neqs++; + } else { + nr_eqs++; + } + } + + for (auto ptr : ptrs[tid]) { + if (!allocator->free(*ptr)) + // nr_errs++; + continue; + } + })); + } + + for (auto &thd : threads) { + thd.join(); + } + + std::cout << "Equal/Not-equal/Total: " << nr_eqs << "/" << nr_neqs << "/" + << kNumObjs * kNumThds << std::endl; + if (nr_errs == 0) + std::cout << "Test passed!" << std::endl; + else + std::cout << "Test failed: " << nr_errs << "/" << kNumObjs * kNumThds + << " failed cases." << std::endl; + + return 0; +} \ No newline at end of file diff --git a/test/test_log.cpp b/test/test_log.cpp new file mode 100644 index 0000000..e3d33bf --- /dev/null +++ b/test/test_log.cpp @@ -0,0 +1,86 @@ +#include +#include +#include +#include +#include +#include +#include + +#include "log.hpp" +#include "object.hpp" + +constexpr int kNumThds = 10; +constexpr int kNumObjs = 102400; + +constexpr int kObjSize = 16; + +struct Object { + char data[kObjSize]; + + void random_fill() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution dist('A', 'z'); + + for (uint32_t i = 0; i < kObjSize; i++) { + data[i] = dist(mt); + } + } + + bool equal(Object &other) { + return (strncmp(data, other.data, kObjSize) == 0); + } +}; + +int main(int argc, char *argv[]) { + auto *allocator = midas::LogAllocator::global_allocator(); + std::vector threads; + + std::atomic_int nr_errs(0); + std::vector> ptrs[kNumThds]; + std::vector objs[kNumThds]; + + for (int tid = 0; tid < kNumThds; tid++) { + threads.push_back(std::thread([&, tid = tid]() { + for (int i = 0; i < kNumObjs; i++) { + Object obj; + obj.random_fill(); + auto objptr = std::make_shared(); + + if (!allocator->alloc_to(sizeof(Object), objptr.get()) || + !objptr->copy_from(&obj, sizeof(Object))) { + nr_errs++; + continue; + } + ptrs[tid].push_back(objptr); + objs[tid].push_back(obj); + } + + for (int i = 0; i < ptrs[tid].size(); i++) { + bool ret = false; + auto ptr = ptrs[tid][i]; + Object stored_o; + if (!ptr->copy_to(&stored_o, sizeof(Object)) || + !objs[tid][i].equal(stored_o)) + nr_errs++; + } + + for (auto ptr : ptrs[tid]) { + if (!allocator->free(*ptr)) + nr_errs++; + } + })); + } + + for (auto &thd : threads) { + thd.join(); + } + + if (nr_errs == 0) + std::cout << "Test passed!" << std::endl; + else + std::cout << "Test failed: " << nr_errs << "/" << kNumObjs * kNumThds + << " failed cases." << std::endl; + + return 0; +} \ No newline at end of file diff --git a/test/test_memcpy.cpp b/test/test_memcpy.cpp new file mode 100644 index 0000000..dc2292b --- /dev/null +++ b/test/test_memcpy.cpp @@ -0,0 +1,116 @@ +#include +#include +#include +#include +#include + +#include "resilient_func.hpp" +#include "time.hpp" + +constexpr static int kBufLens[] = { + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 24, 32, 48, 64, 80, 128, 199, 256, 512, 1024, 2048, 4096}; + +constexpr static int kPerfBufLen = 8; +constexpr static int kNumRepeat = 5000000; + +void random_fill(char buf[], size_t len) { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution dist('A', 'z'); + + for (uint32_t i = 0; i < len; i++) { + buf[i] = dist(mt); + } +} + +bool is_same(const char *src, const char *dst, size_t len) { + while (len-- > 0) { + if (*(src++) != *(dst++)) { + return false; + } + } + return true; +} + +void correctness() { + int nr_test = 0; + int nr_succ = 0; + for (auto len : kBufLens) { + char *src = reinterpret_cast(malloc(len)); + char *dst = reinterpret_cast(malloc(len)); + random_fill(src, len); + auto ret = midas::rmemcpy(dst, src, len); + // memcpy(dst, src, len); + // auto ret = 1; + if (!is_same(src, dst, len)) { + std::cout << "memcpy(len = " << len << ") is wrong!" << std::endl; + } + free(src); + free(dst); + + nr_succ += ret; + nr_test++; + } + if (nr_succ == nr_test) + std::cout << "Test passed! Correct." << std::endl; + else + std::cout << "Test failed! " << nr_succ << "/" << nr_test << " succeeded. " + << std::endl; +} + +void performance(int buf_len) { + std::cout << "Perf test -- buf length = " << buf_len << std::endl; + char *src = new char[buf_len]; + char **dsts = new char *[kNumRepeat]; + + random_fill(src, buf_len); + for (int i = 0; i < kNumRepeat; i++) { + dsts[i] = new char[buf_len]; + memset(dsts[i], 0, buf_len); + } + + double t_memcpy, t_rmemcpy; + { + auto stt = midas::chrono_utils::now(); + for (int i = 0; i < kNumRepeat; i++) { + memcpy(dsts[i], src, buf_len); + } + auto end = midas::chrono_utils::now(); + t_memcpy = midas::chrono_utils::duration(stt, end); + std::cout << " memcpy takes " << std::setprecision(5) << t_memcpy << " s" + << std::endl; + } + + { + auto stt = midas::chrono_utils::now(); + for (int i = 0; i < kNumRepeat; i++) { + midas::rmemcpy(dsts[i], src, buf_len); + } + auto end = midas::chrono_utils::now(); + t_rmemcpy = midas::chrono_utils::duration(stt, end); + std::cout << "rmemcpy takes " << std::setprecision(5) << t_rmemcpy << " s" + << std::endl; + } + std::cout << "Speedup of rmemcpy: " << std::setprecision(5) + << t_memcpy / t_rmemcpy << std::endl; + + delete[] src; + for (int i = 0; i < kNumRepeat; i++) { + delete[] dsts[i]; + } + delete[] dsts; +} + +void perf_loop() { + for (auto len : kBufLens) { + performance(len); + } +} + +int main() { + correctness(); + perf_loop(); + + return 0; +} \ No newline at end of file diff --git a/test/test_object.cpp b/test/test_object.cpp new file mode 100644 index 0000000..af30842 --- /dev/null +++ b/test/test_object.cpp @@ -0,0 +1,41 @@ +#include +#include + +#include "object.hpp" +#include "transient_ptr.hpp" + +int main() { + void *ref = nullptr; + std::cout << "ref pointer addr: " << &ref << std::endl; + + midas::SmallObjectHdr small_obj; + small_obj.init(32, reinterpret_cast(&ref)); + std::cout << std::hex << small_obj.get_size() << " " << small_obj.get_rref() + << std::endl; + std::cout << std::hex << *(reinterpret_cast(&small_obj)) + << std::endl; + small_obj.init(32 * 1024 - 8, reinterpret_cast(&ref)); + std::cout << small_obj.get_size() << " " << std::hex << small_obj.get_rref() + << std::endl; + std::cout << std::hex << *(reinterpret_cast(&small_obj)) + << std::endl; + /* size is too large, failed */ + // small_obj.init(32 * 1024 - 7, reinterpret_cast(&ref)); // failed + // std::cout << small_obj.get_size() << " " << std::hex << small_obj.get_rref() + // << std::endl; + // std::cout << std::hex << *(reinterpret_cast(&small_obj)) + // << std::endl; + + midas::LargeObjectHdr large_obj; + large_obj.init(32 * 1024 - 7, true, + midas::TransientPtr(reinterpret_cast(&ref), + sizeof(midas::LargeObjectHdr)), + midas::TransientPtr()); + std::cout << large_obj.get_size() << " " << std::hex << large_obj.get_rref() + << std::endl; + std::cout << std::hex << *(reinterpret_cast(&large_obj)) << " " + << *(reinterpret_cast(&large_obj) + 1) << " " + << *(reinterpret_cast(&large_obj) + 1) << std::endl; + + return 0; +} \ No newline at end of file diff --git a/test/test_ordered_set.cpp b/test/test_ordered_set.cpp new file mode 100644 index 0000000..2db9f26 --- /dev/null +++ b/test/test_ordered_set.cpp @@ -0,0 +1,295 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "resource_manager.hpp" +#include "sync_kv.hpp" + +#define TEST_OBJECT 0 +#define TEST_LARGE 0 + +constexpr static size_t kCacheSize = 1024ull * 1024 * 2000; +constexpr static int kNBuckets = (1 << 20); +constexpr static int kOSetSize = 20; +constexpr static int kNumInsertThds = 10; +constexpr static int kNumRemoveThds = 1; + +#if TEST_OBJECT +#if TEST_LARGE +constexpr static int kNumObjs = 1024; +constexpr static int kKLen = 32; +constexpr static int kVLen = 8192; +#else // !TEST_LARGE +constexpr static int kNumObjs = 1024; +constexpr static int kKLen = 61; +constexpr static int kVLen = 10; +#endif // TEST_LARGE +#else // !TEST_OBJECT +constexpr static int kNumObjs = 10240; +#endif + +template struct Object; + +#if TEST_OBJECT +using K = Object; +using V = Object; +#else // !TEST_OBJECT +using K = int; +using V = int; +#endif // TEST_OBJECT + +/** YIFAN: string is not supported! Its reference will be lost after data copy + */ +// using K = std::string; +// using V = std::string; + +template struct Object { + char data[Len]; + bool operator==(const Object &other) const { + return strncmp(data, other.data, Len) == 0; + } + bool operator!=(const Object &other) const { + return strncmp(data, other.data, Len) != 0; + } + bool operator<(const Object &other) const { + return strncmp(data, other.data, Len) < 0; + } + void random_fill() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution dist('A', 'z'); + + for (uint32_t i = 0; i < Len - 1; i++) + data[i] = dist(mt); + data[Len - 1] = 0; + } +}; + +template K get_K() { return K(); } +template V get_V() { return V(); } + +template <> int get_K() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution dist(0, 1 << 30); + return dist(mt); +} + +template <> int get_V() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution dist(0, 1 << 30); + return dist(mt); +} + +#if TEST_OBJECT +template <> std::string get_K() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution dist('A', 'z'); + + std::string str = ""; + str.reserve(kKLen); + for (uint32_t i = 0; i < kKLen - 1; i++) + str += dist(mt); + str += '\0'; + return str; +} + +template <> std::string get_V() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution dist('A', 'z'); + + std::string str = ""; + str.reserve(kVLen); +#if TEST_LARGE == 1 + constexpr static int kFillStride = 2000; + for (uint32_t i = 0; i < kVLen - 1; i++) + if (i % kFillStride == 0) + str += dist(mt); + else + str += static_cast(i % ('z' - 'A')) + 'A'; +#else // !TEST_LARGE + for (uint32_t i = 0; i < kVLen - 1; i++) + str += dist(mt); +#endif + str += '\0'; + return str; +} + +template <> Object get_K() { + Object k; + k.random_fill(); + return k; +} + +template <> Object get_V() { + Object v; + v.random_fill(); + return v; +} +namespace std { +template <> struct hash> { + size_t operator()(const Object &k) const { + return std::hash()( + std::string_view(k.data, strlen(k.data))); + } +}; +} // namespace std +#endif // TEST_OBJECT + +struct Op { + enum OpCode { Set, Get, Remove } opcode; + K key; + V vals[kOSetSize]; + double scores[kOSetSize]; +}; +std::vector ops[kNumInsertThds]; + +void gen_workload() { + std::vector thds; + for (int tid = 0; tid < kNumInsertThds; tid++) { + std::random_device rd; + std::mt19937 mt(rd()); + std::uniform_real_distribution dist(0.0, 100.0); + for (int o = 0; o < kNumObjs; o++) { + K k = get_K(); + Op op{.opcode = Op::Set, .key = k}; + for (int j = 0; j < kOSetSize; j++) { + op.vals[j] = get_V(); + op.scores[j] = (double)op.vals[j]; + // op.scores[j] = dist(mt); + } + ops[tid].push_back(op); + } + } + for (auto &thd : thds) + thd.join(); + std::cout << "Finish generate workload." << std::endl; +} + +int main(int argc, char *argv[]) { + auto *rmanager = midas::ResourceManager::global_manager(); + rmanager->UpdateLimit(kCacheSize); + + auto *kvstore = new midas::SyncKV(); + std::mutex std_map_lock; + std::unordered_map std_map; + + std::atomic_int32_t nr_succ{0}; + std::atomic_int32_t nr_err{0}; + + gen_workload(); + + std::vector thds; + for (int tid = 0; tid < kNumInsertThds; tid++) { + thds.push_back(std::thread([&, tid = tid]() { + for (int i = 0; i < kNumObjs; i++) { + auto &k = ops[tid][i].key; + auto &vs = ops[tid][i].vals; + for (int j = 0; j < kOSetSize; j++) { + bool ret = kvstore->zadd( + &k, sizeof(k), &vs[j], sizeof(vs[j]), ops[tid][i].scores[j], + midas::SyncKV::UpdateType::NOT_EXIST); + if (!ret) + nr_err++; + else + nr_succ++; + } + } + })); + } + for (auto &thd : thds) + thd.join(); + thds.clear(); + + if (nr_err == 0) + std::cout << "Set test passed!" << std::endl; + else + std::cout << "Set test failed! " << nr_succ << " passed, " << nr_err + << " failed." << std::endl; + + std::atomic_int32_t nr_equal{0}; + std::atomic_int32_t nr_nequal{0}; + nr_succ = nr_err = 0; + for (int tid = 0; tid < kNumInsertThds; tid++) { + thds.push_back(std::thread([&, tid=tid]() { + for (int i = 0; i < kNumObjs; i++) { + const auto &op = ops[tid][i]; + const K &k = op.key; + size_t stored_vn = 0; + std::vector raw_vs; + std::vector vs; + if (!kvstore->zrange(&k, sizeof(k), 0, kOSetSize - 1, + std::back_inserter(raw_vs))) { + nr_err++; + } else { + nr_succ++; + } + for (auto [vp, size] : raw_vs) { + // std::cout << *(V *)(vp) << " " << size << std::endl; + vs.emplace_back(*(V *)vp); + free(vp); + } + bool sorted = true; + for (int j = 0; j < vs.size() - 1; j++) { + if (vs[j] > vs[j + 1]) { + sorted = false; + break; + } + } + if (sorted) + nr_equal++; + else + nr_nequal++; + } + })); + } + for (auto &thd : thds) + thd.join(); + thds.clear(); + + if (nr_nequal == 0) + std::cout << "Get test passed! " << nr_succ << " passed, " << nr_err + << " failed. " << nr_equal << "/" << nr_equal + nr_nequal + << " pairs are equal" << std::endl; + else + std::cout << "Get test failed! " << nr_succ << " passed, " << nr_err + << " failed. " << nr_equal << "/" << nr_equal + nr_nequal + << " pairs are equal" << std::endl + << "NOTE: a small amount of failures are expected if only " + "std_map is protected by lock, as keys can conflict in our " + "sync_hash_map and the result of races are uncertain." + << std::endl; + + nr_succ = nr_err = 0; + for (int tid = 0; tid < kNumRemoveThds; tid++) { + thds.push_back(std::thread([&, tid = tid]() { + for (auto &op : ops[tid]) { + const K &k = op.key; + bool ret = kvstore->remove(&k, sizeof(k)); + if (!ret) + nr_err++; + else + nr_succ++; + } + })); + } + for (auto &thd : thds) + thd.join(); + + if (nr_err == 0) + std::cout << "Remove test passed!" << std::endl; + else + std::cout << "Remove test failed! " << nr_succ << " passed, " << nr_err + << " failed." << std::endl; + return 0; +} \ No newline at end of file diff --git a/test/test_parallel_evacuator.cpp b/test/test_parallel_evacuator.cpp new file mode 100644 index 0000000..1c751cd --- /dev/null +++ b/test/test_parallel_evacuator.cpp @@ -0,0 +1,115 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "evacuator.hpp" +#include "log.hpp" +#include "object.hpp" +#include "resource_manager.hpp" + +constexpr int kNumGCThds = 3; +constexpr int kNumThds = 10; +constexpr int kNumObjs = 40960; + +constexpr int kObjSize = 111; + +struct Object { + char data[kObjSize]; + + void random_fill() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution dist('A', 'z'); + + for (uint32_t i = 0; i < kObjSize; i++) { + data[i] = dist(mt); + } + } + + bool equal(Object &other) { + return (strncmp(data, other.data, kObjSize) == 0); + } +}; + +int main(int argc, char *argv[]) { + auto *allocator = midas::LogAllocator::global_allocator(); + std::vector threads; + + std::atomic_int nr_errs(0); + std::vector> ptrs[kNumThds]; + std::vector objs[kNumThds]; + + for (int tid = 0; tid < kNumThds; tid++) { + threads.push_back(std::thread([&, tid = tid]() { + for (int i = 0; i < kNumObjs; i++) { + Object obj; + obj.random_fill(); + auto objptr = std::make_shared(); + + if (!allocator->alloc_to(sizeof(Object), objptr.get()) || + !objptr->copy_from(&obj, sizeof(Object))) { + nr_errs++; + continue; + } + ptrs[tid].push_back(objptr); + objs[tid].push_back(obj); + } + + for (int i = 0; i < ptrs[tid].size(); i++) { + bool ret = false; + auto ptr = ptrs[tid][i]; + Object stored_o; + if (!ptr->copy_to(&stored_o, sizeof(Object)) || + !objs[tid][i].equal(stored_o)) + nr_errs++; + } + })); + } + + for (auto &thd : threads) + thd.join(); + threads.clear(); + + midas::Evacuator evacuator( + nullptr, midas::ResourceManager::global_manager_shared_ptr(), + midas::LogAllocator::global_allocator_shared_ptr()); + evacuator.parallel_gc(kNumGCThds); + + bool kTestFree = false; + for (int tid = 0; tid < kNumThds; tid++) { + threads.push_back(std::thread([&, tid = tid]() { + auto nr_ptrs = ptrs[tid].size(); + for (int i = 0; i < nr_ptrs; i++) { + auto ptr = ptrs[tid][i]; + Object stored_o; + if (!ptr || !ptr->copy_to(&stored_o, sizeof(Object)) || + !objs[tid][i].equal(stored_o)) + nr_errs++; + } + + if (kTestFree) { + for (auto ptr : ptrs[tid]) { + if (!allocator->free(*ptr)) + nr_errs++; + } + } + })); + } + for (auto &thd : threads) + thd.join(); + threads.clear(); + + // Then evacuate all hot objs. + evacuator.parallel_gc(kNumGCThds); + evacuator.parallel_gc(kNumGCThds); + + if (nr_errs == 0) + std::cout << "Test passed!" << std::endl; + + return 0; +} \ No newline at end of file diff --git a/test/test_resource_manager.cpp b/test/test_resource_manager.cpp new file mode 100644 index 0000000..af551a6 --- /dev/null +++ b/test/test_resource_manager.cpp @@ -0,0 +1,30 @@ +#include +#include +#include + +#include "resource_manager.hpp" +#include "utils.hpp" + +constexpr int kNumThds = 10; +constexpr int kNumRegions = 10; + +int main(int argc, char *argv[]) { + std::vector thds; + for (int tid = 0; tid < kNumThds; tid++) { + thds.push_back(std::thread([]() { + midas::ResourceManager *rmanager = + midas::ResourceManager::global_manager(); + + for (int i = 0; i < kNumRegions; i++) { + rmanager->AllocRegion(); + } + for (int i = 0; i < kNumRegions; i++) { + rmanager->FreeRegions(); + } + })); + } + + for (auto &thd : thds) + thd.join(); + return 0; +} \ No newline at end of file diff --git a/test/test_sighandler.cpp b/test/test_sighandler.cpp new file mode 100644 index 0000000..f8b2b59 --- /dev/null +++ b/test/test_sighandler.cpp @@ -0,0 +1,50 @@ +#include +#include +#include + +#include "sig_handler.hpp" +#include "time.hpp" +#include "transient_ptr.hpp" +#include "utils.hpp" + +constexpr static int kNumThds = 20; +constexpr static int kNumRepeat = 1000'000; +constexpr static int kSize = 4096; + +void do_work() { + auto inv_addr = midas::kVolatileSttAddr + 0x100200300; + uint8_t buf[kSize]; + midas::TransientPtr tptr(inv_addr, kSize); + int nr_failed = 0; + auto stt = midas::chrono_utils::now(); + for (int i = 0; i < kNumRepeat; i++) { + nr_failed += !!tptr.copy_from(buf, kSize); // we expect false to be returned + nr_failed += !!tptr.copy_to(buf, kSize); + } + auto end = midas::chrono_utils::now(); + auto dur = midas::chrono_utils::duration(stt, end); + if (!nr_failed) + std::cout << "Test passed! Duration: " << dur + << "s, tput: " << kNumRepeat * 2 / dur << " ops" << std::endl; + else + std::cout << "Test failed! " << nr_failed << "/" << kNumRepeat * 2 + << " failed" << std::endl; +} + +int main() { + auto sig_handler = midas::SigHandler::global_sighandler(); + sig_handler->init(); + + std::vector thds; + for (int i = 0; i < kNumThds; i++) { + thds.emplace_back([] { + do_work(); + }); + } + + for (auto &thd : thds) { + thd.join(); + } + + return 0; +} \ No newline at end of file diff --git a/test/test_sighandler_proto.cpp b/test/test_sighandler_proto.cpp new file mode 100644 index 0000000..c43f7f2 --- /dev/null +++ b/test/test_sighandler_proto.cpp @@ -0,0 +1,62 @@ +#include + +#include "sig_handler.hpp" +#include "utils.hpp" + +class SoftPtr { +public: + SoftPtr(bool valid = true) { + if (valid) { + ptr_ = new int[10]; + } else { + // ptr_ = nullptr; + ptr_ = reinterpret_cast(kInvPtr); + } + // std::cout << "SoftPtr(): ptr_ = " << ptr_ << std::endl; + } + ~SoftPtr() { + // std::cout << "~SoftPtr(): ptr_ = " << ptr_ << std::endl; + if (reinterpret_cast(ptr_) != kInvPtr && ptr_) + delete[] ptr_; + } + + void reset() { ptr_ = nullptr; } + + bool copy_from(void *src, size_t size, int64_t offset = 0); + bool copy_to(void *dst, size_t size, int64_t offset = 0); + // bool copy_from(void *src, size_t size, int64_t offset = 0); + // bool copy_to(void *dst, size_t size, int64_t offset = 0); + +private: + constexpr static uint64_t kInvPtr = midas::kVolatileSttAddr + 0x100200300; + int *ptr_; +}; + +bool SoftPtr::copy_from(void *src, size_t size, int64_t offset) { + return midas::rmemcpy(ptr_, src, size); +} + +bool SoftPtr::copy_to(void *dst, size_t size, int64_t offset) { + return midas::rmemcpy(dst, ptr_, size); +} + +void do_work() { + SoftPtr sptr(false); + int a = 10; + bool ret = sptr.copy_from(&a, sizeof(int)); + std::cout << "copy_from returns " << ret << std::endl; + ret = sptr.copy_to(&a, sizeof(int)); + std::cout << "copy_to returns " << ret << std::endl; + if (!ret) + sptr.reset(); +} + +int main() { + auto sig_handler = midas::SigHandler::global_sighandler(); + sig_handler->init(); + + do_work(); + std::cout << "Test passed!" << std::endl; + + return 0; +} \ No newline at end of file diff --git a/test/test_skewed_hashmap.cpp b/test/test_skewed_hashmap.cpp new file mode 100644 index 0000000..a9d035a --- /dev/null +++ b/test/test_skewed_hashmap.cpp @@ -0,0 +1,255 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cache_manager.hpp" +#include "sync_hashmap.hpp" +#include "utils.hpp" +#include "zipf.hpp" + +constexpr static double kZipfSkew = 0.9; + +constexpr static int kNBuckets = (1 << 28); +constexpr static int kNumMutatorThds = 20; +constexpr static int kNumGCThds = 8; +constexpr static int64_t kNumTotalKVPairs = 64 * 1024 * 1024; +constexpr static int kNumOps = 8 * 1024 * 1024; +constexpr static int kKLen = 18; +constexpr static int kVLen = 61; + +constexpr static int kNumKVPairs = kNumTotalKVPairs / kNumMutatorThds; + +static std::unique_ptr mts[kNumMutatorThds]; + +template struct Object; +using K = Object; +using V = Object; +/** Define Object */ +template struct Object { + char data[Len]; + bool operator==(const Object &other) const { + return strncmp(data, other.data, Len) == 0; + } + bool operator!=(const Object &other) const { + return strncmp(data, other.data, Len) != 0; + } + bool operator<(const Object &other) const { + return strncmp(data, other.data, Len) < 0; + } + void random_fill(int tid) { + static std::uniform_int_distribution dist('A', 'z'); + + for (uint32_t i = 0; i < Len - 1; i++) + data[i] = dist(*mts[tid]); + data[Len - 1] = 0; + } +}; + +Object get_K(int tid) { + Object k; + k.random_fill(tid); + return k; +} +Object get_V(int tid) { + Object v; + v.random_fill(tid); + return v; +} +namespace std { +template <> struct hash> { + size_t operator()(const Object &k) const { + return std::hash()( + std::string_view(k.data, strlen(k.data))); + } +}; +} // namespace std +/** Define Object [End] */ + +/** Generate requests */ +struct Op { + enum OpCode { Set, Get, Remove } opcode; + K key; + V val; +}; + +class MidasTest { +private: + midas::SyncHashMap *hashmap; + + std::vector ks[kNumMutatorThds]; + std::vector vs[kNumMutatorThds]; + std::vector zipf_idxes[kNumMutatorThds]; + + struct alignas(64) Stats { + int64_t nr_succ; + int64_t nr_err; + int64_t nr_hit; + int64_t nr_miss; + + Stats() : nr_succ(0), nr_err(0), nr_hit(0), nr_miss(0) {} + void reset() { nr_succ = nr_err = nr_hit = nr_miss = 0; } + Stats &accum(const Stats &rhs) { + nr_succ += rhs.nr_succ; + nr_err += rhs.nr_err; + nr_hit += rhs.nr_hit; + nr_miss += rhs.nr_miss; + return *this; + } + }; + Stats stats; + + bool stop; + std::shared_ptr evac_thd; + + void reset() { + for (int i = 0; i < kNumMutatorThds; i++) { + std::random_device rd; + mts[i].reset(new std::mt19937(rd())); + } + + hashmap = new midas::SyncHashMap(); + stop = false; + + stats.reset(); + } + + void prepare() { + stats.reset(); + + Stats perthd_stats[kNumMutatorThds]; + std::vector thds; + for (int tid = 0; tid < kNumMutatorThds; tid++) { + thds.push_back(std::thread([&, tid = tid]() { + for (int o = 0; o < kNumKVPairs; o++) { + const K &k = get_K(tid); + const V &v = get_V(tid); + ks[tid].emplace_back(k); + vs[tid].emplace_back(v); + if (hashmap->set(k, v)) { + perthd_stats[tid].nr_succ++; + // auto _ = hashmap->get(k); + } else + perthd_stats[tid].nr_err++; + } + })); + } + for (auto &thd : thds) + thd.join(); + std::cout << "Finish preparation." << std::endl; + for (int i = 0; i < kNumMutatorThds; i++) { + stats.accum(perthd_stats[i]); + } + if (stats.nr_err > 0) + std::cout << "Set succ " << stats.nr_succ << ", fail " << stats.nr_err + << std::endl; + // std::this_thread::sleep_for(std::chrono::seconds(5)); + } + + void gen_load() { + std::vector thds; + for (int tid = 0; tid < kNumMutatorThds; tid++) { + thds.push_back(std::thread([&, tid = tid]() { + midas::zipf_table_distribution<> dist(kNumKVPairs, kZipfSkew); + // std::uniform_int_distribution<> dist(0, kNumKVPairs); + zipf_idxes[tid].clear(); + for (int o = 0; o < kNumOps; o++) { + auto idx = dist(*mts[tid]); + zipf_idxes[tid].push_back(idx); + } + })); + } + for (auto &thd : thds) + thd.join(); + std::cout << "Finish load generation." << std::endl; + } + +public: + void init() { + reset(); + prepare(); + } + + int run() { + stats.reset(); + gen_load(); + + auto stt = std::chrono::steady_clock::now(); + + Stats perthd_stats[kNumMutatorThds]; + std::vector thds; + for (int tid = 0; tid < kNumMutatorThds; tid++) { + thds.push_back(std::thread([&, tid = tid] { + for (auto idx : zipf_idxes[tid]) { + bool ret = false; + + auto &k = ks[tid][idx]; + auto &v = vs[tid][idx]; + auto optv = hashmap->get(k); + if (optv) { + ret = (v == *optv); + perthd_stats[tid].nr_hit++; + } else { + ret = hashmap->set(k, v); + perthd_stats[tid].nr_miss++; + } + + ret ? perthd_stats[tid].nr_succ++ : perthd_stats[tid].nr_err++; + } + })); + } + for (auto &thd : thds) + thd.join(); + thds.clear(); + auto end = std::chrono::steady_clock::now(); + + std::cout << "Finish executing workload in " + << std::chrono::duration(end - stt).count() << "s" + << std::endl; + + for (int i = 0; i < kNumMutatorThds; i++) { + stats.accum(perthd_stats[i]); + } + std::cout << "cache hit ratio: " + << static_cast(stats.nr_hit) / + (stats.nr_hit + stats.nr_miss) + << std::endl; + + if (stats.nr_err == 0) + std::cout << "Test passed!" << std::endl; + else + std::cout << "Get test failed! " << stats.nr_succ << " passed, " + << stats.nr_err << " failed." << std::endl; + + std::cout << "Cooling down..." << std::endl; + std::this_thread::sleep_for(std::chrono::milliseconds(5000)); + + return 0; + } +}; + +void signalHandler(int signum) { + // Let the process exit normally so that daemon_ can be naturally destroyed. + exit(signum); +} + +int main(int argc, char *argv[]) { + signal(SIGINT, signalHandler); + + midas::CachePool::global_cache_pool()->update_limit(kNumTotalKVPairs * + (kKLen + kVLen)); + + MidasTest test; + test.init(); + for (int i = 0; i < 100; i++) + test.run(); + return 0; +} diff --git a/test/test_slab.cpp b/test/test_slab.cpp new file mode 100644 index 0000000..ba2eb05 --- /dev/null +++ b/test/test_slab.cpp @@ -0,0 +1,47 @@ +#include +#include +#include +#include + +#include "slab.hpp" + +#ifdef ENABLE_SLAB + +constexpr int kNumThds = 10; +constexpr int kNumObjs = 1024; + +int main(int argc, char *argv[]) { + auto *allocator = midas::SlabAllocator::global_allocator(); + + std::random_device rd; + std::mt19937 rand(rd()); + std::uniform_int_distribution<> dist(1, 256); + + std::vector thds; + for (int tid = 0; tid < kNumThds; tid++) { + thds.push_back(std::thread([&]() { + std::vector ptrs; + for (int i = 0; i < kNumObjs; i++) { + auto *ptr = allocator->alloc(dist(rand)); + std::cout << "Alloc obj at " << ptr << std::endl; + ptrs.push_back(ptr); + } + for (auto ptr : ptrs) { + std::cout << "Free obj at " << ptr << std::endl; + allocator->free(ptr); + } + })); + } + + for (auto &thd : thds) + thd.join(); + return 0; +} + +#else // !ENABLE_SLAB + +int main() { + return 0; +} + +#endif // ENABLE_SLAB \ No newline at end of file diff --git a/test/test_softptr_read_cost.cpp b/test/test_softptr_read_cost.cpp new file mode 100644 index 0000000..018234e --- /dev/null +++ b/test/test_softptr_read_cost.cpp @@ -0,0 +1,234 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cache_manager.hpp" +#include "object.hpp" +#include "time.hpp" + +#define ACCESS_ONCE(x) \ + (*static_cast::type volatile *>(&(x))) + +using data_t = uint64_t; + +constexpr static int kMeasureTimes = 1'000'000; // 1M times +constexpr static uint64_t kRawMemAccessCycles = 170; +constexpr static uint64_t kCachePoolSize = 100ull * 1024 * 1024 * 1024; + +constexpr static int kNumSmallObjs = 1'000'000; // 1M objs +constexpr static int kSmallObjSize = 64; + +constexpr static int kNumLargeObjs = 10'000; // 10K objs +constexpr static int kLargeObjSize = 4 * 1024 * 1024; // 4MB + +struct SmallObject { + char data[kSmallObjSize]; + + SmallObject() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution dist('A', 'z'); + + for (uint32_t i = 0; i < kSmallObjSize; i++) { + data[i] = dist(mt); + } + } +}; + +struct LargeObject { + char data[kLargeObjSize]; + + LargeObject() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution dist('A', 'z'); + + constexpr static int kFillStride = 100; + for (uint32_t i = 0; i < kLargeObjSize / kFillStride; i++) { + data[i * kFillStride] = dist(mt); + } + } +}; + +void print_lats(std::vector &durs) { + std::sort(durs.begin(), durs.end()); + uint64_t avg = std::reduce(durs.begin(), durs.end(), 0.0) / durs.size(); + auto median = durs[durs.size() / 2]; + auto p90 = durs[durs.size() * 9 / 10]; + auto p99 = durs[durs.size() * 99 / 100]; + + printf("avg: %lu, median: %lu, p90: %lu, p99: %lu\n", avg, median, p90, p99); +} + +void unique_ptr_read_small_cost() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution dist(0, kNumSmallObjs - 1); + + std::vector> objs; + for (int i = 0; i < kNumSmallObjs; i++) { + objs.push_back(std::make_unique()); + } + + uint64_t stt, end; + std::vector durs; + for (int i = 0; i < kMeasureTimes; i++) { + auto idx = dist(mt); + + stt = midas::Time::get_cycles_stt(); + { + const data_t *ptr = reinterpret_cast(objs[idx].get()->data); + ACCESS_ONCE(*ptr); + } + end = midas::Time::get_cycles_end(); + auto dur_cycles = end - stt; + durs.push_back(dur_cycles); + } + printf("Access unique_ptr:\n"); + print_lats(durs); +} + +void unique_ptr_read_large_cost() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution idx_dist(0, kNumLargeObjs - 1); + static std::uniform_int_distribution off_dist( + 0, kLargeObjSize / sizeof(data_t) - 1); + + std::vector> objs; + for (int i = 0; i < kNumLargeObjs; i++) { + objs.push_back(std::make_unique()); + } + + uint64_t stt, end; + std::vector durs; + for (int i = 0; i < kMeasureTimes; i++) { + auto idx = idx_dist(mt); + auto off = off_dist(mt); + + stt = midas::Time::get_cycles_stt(); + { + const data_t *ptr = reinterpret_cast(objs[idx].get()->data); + ACCESS_ONCE(ptr[off]); + } + end = midas::Time::get_cycles_end(); + auto dur_cycles = end - stt; + durs.push_back(dur_cycles); + } + printf("Access unique_ptr:\n"); + print_lats(durs); +} + +void softptr_read_small_cost() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution dist(0, kNumSmallObjs - 1); + + auto cmanager = midas::CacheManager::global_cache_manager(); + cmanager->create_pool("small"); + auto pool = cmanager->get_pool("small"); + pool->update_limit(kCachePoolSize); + auto allocator = pool->get_allocator(); + + std::vector objs; + for (int i = 0; i < kNumSmallObjs; i++) { + objs.emplace_back(midas::ObjectPtr()); + } + for (int i = 0; i < kNumSmallObjs; i++) { + auto succ = allocator->alloc_to(kSmallObjSize, &objs[i]); + assert(succ); + } + for (int i = 0; i < kNumSmallObjs; i++) { + SmallObject obj; + auto succ = objs[i].copy_from(obj.data, kSmallObjSize); + assert(succ); + } + + uint64_t stt, end; + std::vector durs; + for (int i = 0; i < kMeasureTimes; i++) { + auto idx = dist(mt); + + stt = midas::Time::get_cycles_stt(); + { + data_t dst; + objs[idx].copy_to(&dst, sizeof(dst)); + } + end = midas::Time::get_cycles_end(); + auto dur_cycles = end - stt; + if (dur_cycles > kRawMemAccessCycles) + durs.push_back(dur_cycles); + } + printf("Access soft_ptr:\n"); + print_lats(durs); +} + +void softptr_read_large_cost() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution idx_dist(0, kNumLargeObjs - 1); + static std::uniform_int_distribution off_dist( + 0, kLargeObjSize / sizeof(data_t) - 1); + + auto cmanager = midas::CacheManager::global_cache_manager(); + cmanager->create_pool("large"); + auto pool = cmanager->get_pool("large"); + pool->update_limit(kCachePoolSize); + auto allocator = pool->get_allocator(); + + std::vector objs; + for (int i = 0; i < kNumLargeObjs; i++) { + objs.emplace_back(midas::ObjectPtr()); + } + for (int i = 0; i < kNumLargeObjs; i++) { + auto succ = allocator->alloc_to(kLargeObjSize, &objs[i]); + assert(succ); + } + for (int i = 0; i < kNumLargeObjs; i++) { + LargeObject obj; + auto succ = objs[i].copy_from(obj.data, kLargeObjSize); + assert(succ); + } + + uint64_t stt, end; + std::vector durs; + for (int i = 0; i < kMeasureTimes; i++) { + auto idx = idx_dist(mt); + auto off = off_dist(mt); + + volatile data_t dst; + stt = midas::Time::get_cycles_stt(); + { + data_t dst_; + objs[idx].copy_to(&dst_, sizeof(dst_), off); + dst = dst_; + } + end = midas::Time::get_cycles_end(); + auto dur_cycles = end - stt; + if (dur_cycles > kRawMemAccessCycles) + durs.push_back(dur_cycles); + } + printf("Access soft_ptr:\n"); + print_lats(durs); +} + +int main(int argc, char *argv[]) { + softptr_read_small_cost(); + std::this_thread::sleep_for(std::chrono::milliseconds(1000)); + unique_ptr_read_small_cost(); + std::this_thread::sleep_for(std::chrono::milliseconds(1000)); + + unique_ptr_read_large_cost(); + std::this_thread::sleep_for(std::chrono::milliseconds(1000)); + softptr_read_large_cost(); + + return 0; +} \ No newline at end of file diff --git a/test/test_softptr_write_cost.cpp b/test/test_softptr_write_cost.cpp new file mode 100644 index 0000000..971c5cc --- /dev/null +++ b/test/test_softptr_write_cost.cpp @@ -0,0 +1,231 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cache_manager.hpp" +#include "object.hpp" +#include "time.hpp" + +#define ACCESS_ONCE(x) \ + (*static_cast::type volatile *>(&(x))) + +using data_t = uint64_t; +constexpr static data_t kWriteContent = 0x1f1f1f1f'1f1f1f1full; + +constexpr static int kMeasureTimes = 1'000'000; // 1M times +constexpr static uint64_t kRawMemAccessCycles = 170; +constexpr static uint64_t kCachePoolSize = 100ull * 1024 * 1024 * 1024; + +constexpr static int kNumSmallObjs = 1'000'000; // 1M objs +constexpr static int kSmallObjSize = 64; + +constexpr static int kNumLargeObjs = 10'000; // 10K objs +constexpr static int kLargeObjSize = 4 * 1024 * 1024; // 4MB + +struct SmallObject { + char data[kSmallObjSize]; + + SmallObject() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution dist('A', 'z'); + + for (uint32_t i = 0; i < kSmallObjSize; i++) { + data[i] = dist(mt); + } + } +}; + +struct LargeObject { + char data[kLargeObjSize]; + + LargeObject() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution dist('A', 'z'); + + constexpr static int kFillStride = 100; + for (uint32_t i = 0; i < kLargeObjSize / kFillStride; i++) { + data[i * kFillStride] = dist(mt); + } + } +}; + +void print_lats(std::vector &durs) { + std::sort(durs.begin(), durs.end()); + uint64_t avg = std::reduce(durs.begin(), durs.end(), 0.0) / durs.size(); + auto median = durs[durs.size() / 2]; + auto p90 = durs[durs.size() * 9 / 10]; + auto p99 = durs[durs.size() * 99 / 100]; + + printf("avg: %lu, median: %lu, p90: %lu, p99: %lu\n", avg, median, p90, p99); +} + +void unique_ptr_write_small_cost() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution dist(0, kNumSmallObjs - 1); + + std::vector> objs; + for (int i = 0; i < kNumSmallObjs; i++) { + objs.push_back(std::make_unique()); + } + + uint64_t stt, end; + std::vector durs; + for (int i = 0; i < kMeasureTimes; i++) { + auto idx = dist(mt); + + stt = midas::Time::get_cycles_stt(); + { + data_t *ptr = reinterpret_cast(objs[idx].get()->data); + ACCESS_ONCE(*ptr) = kWriteContent; + } + end = midas::Time::get_cycles_end(); + auto dur_cycles = end - stt; + durs.push_back(dur_cycles); + } + printf("Access unique_ptr:\n"); + print_lats(durs); +} + +void unique_ptr_write_large_cost() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution idx_dist(0, kNumLargeObjs - 1); + static std::uniform_int_distribution off_dist( + 0, kLargeObjSize / sizeof(data_t) - 1); + + std::vector> objs; + for (int i = 0; i < kNumLargeObjs; i++) { + objs.push_back(std::make_unique()); + } + + uint64_t stt, end; + std::vector durs; + for (int i = 0; i < kMeasureTimes; i++) { + auto idx = idx_dist(mt); + auto off = off_dist(mt); + + stt = midas::Time::get_cycles_stt(); + { + data_t *ptr = reinterpret_cast(objs[idx].get()->data); + ACCESS_ONCE(ptr[off]) = kWriteContent; + } + end = midas::Time::get_cycles_end(); + auto dur_cycles = end - stt; + durs.push_back(dur_cycles); + } + printf("Access unique_ptr:\n"); + print_lats(durs); +} + +void softptr_write_small_cost() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution dist(0, kNumSmallObjs - 1); + + auto cmanager = midas::CacheManager::global_cache_manager(); + cmanager->create_pool("small"); + auto pool = cmanager->get_pool("small"); + pool->update_limit(kCachePoolSize); + auto allocator = pool->get_allocator(); + + std::vector objs; + for (int i = 0; i < kNumSmallObjs; i++) { + objs.emplace_back(midas::ObjectPtr()); + } + for (int i = 0; i < kNumSmallObjs; i++) { + auto succ = allocator->alloc_to(kSmallObjSize, &objs[i]); + assert(succ); + } + for (int i = 0; i < kNumSmallObjs; i++) { + SmallObject obj; + auto succ = objs[i].copy_from(obj.data, kSmallObjSize); + assert(succ); + } + + uint64_t stt, end; + std::vector durs; + for (int i = 0; i < kMeasureTimes; i++) { + auto idx = dist(mt); + + stt = midas::Time::get_cycles_stt(); + { + objs[idx].copy_from(&kWriteContent, sizeof(data_t)); + } + end = midas::Time::get_cycles_end(); + auto dur_cycles = end - stt; + if (dur_cycles > kRawMemAccessCycles) + durs.push_back(dur_cycles); + } + printf("Access soft_ptr:\n"); + print_lats(durs); +} + +void softptr_write_large_cost() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution idx_dist(0, kNumLargeObjs - 1); + static std::uniform_int_distribution off_dist( + 0, kLargeObjSize / sizeof(data_t) - 1); + + auto cmanager = midas::CacheManager::global_cache_manager(); + cmanager->create_pool("large"); + auto pool = cmanager->get_pool("large"); + pool->update_limit(kCachePoolSize); + auto allocator = pool->get_allocator(); + + std::vector objs; + for (int i = 0; i < kNumLargeObjs; i++) { + objs.emplace_back(midas::ObjectPtr()); + } + for (int i = 0; i < kNumLargeObjs; i++) { + auto succ = allocator->alloc_to(kLargeObjSize, &objs[i]); + assert(succ); + } + for (int i = 0; i < kNumLargeObjs; i++) { + LargeObject obj; + auto succ = objs[i].copy_from(obj.data, kLargeObjSize); + assert(succ); + } + + uint64_t stt, end; + std::vector durs; + for (int i = 0; i < kMeasureTimes; i++) { + auto idx = idx_dist(mt); + auto off = off_dist(mt); + + stt = midas::Time::get_cycles_stt(); + { + objs[idx].copy_from(&kWriteContent, sizeof(data_t), off); + } + end = midas::Time::get_cycles_end(); + auto dur_cycles = end - stt; + if (dur_cycles > kRawMemAccessCycles) + durs.push_back(dur_cycles); + } + printf("Access soft_ptr:\n"); + print_lats(durs); +} + +int main(int argc, char *argv[]) { + unique_ptr_write_small_cost(); + std::this_thread::sleep_for(std::chrono::milliseconds(1000)); + softptr_write_small_cost(); + std::this_thread::sleep_for(std::chrono::milliseconds(1000)); + + unique_ptr_write_large_cost(); + std::this_thread::sleep_for(std::chrono::milliseconds(1000)); + softptr_write_large_cost(); + + return 0; +} \ No newline at end of file diff --git a/test/test_sync_hashmap.cpp b/test/test_sync_hashmap.cpp new file mode 100644 index 0000000..41af8f5 --- /dev/null +++ b/test/test_sync_hashmap.cpp @@ -0,0 +1,277 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "resource_manager.hpp" +#include "sync_hashmap.hpp" + +#define TEST_OBJECT 1 +#define TEST_LARGE 0 + +constexpr static int kNBuckets = (1 << 20); +constexpr static int kNumInsertThds = 10; +constexpr static int kNumRemoveThds = 10; + +#if TEST_LARGE +constexpr static int kNumObjs = 10240; +constexpr static int kKLen = 32; +constexpr static int kVLen = 8192; +#else // !TEST_LARGE +constexpr static int kNumObjs = 102400; +constexpr static int kKLen = 61; +constexpr static int kVLen = 10; +#endif // TEST_LARGE + +template struct Object; + +#if TEST_OBJECT +using K = Object; +using V = Object; +#else // !TEST_OBJECT +using K = int; +using V = int; +#endif // TEST_OBJECT + +/** YIFAN: string is not supported! Its reference will be lost after data copy + */ +// using K = std::string; +// using V = std::string; + +template struct Object { + char data[Len]; + bool operator==(const Object &other) const { + return strncmp(data, other.data, Len) == 0; + } + bool operator!=(const Object &other) const { + return strncmp(data, other.data, Len) != 0; + } + bool operator<(const Object &other) const { + return strncmp(data, other.data, Len) < 0; + } + void random_fill() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution dist('A', 'z'); + + for (uint32_t i = 0; i < Len - 1; i++) + data[i] = dist(mt); + data[Len - 1] = 0; + } +}; + +template K get_K() { return K(); } +template V get_V() { return V(); } + +template <> std::string get_K() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution dist('A', 'z'); + + std::string str = ""; + str.reserve(kKLen); + for (uint32_t i = 0; i < kKLen - 1; i++) + str += dist(mt); + str += '\0'; + return str; +} + +template <> std::string get_V() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution dist('A', 'z'); + + std::string str = ""; + str.reserve(kVLen); +#if TEST_LARGE == 1 + constexpr static int kFillStride = 2000; + for (uint32_t i = 0; i < kVLen - 1; i++) + if (i % kFillStride == 0) + str += dist(mt); + else + str += static_cast(i % ('z' - 'A')) + 'A'; +#else // !TEST_LARGE + for (uint32_t i = 0; i < kVLen - 1; i++) + str += dist(mt); +#endif + str += '\0'; + return str; +} + +template <> int get_K() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution dist(0, 1 << 30); + return dist(mt); +} + +template <> int get_V() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution dist(0, 1 << 30); + return dist(mt); +} + +template <> Object get_K() { + Object k; + k.random_fill(); + return k; +} + +template <> Object get_V() { + Object v; + v.random_fill(); + return v; +} +namespace std { +template <> struct hash> { + size_t operator()(const Object &k) const { + return std::hash()( + std::string_view(k.data, strlen(k.data))); + } +}; +} // namespace std + +struct Op { + enum OpCode { Set, Get, Remove } opcode; + K key; + V val; +}; +std::vector ks[kNumInsertThds]; +std::vector vs[kNumInsertThds]; +std::vector ops[kNumInsertThds]; + +void gen_workload() { + std::vector thds; + for (int tid = 0; tid < kNumInsertThds; tid++) { + for (int o = 0; o < kNumObjs; o++) { + K k = get_K(); + V v = get_V(); + ks[tid].push_back(k); + vs[tid].push_back(v); + Op op{.opcode = Op::Set, .key = k, .val = v}; + ops[tid].push_back(op); + } + } + for (auto &thd : thds) + thd.join(); + std::cout << "Finish generate workload." << std::endl; +} + +int main(int argc, char *argv[]) { + auto *rmanager = midas::ResourceManager::global_manager(); + std::vector thds; + + auto *hashmap = new midas::SyncHashMap(); + std::mutex std_map_lock; + std::unordered_map std_map; + + std::atomic_int32_t nr_succ = 0; + std::atomic_int32_t nr_err = 0; + + gen_workload(); + + for (int tid = 0; tid < kNumInsertThds; tid++) { + thds.push_back(std::thread([&, tid = tid]() { + for (int i = 0; i < kNumObjs; i++) { + auto &k = ks[tid][i]; + auto &v = vs[tid][i]; + std_map_lock.lock(); + std_map[k] = v; + std_map_lock.unlock(); + bool ret = hashmap->set(k, v); + if (!ret) + nr_err++; + else + nr_succ++; + } + })); + } + for (auto &thd : thds) + thd.join(); + thds.clear(); + + if (nr_err == 0) + std::cout << "Set test passed!" << std::endl; + else + std::cout << "Set test failed! " << nr_succ << " passed, " << nr_err + << " failed." << std::endl; + + std::atomic_int32_t nr_equal = 0; + std::atomic_int32_t nr_nequal = 0; + nr_succ = nr_err = 0; + for (int tid = 0; tid < kNumInsertThds; tid++) { + thds.push_back(std::thread([&]() { + for (auto &pair : std_map) { + const K &k = pair.first; + V &v = pair.second; + auto v2 = hashmap->get(k); + + if (!v2) { + nr_err++; + } else { + nr_succ++; + if (v == *v2) { + nr_equal++; + } else { + nr_nequal++; + } + } + } + })); + } + for (auto &thd : thds) + thd.join(); + thds.clear(); + + if (nr_nequal == 0) + std::cout << "Get test passed! " << nr_succ << " passed, " << nr_err + << " failed. " << nr_equal << "/" << nr_equal + nr_nequal + << " pairs are equal" << std::endl; + else + std::cout << "Get test failed! " << nr_succ << " passed, " << nr_err + << " failed. " << nr_equal << "/" << nr_equal + nr_nequal + << " pairs are equal" << std::endl + << "NOTE: a small amount of failures are expected if only " + "std_map is protected by lock, as keys can conflict in our " + "sync_hash_map and the result of races are uncertain." + << std::endl; + + std::vector> pairs; + for (auto pair : std_map) { + pairs.push_back(pair); + } + + nr_succ = nr_err = 0; + for (int tid = 0; tid < kNumRemoveThds; tid++) { + thds.push_back(std::thread([&, tid = tid]() { + int chunk = (pairs.size() + kNumRemoveThds - 1) / kNumRemoveThds; + int stt = chunk * tid; + int end = std::min(stt + chunk, pairs.size()); + + for (int i = stt; i < end; i++) { + auto pair = pairs[i]; + const K &k = pair.first; + bool ret = hashmap->remove(k); + if (!ret) + nr_err++; + else + nr_succ++; + } + })); + } + for (auto &thd : thds) + thd.join(); + + if (nr_err == 0) + std::cout << "Remove test passed!" << std::endl; + else + std::cout << "Remove test failed! " << nr_succ << " passed, " << nr_err + << " failed." << std::endl; + return 0; +} \ No newline at end of file diff --git a/test/test_sync_kv.cpp b/test/test_sync_kv.cpp new file mode 100644 index 0000000..24f0b4f --- /dev/null +++ b/test/test_sync_kv.cpp @@ -0,0 +1,282 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "resource_manager.hpp" +#include "sync_kv.hpp" + +#define TEST_OBJECT 1 +#define TEST_LARGE 1 + +constexpr static size_t kCacheSize = 1024ull * 1024 * 200; +constexpr static int kNBuckets = (1 << 20); +constexpr static int kNumInsertThds = 10; +constexpr static int kNumRemoveThds = 10; + +#if TEST_LARGE +constexpr static int kNumObjs = 10240; +constexpr static int kKLen = 32; +constexpr static int kVLen = 8192; +#else // !TEST_LARGE +constexpr static int kNumObjs = 102400; +constexpr static int kKLen = 61; +constexpr static int kVLen = 10; +#endif // TEST_LARGE + +template struct Object; + +#if TEST_OBJECT +using K = Object; +using V = Object; +#else // !TEST_OBJECT +using K = int; +using V = int; +#endif // TEST_OBJECT + +/** YIFAN: string is not supported! Its reference will be lost after data copy + */ +// using K = std::string; +// using V = std::string; + +template struct Object { + char data[Len]; + bool operator==(const Object &other) const { + return strncmp(data, other.data, Len) == 0; + } + bool operator!=(const Object &other) const { + return strncmp(data, other.data, Len) != 0; + } + bool operator<(const Object &other) const { + return strncmp(data, other.data, Len) < 0; + } + void random_fill() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution dist('A', 'z'); + + for (uint32_t i = 0; i < Len - 1; i++) + data[i] = dist(mt); + data[Len - 1] = 0; + } +}; + +template K get_K() { return K(); } +template V get_V() { return V(); } + +template <> std::string get_K() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution dist('A', 'z'); + + std::string str = ""; + str.reserve(kKLen); + for (uint32_t i = 0; i < kKLen - 1; i++) + str += dist(mt); + str += '\0'; + return str; +} + +template <> std::string get_V() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution dist('A', 'z'); + + std::string str = ""; + str.reserve(kVLen); +#if TEST_LARGE == 1 + constexpr static int kFillStride = 2000; + for (uint32_t i = 0; i < kVLen - 1; i++) + if (i % kFillStride == 0) + str += dist(mt); + else + str += static_cast(i % ('z' - 'A')) + 'A'; +#else // !TEST_LARGE + for (uint32_t i = 0; i < kVLen - 1; i++) + str += dist(mt); +#endif + str += '\0'; + return str; +} + +template <> int get_K() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution dist(0, 1 << 30); + return dist(mt); +} + +template <> int get_V() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution dist(0, 1 << 30); + return dist(mt); +} + +template <> Object get_K() { + Object k; + k.random_fill(); + return k; +} + +template <> Object get_V() { + Object v; + v.random_fill(); + return v; +} +namespace std { +template <> struct hash> { + size_t operator()(const Object &k) const { + return std::hash()( + std::string_view(k.data, strlen(k.data))); + } +}; +} // namespace std + +struct Op { + enum OpCode { Set, Get, Remove } opcode; + K key; + V val; +}; +std::vector ks[kNumInsertThds]; +std::vector vs[kNumInsertThds]; +std::vector ops[kNumInsertThds]; + +void gen_workload() { + std::vector thds; + for (int tid = 0; tid < kNumInsertThds; tid++) { + for (int o = 0; o < kNumObjs; o++) { + K k = get_K(); + V v = get_V(); + ks[tid].push_back(k); + vs[tid].push_back(v); + Op op{.opcode = Op::Set, .key = k, .val = v}; + ops[tid].push_back(op); + } + } + for (auto &thd : thds) + thd.join(); + std::cout << "Finish generate workload." << std::endl; +} + +int main(int argc, char *argv[]) { + auto *rmanager = midas::ResourceManager::global_manager(); + rmanager->UpdateLimit(kCacheSize); + + auto *kvstore = new midas::SyncKV(); + std::mutex std_map_lock; + std::unordered_map std_map; + + std::atomic_int32_t nr_succ = 0; + std::atomic_int32_t nr_err = 0; + + gen_workload(); + + std::vector thds; + for (int tid = 0; tid < kNumInsertThds; tid++) { + thds.push_back(std::thread([&, tid = tid]() { + for (int i = 0; i < kNumObjs; i++) { + auto &k = ks[tid][i]; + auto &v = vs[tid][i]; + std_map_lock.lock(); + std_map[k] = v; + std_map_lock.unlock(); + bool ret = kvstore->set(&k, sizeof(k), &v, sizeof(v)); + if (!ret) + nr_err++; + else + nr_succ++; + } + })); + } + for (auto &thd : thds) + thd.join(); + thds.clear(); + + if (nr_err == 0) + std::cout << "Set test passed!" << std::endl; + else + std::cout << "Set test failed! " << nr_succ << " passed, " << nr_err + << " failed." << std::endl; + + std::atomic_int32_t nr_equal = 0; + std::atomic_int32_t nr_nequal = 0; + nr_succ = nr_err = 0; + for (int tid = 0; tid < kNumInsertThds; tid++) { + thds.push_back(std::thread([&]() { + for (auto &pair : std_map) { + const K &k = pair.first; + V &v = pair.second; + size_t stored_vn = 0; + V *v2 = reinterpret_cast(kvstore->get(&k, sizeof(k), &stored_vn)); + if (!v2) { + nr_err++; + } else { + nr_succ++; + assert(stored_vn == sizeof(V)); + if (v == *v2) { + nr_equal++; + } else { + nr_nequal++; + } + } + if (v2) + free(v2); + } + })); + } + for (auto &thd : thds) + thd.join(); + thds.clear(); + + if (nr_nequal == 0) + std::cout << "Get test passed! " << nr_succ << " passed, " << nr_err + << " failed. " << nr_equal << "/" << nr_equal + nr_nequal + << " pairs are equal" << std::endl; + else + std::cout << "Get test failed! " << nr_succ << " passed, " << nr_err + << " failed. " << nr_equal << "/" << nr_equal + nr_nequal + << " pairs are equal" << std::endl + << "NOTE: a small amount of failures are expected if only " + "std_map is protected by lock, as keys can conflict in our " + "sync_hash_map and the result of races are uncertain." + << std::endl; + + std::vector> pairs; + for (auto pair : std_map) { + pairs.push_back(pair); + } + + nr_succ = nr_err = 0; + for (int tid = 0; tid < kNumRemoveThds; tid++) { + thds.push_back(std::thread([&, tid = tid]() { + int chunk = (pairs.size() + kNumRemoveThds - 1) / kNumRemoveThds; + int stt = chunk * tid; + int end = std::min(stt + chunk, pairs.size()); + + for (int i = stt; i < end; i++) { + auto pair = pairs[i]; + const K &k = pair.first; + bool ret = kvstore->remove(&k, sizeof(k)); + if (!ret) + nr_err++; + else + nr_succ++; + } + })); + } + for (auto &thd : thds) + thd.join(); + + if (nr_err == 0) + std::cout << "Remove test passed!" << std::endl; + else + std::cout << "Remove test failed! " << nr_succ << " passed, " << nr_err + << " failed." << std::endl; + return 0; +} \ No newline at end of file diff --git a/test/test_sync_list.cpp b/test/test_sync_list.cpp new file mode 100644 index 0000000..77aedab --- /dev/null +++ b/test/test_sync_list.cpp @@ -0,0 +1,193 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sync_list.hpp" + +#define TEST_OBJECT 1 +#define TEST_LARGE 0 + +constexpr static int kNumThds = 10; + +#if TEST_LARGE +constexpr static int kNumObjs = 10240; +constexpr static int kVLen = 8192; +#else // !TEST_LARGE +constexpr static int kNumObjs = 102400; +constexpr static int kVLen = 128; +#endif // TEST_LARGE + +template struct Object; + +#if TEST_OBJECT +using V = Object; +#else // !TEST_OBJECT +using V = int; +#endif // TEST_OBJECT + +template struct Object { + char data[Len]; + bool operator==(const Object &other) const { + return strncmp(data, other.data, Len) == 0; + } + bool operator!=(const Object &other) const { + return strncmp(data, other.data, Len) != 0; + } + bool operator<(const Object &other) const { + return strncmp(data, other.data, Len) < 0; + } + void random_fill() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution dist('A', 'z'); + +#if TEST_LARGE + const int STRIDE = 100; + for (uint32_t i = 0; i < (Len - 1) / STRIDE; i++) + data[i * STRIDE] = dist(mt); +#else + for (uint32_t i = 0; i < Len - 1; i++) + data[i] = dist(mt); +#endif + data[Len - 1] = 0; + } +}; + +template V get_V() { return V(); } + +template <> std::string get_V() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution dist('A', 'z'); + + std::string str = ""; + str.reserve(kVLen); +#if TEST_LARGE == 1 + constexpr static int kFillStride = 2000; + for (uint32_t i = 0; i < kVLen - 1; i++) + if (i % kFillStride == 0) + str += dist(mt); + else + str += static_cast(i % ('z' - 'A')) + 'A'; +#else // !TEST_LARGE + for (uint32_t i = 0; i < kVLen - 1; i++) + str += dist(mt); +#endif + str += '\0'; + return str; +} + +template <> int get_V() { + static std::random_device rd; + static std::mt19937 mt(rd()); + static std::uniform_int_distribution dist(0, 1 << 30); + return dist(mt); +} + +template <> Object get_V() { + Object v; + v.random_fill(); + return v; +} + +namespace std { +template <> struct hash> { + size_t operator()(const Object &k) const { + return std::hash()( + std::string_view(k.data, strlen(k.data))); + } +}; +} // namespace std + +std::vector vs[kNumThds]; +std::unordered_map vmap; + +void gen_workload() { + for (int tid = 0; tid < kNumThds; tid++) { + for (int o = 0; o < kNumObjs; o++) { + V v = get_V(); + vs[tid].emplace_back(v); + vmap[v] = true; + } + } + std::cout << "Finish generate workload." << std::endl; +} + +int main(int argc, char *argv[]) { + std::vector thds; + auto *list = new midas::SyncList(); + + std::atomic_int32_t nr_succ = 0; + std::atomic_int32_t nr_fail = 0; + std::atomic_int32_t nr_err = 0; + + gen_workload(); + + for (int tid = 0; tid < kNumThds; tid++) { + thds.push_back(std::thread([&, tid = tid] { + for (auto &v : vs[tid]) { + if (list->push(v)) + nr_succ++; + else + nr_fail++; + } + })); + } + + for (int oid = 0; oid < kNumObjs; oid++) { + while (list->empty()) + ; + auto v = list->pop(); + if (!v) { + nr_fail++; + continue; + } + if (vmap[*v]) + nr_succ++; + else + nr_err++; + } + + for (auto &thd : thds) { + thd.join(); + } + thds.clear(); + + for (int tid = 0; tid < kNumThds - 1; tid++) { + thds.push_back(std::thread([&, tid = tid] { + for (int oid = 0; oid < kNumObjs; oid++) { + while (list->empty()) + ; + auto v = list->pop(); + if (!v) { + nr_fail++; + continue; + } + if (vmap[*v]) + nr_succ++; + else + nr_err++; + } + })); + } + + for (auto &thd : thds) { + thd.join(); + } + + if (nr_err == 0 && nr_fail == 0) + std::cout << "Test passed! "; + else + std::cout << "Test failed! "; + std::cout << nr_succ << " passed, " << nr_fail << " failed, " << nr_err + << " wrong results." << std::endl; + + return 0; +} \ No newline at end of file diff --git a/test/test_victim_cache.cpp b/test/test_victim_cache.cpp new file mode 100644 index 0000000..b132f31 --- /dev/null +++ b/test/test_victim_cache.cpp @@ -0,0 +1,45 @@ +#include +#include +#include + +#include "object.hpp" +#include "victim_cache.hpp" + +static constexpr int kNumThds = 24; +static constexpr int kNumEntries = 20000; + +int main(int argc, char *argv[]) { + midas::VictimCache vc(10000, 10000); + + std::vector thds; + std::vector optrs[kNumThds]; + for (int i = 0; i < kNumThds; i++) { + thds.emplace_back([&, tid = i] { + for (int j = 0; j < kNumEntries; j++) { + auto optr = new midas::ObjectPtr(); + optrs[tid].push_back(optr); + vc.put(optr, nullptr); + } + }); + } + + for (auto &thd : thds) + thd.join(); + thds.clear(); + + for (int i = 0; i < kNumThds; i++) { + thds.emplace_back([&, tid = i] { + for (auto optr : optrs[tid]) { + vc.remove(optr); + } + }); + } + + for (auto &thd : thds) + thd.join(); + thds.clear(); + + std::cout << "Test passed!" << std::endl; + + return 0; +} \ No newline at end of file