From 420be1185573054960b42a080d8fd63665e7ea95 Mon Sep 17 00:00:00 2001 From: Ming-Hsuan-Tu Date: Mon, 29 Apr 2019 21:34:36 +0800 Subject: [PATCH 01/11] Update README.rst --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 4030f59..d8ef958 100644 --- a/README.rst +++ b/README.rst @@ -8,9 +8,9 @@ Currently I am refactoring the code, and more features like TTL are coming soon. News (2019/04/19) ========= -I have created a branch named `pybind11` which provides the basic functions (`put`, `get` and `delete`) now. And the installtion is much more easily! you can try it if you encounter any installtion issues in the current version of `python-rocksdb`. +I have created a [branch](https://github.com/twmht/python-rocksdb/tree/pybind11) which provides the basic functions (`put`, `get` and `delete`) now. And the installtion is much more easily! you can try it if you encounter any installtion issues in the current version of `python-rocksdb`. -The branch is under development and will be released to PypI after I migrate most of the existing features. +The branch is under development and will be released to PypI after I migrate most of the existing features. The migration is expected to be soon. pyrocksdb ========= From 3a5df052072dfb23fe63992df02d2e971d640320 Mon Sep 17 00:00:00 2001 From: Ming-Hsuan-Tu Date: Mon, 29 Apr 2019 21:35:37 +0800 Subject: [PATCH 02/11] Update README.rst --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index d8ef958..4557382 100644 --- a/README.rst +++ b/README.rst @@ -8,9 +8,9 @@ Currently I am refactoring the code, and more features like TTL are coming soon. News (2019/04/19) ========= -I have created a [branch](https://github.com/twmht/python-rocksdb/tree/pybind11) which provides the basic functions (`put`, `get` and `delete`) now. And the installtion is much more easily! you can try it if you encounter any installtion issues in the current version of `python-rocksdb`. +I have created a new branch(https://github.com/twmht/python-rocksdb/tree/pybind11) which provides the basic functions (`put`, `get` and `delete`) now. And the installtion is much more easily! you can try it if you encounter any installtion issues in the current version of `python-rocksdb`. -The branch is under development and will be released to PypI after I migrate most of the existing features. The migration is expected to be soon. +The branch is under development and will be released to PypI after I migrate most of the existing features. pyrocksdb ========= From b7aa63ec3d92e2e87dc137d3a18b3eaa76cff6ce Mon Sep 17 00:00:00 2001 From: maxibor Date: Sun, 14 Jun 2020 16:52:21 +0200 Subject: [PATCH 03/11] add conda install --- README.rst | 5 +++++ docs/installation.rst | 6 ++++++ 2 files changed, 11 insertions(+) diff --git a/README.rst b/README.rst index 4557382..58e9272 100644 --- a/README.rst +++ b/README.rst @@ -1,3 +1,8 @@ +|conda-badge| + +.. |conda-badge| image:: https://anaconda.org/conda-forge/python-rocksdb/badges/installer/conda.svg + :target: https://conda.anaconda.org/conda-forge + Note ========= The original pyrocksdb (https://pypi.python.org/pypi/pyrocksdb/0.4) has not been updated for long time. I update pyrocksdb to support the latest rocksdb. Please open issues in github if you have any problem. diff --git a/docs/installation.rst b/docs/installation.rst index a5d59f2..e5b3501 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -2,6 +2,12 @@ Installing ========== .. highlight:: bash +With conda +********** + +.. code-block:: bash + + conda install -c conda-forge python-rocksdb With distro package and pypi **************************** From ec3de4206abbbf473897e4026030b346fd15b837 Mon Sep 17 00:00:00 2001 From: maxibor Date: Sun, 14 Jun 2020 16:53:19 +0200 Subject: [PATCH 04/11] link to conda package --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 58e9272..d46de8a 100644 --- a/README.rst +++ b/README.rst @@ -1,7 +1,7 @@ |conda-badge| .. |conda-badge| image:: https://anaconda.org/conda-forge/python-rocksdb/badges/installer/conda.svg - :target: https://conda.anaconda.org/conda-forge + :target: https://anaconda.org/conda-forge/python-rocksdb Note ========= From 9181044e782f7baf59636a1a561cbebf2487fbac Mon Sep 17 00:00:00 2001 From: Dmitry Orlov Date: Wed, 18 Aug 2021 23:39:49 +0300 Subject: [PATCH 05/11] add rocksdb repo as submodule --- .gitmodules | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .gitmodules diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..d3e6e2c --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "src/rocksdb"] + path = src/rocksdb + url = https://github.com/facebook/rocksdb.git From 0dd52751bf81cecb4385b676b21bb4e4549948e4 Mon Sep 17 00:00:00 2001 From: Dmitry Orlov Date: Wed, 18 Aug 2021 23:40:01 +0300 Subject: [PATCH 06/11] add makefile --- Makefile | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..e6d9cf1 --- /dev/null +++ b/Makefile @@ -0,0 +1,30 @@ +all: src/rocksdb/librocksdb.a + +JOBS=8 + +src/rocksdb/librocksdb.a: + make \ + -e EXTRA_CXXFLAGS="-fPIC" \ + -e EXTRA_CFLAGS="-fPIC" \ + -C src/rocksdb \ + -j $(JOBS) \ + libsnappy.a \ + liblz4.a \ + libbz2.a \ + libzstd.a \ + libz.a + + make \ + -j $(JOBS) \ + -e DEBUG_LEVEL=0 \ + -e EXTRA_CXXFLAGS="-fPIC" \ + -e EXTRA_CFLAGS="-fPIC" \ + -e WITH_SNAPPY=1 \ + -e WITH_LZ4=1 \ + -e WITH_ZLIB=1 \ + -e WITH_ZSTD=1 \ + -C src/rocksdb \ + static_lib + +clean: + make -C src/rocksdb clean From cb29ca9d3e5940cde99a9691e03a955b0b02b8de Mon Sep 17 00:00:00 2001 From: Dmitry Orlov Date: Wed, 18 Aug 2021 23:40:41 +0300 Subject: [PATCH 07/11] fix documentation --- README.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index d46de8a..f7b3a63 100644 --- a/README.rst +++ b/README.rst @@ -4,15 +4,15 @@ :target: https://anaconda.org/conda-forge/python-rocksdb Note -========= +==== The original pyrocksdb (https://pypi.python.org/pypi/pyrocksdb/0.4) has not been updated for long time. I update pyrocksdb to support the latest rocksdb. Please open issues in github if you have any problem. News (2019/04/18) -========= -Currently I am refactoring the code, and more features like TTL are coming soon. And the installation with cmake will be much more easily. +================= +Currently I am refactoring the code, and more features like TTL are coming soon. And the installation with cmake will be much more easily. News (2019/04/19) -========= +================= I have created a new branch(https://github.com/twmht/python-rocksdb/tree/pybind11) which provides the basic functions (`put`, `get` and `delete`) now. And the installtion is much more easily! you can try it if you encounter any installtion issues in the current version of `python-rocksdb`. The branch is under development and will be released to PypI after I migrate most of the existing features. From 9c334ae5841fe9f082d555da0a63f5270471bce5 Mon Sep 17 00:00:00 2001 From: Dmitry Orlov Date: Wed, 18 Aug 2021 23:41:17 +0300 Subject: [PATCH 08/11] add static binaries to setup.py --- setup.py | 68 +++++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 58 insertions(+), 10 deletions(-) diff --git a/setup.py b/setup.py index 6b4dfc4..fd0ea73 100644 --- a/setup.py +++ b/setup.py @@ -1,12 +1,23 @@ import platform +import os from setuptools import setup from setuptools import find_packages from setuptools import Extension +try: + from Cython.Build import cythonize +except ImportError: + def cythonize(extensions): + return extensions -extra_compile_args = [ + SOURCES = ['rocksdb/_rocksdb.cpp'] +else: + SOURCES = ['rocksdb/_rocksdb.pyx'] + +EXTRA_COMPILE_ARGS = [ '-std=c++11', - '-O3', + '-fPIC', + '-Os', '-Wall', '-Wextra', '-Wconversion', @@ -14,15 +25,49 @@ extra_compile_args = [ '-fno-rtti', ] +LIBRARIES = ['rocksdb', 'snappy', 'bz2', 'z', 'lz4'] +EXTRA_OBJECTS = [] +EXTRA_LINK_ARGS = [] +INCLUDE_DIRS = [] + + if platform.system() == 'Darwin': - extra_compile_args += ['-mmacosx-version-min=10.7', '-stdlib=libc++'] + EXTRA_COMPILE_ARGS += ['-mmacosx-version-min=10.7', '-stdlib=libc++'] + EXTRA_LINK_ARGS += ['-Wl,-s'] + +if platform.system() == 'Linux': + EXTRA_LINK_ARGS += ['-Wl,--strip-all'] + + +STATIC_LIBRARIES = [os.path.join("src", "rocksdb", item) for item in [ + "librocksdb.a", + "libbz2.a", + "liblz4.a", + "libsnappy.a", + "libz.a", + "libzstd.a", +]] + +if all(map(os.path.exists, STATIC_LIBRARIES)): + LIBRARIES = [] + EXTRA_OBJECTS = STATIC_LIBRARIES + INCLUDE_DIRS = [ + os.path.join("src", "rocksdb", "bzip2-1.0.8"), + os.path.join("src", "rocksdb", "zstd-1.4.9", "lib"), + os.path.join("src", "rocksdb", "zlib-1.2.11"), + os.path.join("src", "rocksdb", "snappy-1.1.8"), + os.path.join("src", "rocksdb", "snappy-1.1.8", "build"), + os.path.join("src", "rocksdb", "lz4-1.9.3", "lib"), + os.path.join("src", "rocksdb", "include"), + ] setup( name="python-rocksdb", version='0.7.0', + keywords=['rocksdb', 'static', 'build'], description="Python bindings for RocksDB", - keywords='rocksdb', + long_description=open("README.rst").read(), author='Ming Hsuan Tu', author_email="qrnnis2623891@gmail.com", url="https://github.com/twmht/python-rocksdb", @@ -31,17 +76,20 @@ setup( install_requires=['setuptools>=25'], package_dir={'rocksdb': 'rocksdb'}, packages=find_packages('.'), - ext_modules=[Extension( + ext_modules=cythonize([Extension( 'rocksdb._rocksdb', - ['rocksdb/_rocksdb.pyx'], - extra_compile_args=extra_compile_args, + SOURCES, + extra_compile_args=EXTRA_COMPILE_ARGS, language='c++', - libraries=['rocksdb', 'snappy', 'bz2', 'z', 'lz4'], - )], + libraries=LIBRARIES, + include_dirs=INCLUDE_DIRS, + extra_objects=EXTRA_OBJECTS, + extra_link_args=EXTRA_LINK_ARGS, + )]), extras_require={ "doc": ['sphinx_rtd_theme', 'sphinx'], "test": ['pytest'], }, - include_package_data=True, + include_package_data=False, zip_safe=False, ) From 9aa8b1d76123020a775ac8a0e5c80fdc8c68c9e8 Mon Sep 17 00:00:00 2001 From: Dmitry Orlov Date: Wed, 18 Aug 2021 23:42:07 +0300 Subject: [PATCH 09/11] add readme to package --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index fc0223f..e324b5d 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,4 @@ +include README.rst include rocksdb/cpp/*.hpp recursive-include rocksdb *.pxd recursive-include rocksdb *.pyx From 17c7fa0a562c20b3b9be181c4ddf1aff93477281 Mon Sep 17 00:00:00 2001 From: Dmitry Orlov Date: Wed, 18 Aug 2021 23:43:54 +0300 Subject: [PATCH 10/11] add github-ci automation --- .github/workflows/build.yml | 127 ++++++++++++++++++++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 .github/workflows/build.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..f08214b --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,127 @@ +# This workflow will install Python dependencies, run tests and lint with a single version of Python +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: build + +on: + push: + branches: + - 'release/**' + release: + types: + - created + +jobs: + wheel: + + runs-on: ${{ matrix.os }} + + strategy: + fail-fast: false + + matrix: + include: + # MacOS + - python: '3.6' + os: macos-latest + - python: '3.7' + os: macos-latest + - python: '3.8' + os: macos-latest + - python: '3.9' + os: macos-latest + # Windows + # - python: '3.6' + # os: windows-latest + #- python: '3.7' + # os: windows-latest + #- python: '3.8' + # os: windows-latest + #- python: '3.9' + # os: windows-latest + + steps: + - uses: actions/checkout@v2 + with: + submodules: recursive + + - name: Cache .a files + uses: actions/cache@v2 + with: + key: ${{ runner.os }} + path: | + src/rocksdb/libsnappy.a + src/rocksdb/liblz4.a + src/rocksdb/libbz2.a + src/rocksdb/libzstd.a + src/rocksdb/libz.a + src/rocksdb/librocksdb.a + + - name: Setup python${{ matrix.python }} + uses: actions/setup-python@v2 + with: + python-version: "${{ matrix.python }}" + + - name: Make static library files + run: make + + - name: Install requires + run: python -m pip install cython twine wheel + + - name: Build wheel for python "${{ matrix.python }}" + run: python setup.py bdist_wheel + + - name: Publishing to pypi + run: twine upload --skip-existing --disable-progress-bar dist/*.whl + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} + + + linux-wheels: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + with: + submodules: recursive + + - name: Cache .a files + uses: actions/cache@v2 + with: + key: ${{ runner.os }} + path: | + src/rocksdb/libsnappy.a + src/rocksdb/liblz4.a + src/rocksdb/libbz2.a + src/rocksdb/libzstd.a + src/rocksdb/libz.a + src/rocksdb/librocksdb.a + + - name: Make static library files + uses: docker://quay.io/pypa/manylinux2014_x86_64 + with: + args: /usr/bin/make + + - name: Install requires + run: python -m pip install cython twine wheel + + - name: Building manylinux2014 wheels + uses: docker://quay.io/pypa/manylinux2014_x86_64 + with: + args: /bin/bash scripts/make-wheels.sh + + - name: Setup python${{ matrix.python }} + uses: actions/setup-python@v2 + with: + python-version: "3.9" + + - name: Install requires + run: python -m pip install twine + + - name: Publishing to pypi + run: twine upload --skip-existing --disable-progress-bar dist/*.whl + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} From 96fa51e963e3cefe6f4cd218ade831abeeeac9a5 Mon Sep 17 00:00:00 2001 From: Dmitry Orlov Date: Thu, 19 Aug 2021 00:18:09 +0300 Subject: [PATCH 11/11] use cmake instead of make --- Makefile | 36 +++++++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index e6d9cf1..e898854 100644 --- a/Makefile +++ b/Makefile @@ -14,17 +14,31 @@ src/rocksdb/librocksdb.a: libzstd.a \ libz.a - make \ - -j $(JOBS) \ - -e DEBUG_LEVEL=0 \ - -e EXTRA_CXXFLAGS="-fPIC" \ - -e EXTRA_CFLAGS="-fPIC" \ - -e WITH_SNAPPY=1 \ - -e WITH_LZ4=1 \ - -e WITH_ZLIB=1 \ - -e WITH_ZSTD=1 \ - -C src/rocksdb \ - static_lib + (cd src/rocksdb && mkdir -p build && cd build && cmake \ + -DWITH_SNAPPY=1 \ + -DWITH_LZ4=1 \ + -DWITH_ZLIB=1 \ + -DWITH_ZSTD=1 \ + -DWITH_GFLAGS=0 \ + -DROCKSDB_BUILD_SHARED=0 \ + -DWITH_TOOLS=0 \ + -DWITH_BENCHMARK_TOOLS=0 \ + -DWITH_CORE_TOOLS=0 \ + -DWITH_JEMALLOC=0 \ + -DCMAKE_BUILD_TYPE=Release \ + -DSnappy_INCLUDE_DIRS=../snappy-1.1.8/ \ + -DSnappy_LIBRARIES=../snappy-1.1.8/build \ + -Dlz4_INCLUDE_DIRS=../lz4-1.9.3/lib \ + -Dlz4_LIBRARIES=../lz4-1.9.3/lib \ + -Dzstd_INCLUDE_DIRS=../zstd-1.4.9/lib \ + -Dzstd_LIBRARIES=../zstd-1.4.9/lib \ + -DZLIB_INCLUDE_DIR=../zlib-1.2.11 \ + -DZLIB_LIBRARY=./zlib-1.2.11 \ + -DCMAKE_CXX_FLAGS="-fPIC -I../snappy-1.1.8/build -I../zstd-1.4.9/lib/dictBuilder" \ + .. && make -j $(JOBS)) + + cp src/rocksdb/build/librocksdb.a src/rocksdb/librocksdb.a clean: + rm -rf src/rocksdb/build make -C src/rocksdb clean