From 4803938f2aeb7c826020fa979aecfed23f94223a Mon Sep 17 00:00:00 2001 From: Bill Zorn Date: Mon, 20 Aug 2018 16:57:14 -0700 Subject: [PATCH] proof of concept manylinux1 build --- .gitignore | 3 + README.md | 191 ++++++++++++++++++++++++++++++++--------- docker-build-wheels.sh | 20 +++++ requirements.txt | 5 ++ setup.py | 11 ++- 5 files changed, 185 insertions(+), 45 deletions(-) create mode 100644 .gitignore create mode 100755 docker-build-wheels.sh create mode 100644 requirements.txt diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f765a1a --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +*~ +wheelhouse/ +.env/ \ No newline at end of file diff --git a/README.md b/README.md index 99f7c17..673d640 100644 --- a/README.md +++ b/README.md @@ -2,44 +2,157 @@ softfloat and softposit in Python * support for softfloat float16, float32, and float64 * support for softposit posit8, quire8, posit16, and quire16 - + ## Demo ``` ->>> import sfpy ->>> sfpy.Float32(1.3) + sfpy.Float32(1.4) # <-- construct from doubles -Float32(2.6999998092651367) ->>> sfpy.Float32(3) # <-- construct from raw bits -Float32(4.203895392974451e-45) ->>> sfpy.Float32(3).bits -3 ->>> x = sfpy.Float16(0) ->>> x -Float16(0.0) ->>> x += sfpy.Float16(10.0) # <-- in-place operators have better performance ->>> x -Float16(10.0) ->>> sfpy.Posit16(1.3) + sfpy.Posit16(1.4) # <-- posits work the same way as floats -Posit16(2.7001953125) ->>> q = sfpy.Quire16(0) # <-- quire is also supported ->>> q -Quire16(0.0) ->>> q.iqam(sfpy.Posit16(3), sfpy.Posit16(5)) ->>> q -Quire16(3.725290298461914e-09) ->>> q.iqam(sfpy.Posit16(3.0), sfpy.Posit16(5.0)) ->>> q -Quire16(15.0) ->>> q.bits -1080863910568919232 ->>> bin(q.bits) -'0b111100000000000000000000000000000000000000000000000011000000' -``` - -## Building (on Linux) -The Cython module can be built in place in the usual way: - -`python setup.py build_ext --inplace` - -This requires the submodules to be checked out, and the static libraries `SoftPosit/build/Linux-x86_64-GCC/softposit.a` and `berkeley-softfloat-3/build/Linux-x86_64-GCC/softfloat.a` to be built. Note that in order for Cython to be able to build the shared objects for the module, the static libraries must be compiled with -fPIC, which currently requires modifying the appropriate Makefiles manually. SoftPosit can be build with -fPIC using its python2 and python3 targets. - -The package can also be installed to a local Python distribution with pip, i.e. `pip install .` from the top level of the repository using the appropriate pip. This requires that Cython be installed. +# WIP +``` + +## Building (on Linux x86_64, with bash) + +### Python Environment +First, make sure a virtual environment is set up with Cython and twine: + +``` +$ python -m venv .env +$ source .env/bin/activate +(.env) $ pip install --upgrade -r requirements.txt +[...] +(.env) $ +``` + +### Cython extensions +Compile the Cython extensions to C: + +``` +(.env) $ cython sfpy/*.pyx +(.env) $ +``` + +### Static libraries +The extension module depends on the softposit and softfloat static libraries. +They can be built with the following: + +``` +(.env) $ (cd SoftPosit/build/Linux-x86_64-GCC/; make clean; make) +[...] +(.env) $ (cd berkeley-softfloat-3/build/Linux-x86_64-GCC/; make clean; make) +[...] +(.env) $ +``` + +Note that some changes are needed to the Makefiles so that the libraries are +compiled with the correct options. Both libraries need to use -fPIC, and +SoftPosit may need -std=c99 to work on older versions of GCC. See the +diff below. + +### Building locally +The extension modules can be built in place in the usual way: + +``` +(.env) $ python setup.py build_ext --inplace +[...] +(.env) $ +``` + +A local wheel (compatible with the python version installed in the virtual +environment) can be built with the following: + +``` +(.env) $ python setup.py bdist_wheel +[...] +(.env) $ +``` + +The local wheel will be created in the dist/ directory. This is the recommended +way to install the package when building it from source locally: + +``` +$ pip install dist/sfpy*.whl +[...] +$ python +>>> from sfpy import * +>>> Posit8(1.3) +Posit8(1.3125) +>>> +``` + +### Building manylinux1 wheels for distribution +Widely compatible linux wheels can be built with the help of PyPA's manylinux +docker image. This requires that Docker is installed, and that the host can pull +the proper docker image. + +To build the manylinux wheels, run: + +``` +$ sudo docker run -u $(id -u) --rm -v `pwd`:/io quay.io/pypa/manylinux1_x86_64 /io/docker-build-wheels.sh +[...] +$ +``` + +This will create a set of manylinux1 wheel files in the wheelhouse/ directory. + +The docker build will make its own static libraries as part of the build process, +and delete any existing static libraries with `make clean`. + +The wheels can be uploaded to PyPI (if you're the package maintainer) with + +``` +(.env) $ twine upload --repository-url https://test.pypi.org/legacy/ wheelhouse/* +[...] +(.env) $ +``` + +to test on test PyPI, or + +``` +(.env) $ twine upload wheelhouse/* +[...] +(.env) $ +``` + +for a release. + +### Makefile customizations +The Makefiles used to build the static libraries need a few small tweaks to +make sure that all the right flags are given to gcc. The changes are shown +in the following diffs. + +SoftPosit/build/Linux-x86_64/Makefile + +``` +diff --git a/build/Linux-x86_64-GCC/Makefile b/build/Linux-x86_64-GCC/Makefile +index 4409e43..46bb877 100644 +--- a/build/Linux-x86_64-GCC/Makefile ++++ b/build/Linux-x86_64-GCC/Makefile +@@ -60,9 +60,9 @@ LINK_PYTHON = \ + + DELETE = rm -f + C_INCLUDES = -I. -I$(SOURCE_DIR)/$(SPECIALIZE_TYPE) -I$(SOURCE_DIR)/include +-OPTIMISATION = -march=core-avx2 -O2 ++OPTIMISATION = -fPIC -O2 ^M + COMPILE_C = \ +- gcc -c -Werror-implicit-function-declaration -DSOFTPOSIT_FAST_INT64 \ ++ gcc -std=c99 -c -Werror-implicit-function-declaration -DSOFTPOSIT_FAST_INT64 \^M + $(SOFTPOSIT_OPTS) $(C_INCLUDES) $(OPTIMISATION) \ + -o $@ + MAKELIB = ar crs $@ +``` + +berkeley-softfloat-3/build/Linux-x86_64/Makefile + +``` +diff --git a/build/Linux-x86_64-GCC/Makefile b/build/Linux-x86_64-GCC/Makefile +index 2ee5dad..b175964 100644 +--- a/build/Linux-x86_64-GCC/Makefile ++++ b/build/Linux-x86_64-GCC/Makefile +@@ -45,7 +45,7 @@ DELETE = rm -f + C_INCLUDES = -I. -I$(SOURCE_DIR)/$(SPECIALIZE_TYPE) -I$(SOURCE_DIR)/include + COMPILE_C = \ + gcc -c -Werror-implicit-function-declaration -DSOFTFLOAT_FAST_INT64 \ +- $(SOFTFLOAT_OPTS) $(C_INCLUDES) -O2 -o $@ ++ $(SOFTFLOAT_OPTS) $(C_INCLUDES) -O2 -fPIC -o $@ + MAKELIB = ar crs $@ + + OBJ = .o +``` diff --git a/docker-build-wheels.sh b/docker-build-wheels.sh new file mode 100755 index 0000000..d328c08 --- /dev/null +++ b/docker-build-wheels.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -e -x +cd /io/ +ls + +(cd SoftPosit/build/Linux-x86_64-GCC; make clean; make) +(cd berkeley-softfloat-3/build/Linux-x86_64-GCC; make clean; make) + +for PYBIN in /opt/python/*/bin; do + "${PYBIN}/pip" wheel . -w wheelhouse/ +done + +for whl in wheelhouse/*.whl; do + auditwheel repair "$whl" -w wheelhouse/ +done + +rm wheelhouse/*linux_x86_64.whl + +(cd SoftPosit/build/Linux-x86_64-GCC; make clean) +(cd berkeley-softfloat-3/build/Linux-x86_64-GCC; make clean) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..0fd68fc --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +pip +wheel +twine +keyring +cython==0.28.5 diff --git a/setup.py b/setup.py index 53208a4..65daf53 100644 --- a/setup.py +++ b/setup.py @@ -1,17 +1,16 @@ import setuptools -from Cython.Build import cythonize posit_ext = setuptools.Extension( - 'sfpy.posit', ['sfpy/posit.pyx'], + 'sfpy.posit', ['sfpy/posit.c'], include_dirs=['SoftPosit/source/include/'], - extra_objects=['./SoftPosit/build/Linux-x86_64-GCC/softposit.a'], + extra_objects=['SoftPosit/build/Linux-x86_64-GCC/softposit.a'], libraries=['m'], ) float_ext = setuptools.Extension( - 'sfpy.float', ['sfpy/float.pyx'], + 'sfpy.float', ['sfpy/float.c'], include_dirs=['berkeley-softfloat-3/source/include/'], - extra_objects=['./berkeley-softfloat-3/build/Linux-x86_64-GCC/softfloat.a'], + extra_objects=['berkeley-softfloat-3/build/Linux-x86_64-GCC/softfloat.a'], ) setuptools.setup( @@ -22,5 +21,5 @@ setuptools.setup( author_email='bill.zorn@gmail.com', url='https://github.com/billzorn/sfpy', packages=['sfpy'], - ext_modules=cythonize([posit_ext, float_ext]), + ext_modules=[posit_ext, float_ext], ) -- 2.30.2