From e3eccbb42014af40661c181f28d7b5ba07db8d5a Mon Sep 17 00:00:00 2001 From: Yun Tang Date: Thu, 22 Oct 2020 23:21:31 +0800 Subject: [PATCH] Frocksdb release guide and helping scripts This patch combines https://github.com/dataArtisans/frocksdb/commit/92a7360d346706506093791b35a0edda9ef243e0 and https://github.com/dataArtisans/frocksdb/commit/810b42aba4fb19dc402a8a20597af390481ab5db fix DEBUG_LEVEL not set to 0 for the frocksdb* targets This mainly affects the MacOS builds if RELEASE-FROCKSDB.md is followed. --- HISTORY.md | 8 +++ Makefile | 76 +++++++++++++++++++++++- RELEASE-FROCKSDB.md | 109 ++++++++++++++++++++++++++++++++++ java/crossbuild/build-win.bat | 16 +++++ java/deploysettings.xml | 12 ++++ java/publish-frocksdbjni.sh | 42 +++++++++++++ java/rocksjni.pom | 17 ++---- 7 files changed, 268 insertions(+), 12 deletions(-) create mode 100644 RELEASE-FROCKSDB.md create mode 100644 java/crossbuild/build-win.bat create mode 100644 java/deploysettings.xml create mode 100644 java/publish-frocksdbjni.sh diff --git a/HISTORY.md b/HISTORY.md index 399d87714..f91c79b71 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,3 +1,11 @@ +# FRocksdb Change Log +## 6.11.6-veverica-1.0 (10/22/2020) +### Features +* [Flink TTL] compaction filter for background cleanup of state with time-to-live +* [Flink ListState] Separator-free merge operator +### Bug FIxes +* Fix a bug which may cause MultiGet to be slow because it may read more data than requested, but this won't affect correctness. The bug was introduced in 6.10 release. + # Rocksdb Change Log ## 6.10.2 (6/5/2020) ### Bug fix diff --git a/Makefile b/Makefile index c44c5ab66..b0e43d389 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,8 @@ #----------------------------------------------- +FROCKSDB_VERSION ?= 1.0 + BASH_EXISTS := $(shell which bash) SHELL := $(shell which bash) PYTHON?=$(shell which python) @@ -121,6 +123,23 @@ ifeq ($(LITE), 0) else OPT += -Os endif + +ifeq ($(MAKECMDGOALS),frocksdbjavastatic) + ifneq ($(DEBUG_LEVEL),2) + DEBUG_LEVEL=0 + endif +endif + +ifeq ($(MAKECMDGOALS),frocksdbjavastaticrelease) + DEBUG_LEVEL=0 +endif + +ifeq ($(MAKECMDGOALS),frocksdbjavastaticreleasedocker) + DEBUG_LEVEL=0 +endif + +ifeq ($(MAKECMDGOALS),frocksdbjavastaticpublish) + DEBUG_LEVEL=0 endif # compile with -O2 if debug level is not 2 @@ -2032,17 +2051,72 @@ rocksdbjavastatic: $(java_static_all_libobjects) strip $(STRIPFLAGS) $(ROCKSDBJNILIB); \ fi cd java;jar -cf target/$(ROCKSDB_JAR) HISTORY*.md + jar -uf java/target/$(ROCKSDB_JAR) HISTORY*.md cd java/target;jar -uf $(ROCKSDB_JAR) $(ROCKSDBJNILIB) cd java/target/classes;jar -uf ../$(ROCKSDB_JAR) org/rocksdb/*.class org/rocksdb/util/*.class cd java/target/apidocs;jar -cf ../$(ROCKSDB_JAVADOCS_JAR) * cd java/src/main/java;jar -cf ../../../target/$(ROCKSDB_SOURCES_JAR) org + mkdir -p java/target/META-INF + cp LICENSE.Apache java/target/META-INF/LICENSE + cd java/target;jar -uf $(ROCKSDB_JAR) META-INF/LICENSE + rocksdbjavastaticrelease: rocksdbjavastatic cd java/crossbuild && (vagrant destroy -f || true) && vagrant up linux32 && vagrant halt linux32 && vagrant up linux64 && vagrant halt linux64 && vagrant up linux64-musl && vagrant halt linux64-musl cd java;jar -cf target/$(ROCKSDB_JAR_ALL) HISTORY*.md - cd java/target;jar -uf $(ROCKSDB_JAR_ALL) librocksdbjni-*.so librocksdbjni-*.jnilib + jar -uf java/target/$(ROCKSDB_JAR_ALL) HISTORY*.md + cd java/target;jar -uf $(ROCKSDB_JAR_ALL) librocksdbjni-*.so librocksdbjni-*.jnilib librocksdbjni-win64.dll cd java/target/classes;jar -uf ../$(ROCKSDB_JAR_ALL) org/rocksdb/*.class org/rocksdb/util/*.class +frocksdbjavastaticreleasedocker: rocksdbjavastaticreleasedocker + # update license + mkdir -p java/target/META-INF + cp LICENSE.Apache java/target/META-INF/LICENSE + cd java/target;jar -uf $(ROCKSDB_JAR_ALL) META-INF/LICENSE + + # platform jars + $(eval JAR_PREF=rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)) + $(eval JAR_DOCS=$(JAR_PREF)-javadoc.jar) + $(eval JAR_SOURCES=$(JAR_PREF)-sources.jar) + $(eval OSX_JAR=$(JAR_PREF)-osx.jar) + $(eval WIN_JAR=$(JAR_PREF)-win64.jar) + $(eval LINUX32_JAR=$(JAR_PREF)-linux32.jar) + $(eval LINUX64_JAR=$(JAR_PREF)-linux64.jar) + + # update windows jar + cd java/target;cp rocksdbjni_classes.jar $(WIN_JAR) + cd java;jar -uf target/$(WIN_JAR) HISTORY*.md + jar -uf java/target/$(WIN_JAR) HISTORY*.md + cd java/target;jar -uf $(WIN_JAR) librocksdbjni-win64.dll + cd java/target;jar -uf $(WIN_JAR) META-INF/LICENSE + + # update linux 64 jar with ppc64 lib + cd java/target;jar -uf $(LINUX64_JAR) librocksdbjni-linux-ppc64le.so + + cd java/target;jar -uf $(JAR_DOCS) META-INF/LICENSE + cd java/target;jar -uf $(JAR_SOURCES) META-INF/LICENSE + + # prepare frocksdb release + cd java/target;mkdir -p frocksdb-release + + $(eval FJAR_PREF=frocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-ververica-$(FROCKSDB_VERSION)) + $(eval FJAR=$(FJAR_PREF).jar) + $(eval FJAR_DOCS=$(FJAR_PREF)-javadoc.jar) + $(eval FJAR_SOURCES=$(FJAR_PREF)-sources.jar) + $(eval OSX_FJAR=$(FJAR_PREF)-osx.jar) + $(eval WIN_FJAR=$(FJAR_PREF)-win64.jar) + $(eval LINUX32_FJAR=$(FJAR_PREF)-linux32.jar) + $(eval LINUX64_FJAR=$(FJAR_PREF)-linux64.jar) + + cd java/target;cp $(ROCKSDB_JAR_ALL) frocksdb-release/$(FJAR) + cd java/target;cp $(JAR_DOCS) frocksdb-release/$(FJAR_DOCS) + cd java/target;cp $(JAR_SOURCES) frocksdb-release/$(FJAR_SOURCES) + cd java/target;cp $(OSX_JAR) frocksdb-release/$(OSX_FJAR) + cd java/target;cp $(WIN_JAR) frocksdb-release/$(WIN_FJAR) + cd java/target;cp $(LINUX32_JAR) frocksdb-release/$(LINUX32_FJAR) + cd java/target;cp $(LINUX64_JAR) frocksdb-release/$(LINUX64_FJAR) + cd java;cp rocksjni.pom target/frocksdb-release/$(FJAR_PREF).pom + rocksdbjavastaticreleasedocker: rocksdbjavastatic rocksdbjavastaticdockerx86 rocksdbjavastaticdockerx86_64 rocksdbjavastaticdockerx86musl rocksdbjavastaticdockerx86_64musl cd java;jar -cf target/$(ROCKSDB_JAR_ALL) HISTORY*.md cd java/target;jar -uf $(ROCKSDB_JAR_ALL) librocksdbjni-*.so librocksdbjni-*.jnilib diff --git a/RELEASE-FROCKSDB.md b/RELEASE-FROCKSDB.md new file mode 100644 index 000000000..953dc83b0 --- /dev/null +++ b/RELEASE-FROCKSDB.md @@ -0,0 +1,109 @@ + +@@ -0,0 +1,104 @@ +## Build in Windows + +Use Windows 64 bit machine (e.g. base AWS Windows instance: 4 cores, 16GB RAM, 40GB storage for build). + +Install: + * git + * java 8 + * maven + * Visual Studio Community 15 (2017) + +With [chocolatey](https://chocolatey.org/install): + + choco install git.install jdk8 maven visualstudio2017community + +Optionally: + + choco install intellijidea-community vscode + +Open developer command prompt for vs 2017 and run commands: + + git clone git@github.com:ververica/frocksdb.git + cd frocksdb + git checkout FRocksDB-6.11.6 # release branch + java\crossbuild\build-win.bat + +The result native library is `build\java\Release\rocksdbjni-shared.dll`. +The result windows jar is `build\java\rocksdbjni_classes.jar`. + +There is also a how-to in CMakeLists.txt. + +## Build in PPC64LE + +Use Ubuntu 16.04 (e.g. AWS instance 4 cores, 16GB RAM, 40GB storage for build). +Install git if not installed. If docker is installed, it might need to be removed. + +Setup ppc64le docker machine ([source](https://developer.ibm.com/linuxonpower/2017/06/08/build-test-ppc64le-docker-images-intel/)): + + wget http://ftp.unicamp.br/pub/ppc64el/boot2docker/install.sh && chmod +x ./install.sh && ./install.sh -s + docker-machine create -d qemu \ + --qemu-boot2docker-url=/home/ubuntu/.docker/machine/boot2docker.iso \ + --qemu-memory 8192 \ + --qemu-cache-mode none \ + --qemu-arch ppc64le \ + vm-ppc64le + +Regenerate certs as suggested if it did not work at once. + +Prepare docker machine to run rocksdbjni docker image for ppc64le build: + + eval $(docker-machine env vm-ppc64le) + git clone git@github.com:ververica/frocksdb.git + cd frocksdb + git checkout FRocksDB-6.11.6 # release branch + docker-machine ssh vm-ppc64le mkdir -p `pwd` + docker-machine scp -r . vm-ppc64le:`pwd` + +Build frocksdb: + + make rocksdbjavastaticdockerppc64le + docker-machine scp vm-ppc64le:`pwd`/java/target/librocksdbjni-linux-ppc64le.so java/target/. + + make rocksdbjavastaticdockerppc64lemusl + docker-machine scp vm-ppc64le:`pwd`/java/target/librocksdbjni-linux-ppc64le-musl.so java/target/. + +The result native library is in `java/target/librocksdbjni-linux-ppc64le.so` and `java/target/librocksdbjni-linux-ppc64le-musl.so`. + +## Final crossbuild in Mac OSX + +Read how to Build cross jar for Mac OSX and linux as described in java/RELEASE.md but do not run it yet. + +Run commands: + + make jclean clean + mkdir -p java/target + cp /rocksdbjni-shared.dll java/target/librocksdbjni-win64.dll + cp /rocksdbjni_classes.jar java/target/rocksdbjni_classes.jar + cp /librocksdbjni-linux-ppc64le.so java/target/librocksdbjni-linux-ppc64le.so + FROCKSDB_VERSION=1.0 make frocksdbjavastaticreleasedocker + +## Push to maven central + +Edit the `frocksdbjni-.pom` file and replace +`-` +with +`` +where the `` is e.g. `6.11.6-veverica-1.0`. + +Run: +```bash +VERSION= \ +USER= \ +PASSWORD= \ +KEYNAME= \ +PASSPHRASE= \ +java/publish-frocksdbjni.sh +``` + +Go to the staging repositories on Sonatype: + +https://oss.sonatype.org/#stagingRepositories + +Select the open staging repository and click on "Close". + +Test the files in the staging repository +which will look something like this `https://oss.sonatype.org/content/repositories/comdata-artisans-1020`. + +Press the "Release" button (WARNING: this can not be undone). diff --git a/java/crossbuild/build-win.bat b/java/crossbuild/build-win.bat new file mode 100644 index 000000000..963f050b8 --- /dev/null +++ b/java/crossbuild/build-win.bat @@ -0,0 +1,16 @@ +:: install git, java 8, maven, visual studio community 15 (2017) + +set MSBUILD=C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\MSBuild\15.0\Bin\MSBuild.exe + +if exist build rd /s /q build +if exist librocksdbjni-win64.dll del librocksdbjni-win64.dll +mkdir build && cd build + +cmake -G "Visual Studio 15 Win64" -DWITH_JNI=1 .. + +"%MSBUILD%" rocksdb.sln /p:Configuration=Release /m + +cd .. + +copy build\java\Release\rocksdbjni-shared.dll librocksdbjni-win64.dll +echo Result is in librocksdbjni-win64.dll diff --git a/java/deploysettings.xml b/java/deploysettings.xml new file mode 100644 index 000000000..02528b646 --- /dev/null +++ b/java/deploysettings.xml @@ -0,0 +1,12 @@ + + + + sonatype-nexus-staging + ${sonatype_user} + ${sonatype_pw} + + + diff --git a/java/publish-frocksdbjni.sh b/java/publish-frocksdbjni.sh new file mode 100644 index 000000000..ddd14a98f --- /dev/null +++ b/java/publish-frocksdbjni.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# fail on errors +set -e + +PREFIX=java/target/frocksdb-release/frocksdbjni-${VERSION} + +function deploy() { + FILE=$1 + CLASSIFIER=$2 + echo "Deploying file=${FILE} with classifier=${CLASSIFIER} to sonatype with prefix=${PREFIX}" + sonatype_user=${USER} sonatype_pw=${PASSWORD} mvn gpg:sign-and-deploy-file \ + --settings java/deploysettings.xml \ + -Durl=https://oss.sonatype.org/service/local/staging/deploy/maven2/ \ + -DrepositoryId=sonatype-nexus-staging \ + -DpomFile=${PREFIX}.pom \ + -Dfile=$FILE \ + -Dclassifier=$CLASSIFIER \ + -Dgpg.keyname=${KEYNAME} \ + -Dgpg.passphrase=${PASSPHRASE} +} + +deploy ${PREFIX}-sources.jar sources +deploy ${PREFIX}-javadoc.jar javadoc +deploy ${PREFIX}.jar diff --git a/java/rocksjni.pom b/java/rocksjni.pom index 5defdca7d..aebe963ed 100644 --- a/java/rocksjni.pom +++ b/java/rocksjni.pom @@ -4,14 +4,14 @@ xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> 4.0.0 - RocksDB JNI - http://rocksdb.org/ - org.rocksdb - rocksdbjni + FRocksDB JNI + https://github.com/ververica/frocksdb + com.ververica + frocksdbjni - - RocksDB fat jar that contains .so files for linux32 and linux64 (glibc and musl-libc), jnilib files - for Mac OSX, and a .dll for Windows x64. + RocksDB fat jar with modifications specific for Apache Flink + that contains .so files for linux32 and linux64, jnilib files for Mac OSX, and a .dll for Windows x64. @@ -19,11 +19,6 @@ http://www.apache.org/licenses/LICENSE-2.0.html repo - - GNU General Public License, version 2 - http://www.gnu.org/licenses/gpl-2.0.html - repo - scm:git:git://github.com/dropwizard/metrics.git