diff options
Diffstat (limited to 'src/datastore')
25 files changed, 11762 insertions, 0 deletions
diff --git a/src/datastore/Makefile.am b/src/datastore/Makefile.am new file mode 100644 index 0000000..44c5bbe --- /dev/null +++ b/src/datastore/Makefile.am @@ -0,0 +1,234 @@ +INCLUDES = -I$(top_srcdir)/src/include + +plugindir = $(libdir)/gnunet + +pkgcfgdir= $(pkgdatadir)/config.d/ + +pkgcfg_DATA = \ + datastore.conf + +if MINGW + WINFLAGS = -Wl,--no-undefined -Wl,--export-all-symbols +endif + +if USE_COVERAGE + AM_CFLAGS = --coverage -O0 + XLIBS = -lgcov +endif + + +lib_LTLIBRARIES = \ + libgnunetdatastore.la + +libgnunetdatastore_la_SOURCES = \ + datastore_api.c datastore.h +libgnunetdatastore_la_LIBADD = \ + $(top_builddir)/src/statistics/libgnunetstatistics.la \ + $(top_builddir)/src/util/libgnunetutil.la \ + $(GN_LIBINTL) +libgnunetdatastore_la_LDFLAGS = \ + $(GN_LIB_LDFLAGS) $(WINFLAGS) \ + -version-info 1:0:0 + + +bin_PROGRAMS = \ + gnunet-service-datastore + +gnunet_service_datastore_SOURCES = \ + gnunet-service-datastore.c +gnunet_service_datastore_LDADD = \ + $(top_builddir)/src/statistics/libgnunetstatistics.la \ + $(top_builddir)/src/util/libgnunetutil.la \ + $(GN_LIBINTL) + +if HAVE_MYSQL + MYSQL_PLUGIN = libgnunet_plugin_datastore_mysql.la +if HAVE_BENCHMARKS + MYSQL_BENCHMARKS = \ + perf_datastore_api_mysql \ + perf_plugin_datastore_mysql +endif + MYSQL_TESTS = \ + test_datastore_api_mysql \ + test_datastore_api_management_mysql \ + test_plugin_datastore_mysql \ + $(MYSQL_BENCHMARKS) +endif +if HAVE_SQLITE + SQLITE_PLUGIN = libgnunet_plugin_datastore_sqlite.la +if HAVE_BENCHMARKS + SQLITE_BENCHMARKS = \ + perf_datastore_api_sqlite \ + perf_plugin_datastore_sqlite +endif + SQLITE_TESTS = \ + test_datastore_api_sqlite \ + test_datastore_api_management_sqlite \ + test_plugin_datastore_sqlite \ + $(SQLITE_BENCHMARKS) +endif +if HAVE_POSTGRES + POSTGRES_PLUGIN = libgnunet_plugin_datastore_postgres.la +if HAVE_BENCHMARKS + POSTGRES_BENCHMARKS = \ + perf_datastore_api_postgres \ + perf_plugin_datastore_postgres +endif + POSTGRES_TESTS = \ + test_datastore_api_postgres \ + test_datastore_api_management_postgres \ + test_plugin_datastore_postgres \ + $(POSTGRES_BENCHMARKS) +endif + +plugin_LTLIBRARIES = \ + $(SQLITE_PLUGIN) \ + $(MYSQL_PLUGIN) \ + $(POSTGRES_PLUGIN) \ + libgnunet_plugin_datastore_template.la + + +libgnunet_plugin_datastore_sqlite_la_SOURCES = \ + plugin_datastore_sqlite.c +libgnunet_plugin_datastore_sqlite_la_LIBADD = \ + $(top_builddir)/src/statistics/libgnunetstatistics.la \ + $(top_builddir)/src/util/libgnunetutil.la $(XLIBS) -lsqlite3 +libgnunet_plugin_datastore_sqlite_la_LDFLAGS = \ + $(GN_PLUGIN_LDFLAGS) + + +libgnunet_plugin_datastore_mysql_la_SOURCES = \ + plugin_datastore_mysql.c +libgnunet_plugin_datastore_mysql_la_LIBADD = \ + $(top_builddir)/src/statistics/libgnunetstatistics.la \ + $(top_builddir)/src/util/libgnunetutil.la $(XLIBS) -lz -lmysqlclient +libgnunet_plugin_datastore_mysql_la_LDFLAGS = \ + $(GN_PLUGIN_LDFLAGS) $(MYSQL_LDFLAGS) -lmysqlclient +libgnunet_plugin_datastore_mysql_la_CPPFLAGS = \ + $(MYSQL_CPPFLAGS) + +libgnunet_plugin_datastore_postgres_la_SOURCES = \ + plugin_datastore_postgres.c +libgnunet_plugin_datastore_postgres_la_LIBADD = \ + $(top_builddir)/src/statistics/libgnunetstatistics.la \ + $(top_builddir)/src/util/libgnunetutil.la $(XLIBS) -lpq +libgnunet_plugin_datastore_postgres_la_LDFLAGS = \ + $(GN_PLUGIN_LDFLAGS) $(POSTGRES_LDFLAGS) -lpq +libgnunet_plugin_datastore_postgres_la_CPPFLAGS = \ + $(POSTGRES_CPPFLAGS) + + +libgnunet_plugin_datastore_template_la_SOURCES = \ + plugin_datastore_template.c +libgnunet_plugin_datastore_template_la_LIBADD = \ + $(top_builddir)/src/util/libgnunetutil.la $(XLIBS) +libgnunet_plugin_datastore_template_la_LDFLAGS = \ + $(GN_PLUGIN_LDFLAGS) + +check_PROGRAMS = \ + $(SQLITE_TESTS) \ + $(MYSQL_TESTS) \ + $(POSTGRES_TESTS) + +if ENABLE_TEST_RUN +TESTS = $(check_PROGRAMS) +endif + +test_datastore_api_sqlite_SOURCES = \ + test_datastore_api.c +test_datastore_api_sqlite_LDADD = \ + $(top_builddir)/src/datastore/libgnunetdatastore.la \ + $(top_builddir)/src/util/libgnunetutil.la + +test_datastore_api_management_sqlite_SOURCES = \ + test_datastore_api_management.c +test_datastore_api_management_sqlite_LDADD = \ + $(top_builddir)/src/datastore/libgnunetdatastore.la \ + $(top_builddir)/src/util/libgnunetutil.la + +perf_datastore_api_sqlite_SOURCES = \ + perf_datastore_api.c +perf_datastore_api_sqlite_LDADD = \ + $(top_builddir)/src/datastore/libgnunetdatastore.la \ + $(top_builddir)/src/util/libgnunetutil.la + +perf_plugin_datastore_sqlite_SOURCES = \ + perf_plugin_datastore.c +perf_plugin_datastore_sqlite_LDADD = \ + $(top_builddir)/src/util/libgnunetutil.la + +test_plugin_datastore_sqlite_SOURCES = \ + test_plugin_datastore.c +test_plugin_datastore_sqlite_LDADD = \ + $(top_builddir)/src/util/libgnunetutil.la + + +test_datastore_api_mysql_SOURCES = \ + test_datastore_api.c +test_datastore_api_mysql_LDADD = \ + $(top_builddir)/src/datastore/libgnunetdatastore.la \ + $(top_builddir)/src/util/libgnunetutil.la + +test_datastore_api_management_mysql_SOURCES = \ + test_datastore_api_management.c +test_datastore_api_management_mysql_LDADD = \ + $(top_builddir)/src/datastore/libgnunetdatastore.la \ + $(top_builddir)/src/util/libgnunetutil.la + +perf_datastore_api_mysql_SOURCES = \ + perf_datastore_api.c +perf_datastore_api_mysql_LDADD = \ + $(top_builddir)/src/datastore/libgnunetdatastore.la \ + $(top_builddir)/src/util/libgnunetutil.la + +test_plugin_datastore_mysql_SOURCES = \ + test_plugin_datastore.c +test_plugin_datastore_mysql_LDADD = \ + $(top_builddir)/src/util/libgnunetutil.la + +perf_plugin_datastore_mysql_SOURCES = \ + perf_plugin_datastore.c +perf_plugin_datastore_mysql_LDADD = \ + $(top_builddir)/src/util/libgnunetutil.la + + +test_datastore_api_postgres_SOURCES = \ + test_datastore_api.c +test_datastore_api_postgres_LDADD = \ + $(top_builddir)/src/datastore/libgnunetdatastore.la \ + $(top_builddir)/src/util/libgnunetutil.la + +test_datastore_api_management_postgres_SOURCES = \ + test_datastore_api_management.c +test_datastore_api_management_postgres_LDADD = \ + $(top_builddir)/src/datastore/libgnunetdatastore.la \ + $(top_builddir)/src/util/libgnunetutil.la + +perf_datastore_api_postgres_SOURCES = \ + perf_datastore_api.c +perf_datastore_api_postgres_LDADD = \ + $(top_builddir)/src/datastore/libgnunetdatastore.la \ + $(top_builddir)/src/util/libgnunetutil.la + +test_plugin_datastore_postgres_SOURCES = \ + test_plugin_datastore.c +test_plugin_datastore_postgres_LDADD = \ + $(top_builddir)/src/util/libgnunetutil.la + +perf_plugin_datastore_postgres_SOURCES = \ + perf_plugin_datastore.c +perf_plugin_datastore_postgres_LDADD = \ + $(top_builddir)/src/util/libgnunetutil.la + + +EXTRA_DIST = \ + test_defaults.conf \ + test_datastore_api_data_sqlite.conf \ + perf_plugin_datastore_data_sqlite.conf \ + test_datastore_api_data_mysql.conf \ + perf_plugin_datastore_data_mysql.conf \ + test_datastore_api_data_postgres.conf \ + perf_plugin_datastore_data_postgres.conf \ + test_plugin_datastore_data_mysql.conf \ + test_plugin_datastore_data_postgres.conf \ + test_plugin_datastore_data_sqlite.conf
\ No newline at end of file diff --git a/src/datastore/Makefile.in b/src/datastore/Makefile.in new file mode 100644 index 0000000..d85f1ea --- /dev/null +++ b/src/datastore/Makefile.in @@ -0,0 +1,1384 @@ +# Makefile.in generated by automake 1.11.1 from Makefile.am. +# @configure_input@ + +# Copyright (C) 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, +# 2003, 2004, 2005, 2006, 2007, 2008, 2009 Free Software Foundation, +# Inc. +# This Makefile.in is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY, to the extent permitted by law; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. + +@SET_MAKE@ + + + +VPATH = @srcdir@ +pkgdatadir = $(datadir)/@PACKAGE@ +pkgincludedir = $(includedir)/@PACKAGE@ +pkglibdir = $(libdir)/@PACKAGE@ +pkglibexecdir = $(libexecdir)/@PACKAGE@ +am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd +install_sh_DATA = $(install_sh) -c -m 644 +install_sh_PROGRAM = $(install_sh) -c +install_sh_SCRIPT = $(install_sh) -c +INSTALL_HEADER = $(INSTALL_DATA) +transform = $(program_transform_name) +NORMAL_INSTALL = : +PRE_INSTALL = : +POST_INSTALL = : +NORMAL_UNINSTALL = : +PRE_UNINSTALL = : +POST_UNINSTALL = : +build_triplet = @build@ +host_triplet = @host@ +target_triplet = @target@ +bin_PROGRAMS = gnunet-service-datastore$(EXEEXT) +check_PROGRAMS = $(am__EXEEXT_2) $(am__EXEEXT_4) $(am__EXEEXT_6) +subdir = src/datastore +DIST_COMMON = $(srcdir)/Makefile.am $(srcdir)/Makefile.in \ + $(srcdir)/datastore.conf.in +ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 +am__aclocal_m4_deps = $(top_srcdir)/m4/absolute-header.m4 \ + $(top_srcdir)/m4/align.m4 $(top_srcdir)/m4/argz.m4 \ + $(top_srcdir)/m4/gettext.m4 $(top_srcdir)/m4/iconv.m4 \ + $(top_srcdir)/m4/lib-ld.m4 $(top_srcdir)/m4/lib-link.m4 \ + $(top_srcdir)/m4/lib-prefix.m4 $(top_srcdir)/m4/libcurl.m4 \ + $(top_srcdir)/m4/libgcrypt.m4 $(top_srcdir)/m4/libtool.m4 \ + $(top_srcdir)/m4/libunistring.m4 $(top_srcdir)/m4/ltdl.m4 \ + $(top_srcdir)/m4/ltoptions.m4 $(top_srcdir)/m4/ltsugar.m4 \ + $(top_srcdir)/m4/ltversion.m4 $(top_srcdir)/m4/lt~obsolete.m4 \ + $(top_srcdir)/m4/nls.m4 $(top_srcdir)/m4/po.m4 \ + $(top_srcdir)/m4/progtest.m4 $(top_srcdir)/acinclude.m4 \ + $(top_srcdir)/configure.ac +am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \ + $(ACLOCAL_M4) +mkinstalldirs = $(install_sh) -d +CONFIG_HEADER = $(top_builddir)/gnunet_config.h +CONFIG_CLEAN_FILES = datastore.conf +CONFIG_CLEAN_VPATH_FILES = +am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`; +am__vpath_adj = case $$p in \ + $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \ + *) f=$$p;; \ + esac; +am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`; +am__install_max = 40 +am__nobase_strip_setup = \ + srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'` +am__nobase_strip = \ + for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||" +am__nobase_list = $(am__nobase_strip_setup); \ + for p in $$list; do echo "$$p $$p"; done | \ + sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \ + $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \ + if (++n[$$2] == $(am__install_max)) \ + { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \ + END { for (dir in files) print dir, files[dir] }' +am__base_list = \ + sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \ + sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g' +am__installdirs = "$(DESTDIR)$(libdir)" "$(DESTDIR)$(plugindir)" \ + "$(DESTDIR)$(bindir)" "$(DESTDIR)$(pkgcfgdir)" +LTLIBRARIES = $(lib_LTLIBRARIES) $(plugin_LTLIBRARIES) +am__DEPENDENCIES_1 = +libgnunet_plugin_datastore_mysql_la_DEPENDENCIES = \ + $(top_builddir)/src/statistics/libgnunetstatistics.la \ + $(top_builddir)/src/util/libgnunetutil.la \ + $(am__DEPENDENCIES_1) +am_libgnunet_plugin_datastore_mysql_la_OBJECTS = \ + libgnunet_plugin_datastore_mysql_la-plugin_datastore_mysql.lo +libgnunet_plugin_datastore_mysql_la_OBJECTS = \ + $(am_libgnunet_plugin_datastore_mysql_la_OBJECTS) +AM_V_lt = $(am__v_lt_$(V)) +am__v_lt_ = $(am__v_lt_$(AM_DEFAULT_VERBOSITY)) +am__v_lt_0 = --silent +libgnunet_plugin_datastore_mysql_la_LINK = $(LIBTOOL) $(AM_V_lt) \ + --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link \ + $(CCLD) $(AM_CFLAGS) $(CFLAGS) \ + $(libgnunet_plugin_datastore_mysql_la_LDFLAGS) $(LDFLAGS) -o \ + $@ +@HAVE_MYSQL_TRUE@am_libgnunet_plugin_datastore_mysql_la_rpath = \ +@HAVE_MYSQL_TRUE@ -rpath $(plugindir) +libgnunet_plugin_datastore_postgres_la_DEPENDENCIES = \ + $(top_builddir)/src/statistics/libgnunetstatistics.la \ + $(top_builddir)/src/util/libgnunetutil.la \ + $(am__DEPENDENCIES_1) +am_libgnunet_plugin_datastore_postgres_la_OBJECTS = libgnunet_plugin_datastore_postgres_la-plugin_datastore_postgres.lo +libgnunet_plugin_datastore_postgres_la_OBJECTS = \ + $(am_libgnunet_plugin_datastore_postgres_la_OBJECTS) +libgnunet_plugin_datastore_postgres_la_LINK = $(LIBTOOL) $(AM_V_lt) \ + --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link \ + $(CCLD) $(AM_CFLAGS) $(CFLAGS) \ + $(libgnunet_plugin_datastore_postgres_la_LDFLAGS) $(LDFLAGS) \ + -o $@ +@HAVE_POSTGRES_TRUE@am_libgnunet_plugin_datastore_postgres_la_rpath = \ +@HAVE_POSTGRES_TRUE@ -rpath $(plugindir) +libgnunet_plugin_datastore_sqlite_la_DEPENDENCIES = \ + $(top_builddir)/src/statistics/libgnunetstatistics.la \ + $(top_builddir)/src/util/libgnunetutil.la \ + $(am__DEPENDENCIES_1) +am_libgnunet_plugin_datastore_sqlite_la_OBJECTS = \ + plugin_datastore_sqlite.lo +libgnunet_plugin_datastore_sqlite_la_OBJECTS = \ + $(am_libgnunet_plugin_datastore_sqlite_la_OBJECTS) +libgnunet_plugin_datastore_sqlite_la_LINK = $(LIBTOOL) $(AM_V_lt) \ + --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link \ + $(CCLD) $(AM_CFLAGS) $(CFLAGS) \ + $(libgnunet_plugin_datastore_sqlite_la_LDFLAGS) $(LDFLAGS) -o \ + $@ +@HAVE_SQLITE_TRUE@am_libgnunet_plugin_datastore_sqlite_la_rpath = \ +@HAVE_SQLITE_TRUE@ -rpath $(plugindir) +libgnunet_plugin_datastore_template_la_DEPENDENCIES = \ + $(top_builddir)/src/util/libgnunetutil.la \ + $(am__DEPENDENCIES_1) +am_libgnunet_plugin_datastore_template_la_OBJECTS = \ + plugin_datastore_template.lo +libgnunet_plugin_datastore_template_la_OBJECTS = \ + $(am_libgnunet_plugin_datastore_template_la_OBJECTS) +libgnunet_plugin_datastore_template_la_LINK = $(LIBTOOL) $(AM_V_lt) \ + --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link \ + $(CCLD) $(AM_CFLAGS) $(CFLAGS) \ + $(libgnunet_plugin_datastore_template_la_LDFLAGS) $(LDFLAGS) \ + -o $@ +libgnunetdatastore_la_DEPENDENCIES = \ + $(top_builddir)/src/statistics/libgnunetstatistics.la \ + $(top_builddir)/src/util/libgnunetutil.la \ + $(am__DEPENDENCIES_1) +am_libgnunetdatastore_la_OBJECTS = datastore_api.lo +libgnunetdatastore_la_OBJECTS = $(am_libgnunetdatastore_la_OBJECTS) +libgnunetdatastore_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CCLD) \ + $(AM_CFLAGS) $(CFLAGS) $(libgnunetdatastore_la_LDFLAGS) \ + $(LDFLAGS) -o $@ +@HAVE_BENCHMARKS_TRUE@@HAVE_SQLITE_TRUE@am__EXEEXT_1 = perf_datastore_api_sqlite$(EXEEXT) \ +@HAVE_BENCHMARKS_TRUE@@HAVE_SQLITE_TRUE@ perf_plugin_datastore_sqlite$(EXEEXT) +@HAVE_SQLITE_TRUE@am__EXEEXT_2 = test_datastore_api_sqlite$(EXEEXT) \ +@HAVE_SQLITE_TRUE@ test_datastore_api_management_sqlite$(EXEEXT) \ +@HAVE_SQLITE_TRUE@ test_plugin_datastore_sqlite$(EXEEXT) \ +@HAVE_SQLITE_TRUE@ $(am__EXEEXT_1) +@HAVE_BENCHMARKS_TRUE@@HAVE_MYSQL_TRUE@am__EXEEXT_3 = perf_datastore_api_mysql$(EXEEXT) \ +@HAVE_BENCHMARKS_TRUE@@HAVE_MYSQL_TRUE@ perf_plugin_datastore_mysql$(EXEEXT) +@HAVE_MYSQL_TRUE@am__EXEEXT_4 = test_datastore_api_mysql$(EXEEXT) \ +@HAVE_MYSQL_TRUE@ test_datastore_api_management_mysql$(EXEEXT) \ +@HAVE_MYSQL_TRUE@ test_plugin_datastore_mysql$(EXEEXT) \ +@HAVE_MYSQL_TRUE@ $(am__EXEEXT_3) +@HAVE_BENCHMARKS_TRUE@@HAVE_POSTGRES_TRUE@am__EXEEXT_5 = perf_datastore_api_postgres$(EXEEXT) \ +@HAVE_BENCHMARKS_TRUE@@HAVE_POSTGRES_TRUE@ perf_plugin_datastore_postgres$(EXEEXT) +@HAVE_POSTGRES_TRUE@am__EXEEXT_6 = \ +@HAVE_POSTGRES_TRUE@ test_datastore_api_postgres$(EXEEXT) \ +@HAVE_POSTGRES_TRUE@ test_datastore_api_management_postgres$(EXEEXT) \ +@HAVE_POSTGRES_TRUE@ test_plugin_datastore_postgres$(EXEEXT) \ +@HAVE_POSTGRES_TRUE@ $(am__EXEEXT_5) +PROGRAMS = $(bin_PROGRAMS) +am_gnunet_service_datastore_OBJECTS = \ + gnunet-service-datastore.$(OBJEXT) +gnunet_service_datastore_OBJECTS = \ + $(am_gnunet_service_datastore_OBJECTS) +gnunet_service_datastore_DEPENDENCIES = \ + $(top_builddir)/src/statistics/libgnunetstatistics.la \ + $(top_builddir)/src/util/libgnunetutil.la \ + $(am__DEPENDENCIES_1) +am_perf_datastore_api_mysql_OBJECTS = perf_datastore_api.$(OBJEXT) +perf_datastore_api_mysql_OBJECTS = \ + $(am_perf_datastore_api_mysql_OBJECTS) +perf_datastore_api_mysql_DEPENDENCIES = \ + $(top_builddir)/src/datastore/libgnunetdatastore.la \ + $(top_builddir)/src/util/libgnunetutil.la +am_perf_datastore_api_postgres_OBJECTS = perf_datastore_api.$(OBJEXT) +perf_datastore_api_postgres_OBJECTS = \ + $(am_perf_datastore_api_postgres_OBJECTS) +perf_datastore_api_postgres_DEPENDENCIES = \ + $(top_builddir)/src/datastore/libgnunetdatastore.la \ + $(top_builddir)/src/util/libgnunetutil.la +am_perf_datastore_api_sqlite_OBJECTS = perf_datastore_api.$(OBJEXT) +perf_datastore_api_sqlite_OBJECTS = \ + $(am_perf_datastore_api_sqlite_OBJECTS) +perf_datastore_api_sqlite_DEPENDENCIES = \ + $(top_builddir)/src/datastore/libgnunetdatastore.la \ + $(top_builddir)/src/util/libgnunetutil.la +am_perf_plugin_datastore_mysql_OBJECTS = \ + perf_plugin_datastore.$(OBJEXT) +perf_plugin_datastore_mysql_OBJECTS = \ + $(am_perf_plugin_datastore_mysql_OBJECTS) +perf_plugin_datastore_mysql_DEPENDENCIES = \ + $(top_builddir)/src/util/libgnunetutil.la +am_perf_plugin_datastore_postgres_OBJECTS = \ + perf_plugin_datastore.$(OBJEXT) +perf_plugin_datastore_postgres_OBJECTS = \ + $(am_perf_plugin_datastore_postgres_OBJECTS) +perf_plugin_datastore_postgres_DEPENDENCIES = \ + $(top_builddir)/src/util/libgnunetutil.la +am_perf_plugin_datastore_sqlite_OBJECTS = \ + perf_plugin_datastore.$(OBJEXT) +perf_plugin_datastore_sqlite_OBJECTS = \ + $(am_perf_plugin_datastore_sqlite_OBJECTS) +perf_plugin_datastore_sqlite_DEPENDENCIES = \ + $(top_builddir)/src/util/libgnunetutil.la +am_test_datastore_api_management_mysql_OBJECTS = \ + test_datastore_api_management.$(OBJEXT) +test_datastore_api_management_mysql_OBJECTS = \ + $(am_test_datastore_api_management_mysql_OBJECTS) +test_datastore_api_management_mysql_DEPENDENCIES = \ + $(top_builddir)/src/datastore/libgnunetdatastore.la \ + $(top_builddir)/src/util/libgnunetutil.la +am_test_datastore_api_management_postgres_OBJECTS = \ + test_datastore_api_management.$(OBJEXT) +test_datastore_api_management_postgres_OBJECTS = \ + $(am_test_datastore_api_management_postgres_OBJECTS) +test_datastore_api_management_postgres_DEPENDENCIES = \ + $(top_builddir)/src/datastore/libgnunetdatastore.la \ + $(top_builddir)/src/util/libgnunetutil.la +am_test_datastore_api_management_sqlite_OBJECTS = \ + test_datastore_api_management.$(OBJEXT) +test_datastore_api_management_sqlite_OBJECTS = \ + $(am_test_datastore_api_management_sqlite_OBJECTS) +test_datastore_api_management_sqlite_DEPENDENCIES = \ + $(top_builddir)/src/datastore/libgnunetdatastore.la \ + $(top_builddir)/src/util/libgnunetutil.la +am_test_datastore_api_mysql_OBJECTS = test_datastore_api.$(OBJEXT) +test_datastore_api_mysql_OBJECTS = \ + $(am_test_datastore_api_mysql_OBJECTS) +test_datastore_api_mysql_DEPENDENCIES = \ + $(top_builddir)/src/datastore/libgnunetdatastore.la \ + $(top_builddir)/src/util/libgnunetutil.la +am_test_datastore_api_postgres_OBJECTS = test_datastore_api.$(OBJEXT) +test_datastore_api_postgres_OBJECTS = \ + $(am_test_datastore_api_postgres_OBJECTS) +test_datastore_api_postgres_DEPENDENCIES = \ + $(top_builddir)/src/datastore/libgnunetdatastore.la \ + $(top_builddir)/src/util/libgnunetutil.la +am_test_datastore_api_sqlite_OBJECTS = test_datastore_api.$(OBJEXT) +test_datastore_api_sqlite_OBJECTS = \ + $(am_test_datastore_api_sqlite_OBJECTS) +test_datastore_api_sqlite_DEPENDENCIES = \ + $(top_builddir)/src/datastore/libgnunetdatastore.la \ + $(top_builddir)/src/util/libgnunetutil.la +am_test_plugin_datastore_mysql_OBJECTS = \ + test_plugin_datastore.$(OBJEXT) +test_plugin_datastore_mysql_OBJECTS = \ + $(am_test_plugin_datastore_mysql_OBJECTS) +test_plugin_datastore_mysql_DEPENDENCIES = \ + $(top_builddir)/src/util/libgnunetutil.la +am_test_plugin_datastore_postgres_OBJECTS = \ + test_plugin_datastore.$(OBJEXT) +test_plugin_datastore_postgres_OBJECTS = \ + $(am_test_plugin_datastore_postgres_OBJECTS) +test_plugin_datastore_postgres_DEPENDENCIES = \ + $(top_builddir)/src/util/libgnunetutil.la +am_test_plugin_datastore_sqlite_OBJECTS = \ + test_plugin_datastore.$(OBJEXT) +test_plugin_datastore_sqlite_OBJECTS = \ + $(am_test_plugin_datastore_sqlite_OBJECTS) +test_plugin_datastore_sqlite_DEPENDENCIES = \ + $(top_builddir)/src/util/libgnunetutil.la +DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir) +depcomp = $(SHELL) $(top_srcdir)/depcomp +am__depfiles_maybe = depfiles +am__mv = mv -f +COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \ + $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) +LTCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) \ + $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \ + $(AM_CFLAGS) $(CFLAGS) +AM_V_CC = $(am__v_CC_$(V)) +am__v_CC_ = $(am__v_CC_$(AM_DEFAULT_VERBOSITY)) +am__v_CC_0 = @echo " CC " $@; +AM_V_at = $(am__v_at_$(V)) +am__v_at_ = $(am__v_at_$(AM_DEFAULT_VERBOSITY)) +am__v_at_0 = @ +CCLD = $(CC) +LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \ + $(AM_LDFLAGS) $(LDFLAGS) -o $@ +AM_V_CCLD = $(am__v_CCLD_$(V)) +am__v_CCLD_ = $(am__v_CCLD_$(AM_DEFAULT_VERBOSITY)) +am__v_CCLD_0 = @echo " CCLD " $@; +AM_V_GEN = $(am__v_GEN_$(V)) +am__v_GEN_ = $(am__v_GEN_$(AM_DEFAULT_VERBOSITY)) +am__v_GEN_0 = @echo " GEN " $@; +SOURCES = $(libgnunet_plugin_datastore_mysql_la_SOURCES) \ + $(libgnunet_plugin_datastore_postgres_la_SOURCES) \ + $(libgnunet_plugin_datastore_sqlite_la_SOURCES) \ + $(libgnunet_plugin_datastore_template_la_SOURCES) \ + $(libgnunetdatastore_la_SOURCES) \ + $(gnunet_service_datastore_SOURCES) \ + $(perf_datastore_api_mysql_SOURCES) \ + $(perf_datastore_api_postgres_SOURCES) \ + $(perf_datastore_api_sqlite_SOURCES) \ + $(perf_plugin_datastore_mysql_SOURCES) \ + $(perf_plugin_datastore_postgres_SOURCES) \ + $(perf_plugin_datastore_sqlite_SOURCES) \ + $(test_datastore_api_management_mysql_SOURCES) \ + $(test_datastore_api_management_postgres_SOURCES) \ + $(test_datastore_api_management_sqlite_SOURCES) \ + $(test_datastore_api_mysql_SOURCES) \ + $(test_datastore_api_postgres_SOURCES) \ + $(test_datastore_api_sqlite_SOURCES) \ + $(test_plugin_datastore_mysql_SOURCES) \ + $(test_plugin_datastore_postgres_SOURCES) \ + $(test_plugin_datastore_sqlite_SOURCES) +DIST_SOURCES = $(libgnunet_plugin_datastore_mysql_la_SOURCES) \ + $(libgnunet_plugin_datastore_postgres_la_SOURCES) \ + $(libgnunet_plugin_datastore_sqlite_la_SOURCES) \ + $(libgnunet_plugin_datastore_template_la_SOURCES) \ + $(libgnunetdatastore_la_SOURCES) \ + $(gnunet_service_datastore_SOURCES) \ + $(perf_datastore_api_mysql_SOURCES) \ + $(perf_datastore_api_postgres_SOURCES) \ + $(perf_datastore_api_sqlite_SOURCES) \ + $(perf_plugin_datastore_mysql_SOURCES) \ + $(perf_plugin_datastore_postgres_SOURCES) \ + $(perf_plugin_datastore_sqlite_SOURCES) \ + $(test_datastore_api_management_mysql_SOURCES) \ + $(test_datastore_api_management_postgres_SOURCES) \ + $(test_datastore_api_management_sqlite_SOURCES) \ + $(test_datastore_api_mysql_SOURCES) \ + $(test_datastore_api_postgres_SOURCES) \ + $(test_datastore_api_sqlite_SOURCES) \ + $(test_plugin_datastore_mysql_SOURCES) \ + $(test_plugin_datastore_postgres_SOURCES) \ + $(test_plugin_datastore_sqlite_SOURCES) +DATA = $(pkgcfg_DATA) +ETAGS = etags +CTAGS = ctags +am__tty_colors = \ +red=; grn=; lgn=; blu=; std= +DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) +ACLOCAL = @ACLOCAL@ +AMTAR = @AMTAR@ +AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@ +AR = @AR@ +ARGZ_H = @ARGZ_H@ +AS = @AS@ +AUTOCONF = @AUTOCONF@ +AUTOHEADER = @AUTOHEADER@ +AUTOMAKE = @AUTOMAKE@ +AWK = @AWK@ +CC = @CC@ +CCDEPMODE = @CCDEPMODE@ +CFLAGS = @CFLAGS@ +CPP = @CPP@ +CPPFLAGS = @CPPFLAGS@ +CXX = @CXX@ +CXXCPP = @CXXCPP@ +CXXDEPMODE = @CXXDEPMODE@ +CXXFLAGS = @CXXFLAGS@ +CYGPATH_W = @CYGPATH_W@ +DEFAULT_INTERFACE = @DEFAULT_INTERFACE@ +DEFS = @DEFS@ +DEPDIR = @DEPDIR@ +DLLDIR = @DLLDIR@ +DLLTOOL = @DLLTOOL@ +DSYMUTIL = @DSYMUTIL@ +DUMPBIN = @DUMPBIN@ +ECHO_C = @ECHO_C@ +ECHO_N = @ECHO_N@ +ECHO_T = @ECHO_T@ +EGREP = @EGREP@ +EXEEXT = @EXEEXT@ +EXT_LIBS = @EXT_LIBS@ +EXT_LIB_PATH = @EXT_LIB_PATH@ +FGREP = @FGREP@ +GMSGFMT = @GMSGFMT@ +GMSGFMT_015 = @GMSGFMT_015@ +GNUNETDNS_GROUP = @GNUNETDNS_GROUP@ +GN_DAEMON_CONFIG_DIR = @GN_DAEMON_CONFIG_DIR@ +GN_DAEMON_HOME_DIR = @GN_DAEMON_HOME_DIR@ +GN_INTLINCL = @GN_INTLINCL@ +GN_LIBINTL = @GN_LIBINTL@ +GN_LIB_LDFLAGS = @GN_LIB_LDFLAGS@ +GN_PLUGIN_LDFLAGS = @GN_PLUGIN_LDFLAGS@ +GN_USER_HOME_DIR = @GN_USER_HOME_DIR@ +GREP = @GREP@ +HAVE_LIBUNISTRING = @HAVE_LIBUNISTRING@ +INCLTDL = @INCLTDL@ +INSTALL = @INSTALL@ +INSTALL_DATA = @INSTALL_DATA@ +INSTALL_PROGRAM = @INSTALL_PROGRAM@ +INSTALL_SCRIPT = @INSTALL_SCRIPT@ +INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@ +INTLLIBS = @INTLLIBS@ +INTL_MACOSX_LIBS = @INTL_MACOSX_LIBS@ +LD = @LD@ +LDFLAGS = @LDFLAGS@ +LIBADD_DL = @LIBADD_DL@ +LIBADD_DLD_LINK = @LIBADD_DLD_LINK@ +LIBADD_DLOPEN = @LIBADD_DLOPEN@ +LIBADD_SHL_LOAD = @LIBADD_SHL_LOAD@ +LIBCURL = @LIBCURL@ +LIBCURL_CPPFLAGS = @LIBCURL_CPPFLAGS@ +LIBGCRYPT_CFLAGS = @LIBGCRYPT_CFLAGS@ +LIBGCRYPT_CONFIG = @LIBGCRYPT_CONFIG@ +LIBGCRYPT_LIBS = @LIBGCRYPT_LIBS@ +LIBICONV = @LIBICONV@ +LIBINTL = @LIBINTL@ +LIBLTDL = @LIBLTDL@ +LIBOBJS = @LIBOBJS@ +LIBPREFIX = @LIBPREFIX@ +LIBS = @LIBS@ +LIBTOOL = @LIBTOOL@ +LIBUNISTRING = @LIBUNISTRING@ +LIPO = @LIPO@ +LN_S = @LN_S@ +LTDLDEPS = @LTDLDEPS@ +LTDLINCL = @LTDLINCL@ +LTDLOPEN = @LTDLOPEN@ +LTLIBICONV = @LTLIBICONV@ +LTLIBINTL = @LTLIBINTL@ +LTLIBOBJS = @LTLIBOBJS@ +LTLIBUNISTRING = @LTLIBUNISTRING@ +LT_CONFIG_H = @LT_CONFIG_H@ +LT_DLLOADERS = @LT_DLLOADERS@ +LT_DLPREOPEN = @LT_DLPREOPEN@ +MAKEINFO = @MAKEINFO@ +MKDIR_P = @MKDIR_P@ +MSGFMT = @MSGFMT@ +MSGFMT_015 = @MSGFMT_015@ +MSGMERGE = @MSGMERGE@ +MYSQL_CPPFLAGS = @MYSQL_CPPFLAGS@ +MYSQL_LDFLAGS = @MYSQL_LDFLAGS@ +NM = @NM@ +NMEDIT = @NMEDIT@ +OBJC = @OBJC@ +OBJCDEPMODE = @OBJCDEPMODE@ +OBJCFLAGS = @OBJCFLAGS@ +OBJDUMP = @OBJDUMP@ +OBJEXT = @OBJEXT@ +OTOOL = @OTOOL@ +OTOOL64 = @OTOOL64@ +PACKAGE = @PACKAGE@ +PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@ +PACKAGE_NAME = @PACKAGE_NAME@ +PACKAGE_STRING = @PACKAGE_STRING@ +PACKAGE_TARNAME = @PACKAGE_TARNAME@ +PACKAGE_URL = @PACKAGE_URL@ +PACKAGE_VERSION = @PACKAGE_VERSION@ +PATH_SEPARATOR = @PATH_SEPARATOR@ +POSTGRES_CPPFLAGS = @POSTGRES_CPPFLAGS@ +POSTGRES_LDFLAGS = @POSTGRES_LDFLAGS@ +POSUB = @POSUB@ +PYTHON = @PYTHON@ +PYTHON_EXEC_PREFIX = @PYTHON_EXEC_PREFIX@ +PYTHON_PLATFORM = @PYTHON_PLATFORM@ +PYTHON_PREFIX = @PYTHON_PREFIX@ +PYTHON_VERSION = @PYTHON_VERSION@ +RANLIB = @RANLIB@ +SED = @SED@ +SET_MAKE = @SET_MAKE@ +SHELL = @SHELL@ +SQLITE_CPPFLAGS = @SQLITE_CPPFLAGS@ +SQLITE_LDFLAGS = @SQLITE_LDFLAGS@ +STRIP = @STRIP@ +SUDO_BINARY = @SUDO_BINARY@ +UNIXONLY = @UNIXONLY@ +USE_NLS = @USE_NLS@ +VERSION = @VERSION@ +XGETTEXT = @XGETTEXT@ +XGETTEXT_015 = @XGETTEXT_015@ +XMKMF = @XMKMF@ +X_CFLAGS = @X_CFLAGS@ +X_EXTRA_LIBS = @X_EXTRA_LIBS@ +X_LIBS = @X_LIBS@ +X_PRE_LIBS = @X_PRE_LIBS@ +_libcurl_config = @_libcurl_config@ +abs_builddir = @abs_builddir@ +abs_srcdir = @abs_srcdir@ +abs_top_builddir = @abs_top_builddir@ +abs_top_srcdir = @abs_top_srcdir@ +ac_ct_CC = @ac_ct_CC@ +ac_ct_CXX = @ac_ct_CXX@ +ac_ct_DUMPBIN = @ac_ct_DUMPBIN@ +ac_ct_OBJC = @ac_ct_OBJC@ +am__include = @am__include@ +am__leading_dot = @am__leading_dot@ +am__quote = @am__quote@ +am__tar = @am__tar@ +am__untar = @am__untar@ +bindir = @bindir@ +build = @build@ +build_alias = @build_alias@ +build_cpu = @build_cpu@ +build_os = @build_os@ +build_target = @build_target@ +build_vendor = @build_vendor@ +builddir = @builddir@ +datadir = @datadir@ +datarootdir = @datarootdir@ +docdir = @docdir@ +dvidir = @dvidir@ +exec_prefix = @exec_prefix@ +host = @host@ +host_alias = @host_alias@ +host_cpu = @host_cpu@ +host_os = @host_os@ +host_vendor = @host_vendor@ +htmldir = @htmldir@ +includedir = @includedir@ +infodir = @infodir@ +install_sh = @install_sh@ +libdir = @libdir@ +libexecdir = @libexecdir@ +localedir = @localedir@ +localstatedir = @localstatedir@ +lt_ECHO = @lt_ECHO@ +ltdl_LIBOBJS = @ltdl_LIBOBJS@ +ltdl_LTLIBOBJS = @ltdl_LTLIBOBJS@ +mandir = @mandir@ +mkdir_p = @mkdir_p@ +oldincludedir = @oldincludedir@ +pdfdir = @pdfdir@ +pkgpyexecdir = @pkgpyexecdir@ +pkgpythondir = @pkgpythondir@ +prefix = @prefix@ +program_transform_name = @program_transform_name@ +psdir = @psdir@ +pyexecdir = @pyexecdir@ +pythondir = @pythondir@ +sbindir = @sbindir@ +sharedstatedir = @sharedstatedir@ +srcdir = @srcdir@ +subdirs = @subdirs@ +sys_symbol_underscore = @sys_symbol_underscore@ +sysconfdir = @sysconfdir@ +target = @target@ +target_alias = @target_alias@ +target_cpu = @target_cpu@ +target_os = @target_os@ +target_vendor = @target_vendor@ +top_build_prefix = @top_build_prefix@ +top_builddir = @top_builddir@ +top_srcdir = @top_srcdir@ +INCLUDES = -I$(top_srcdir)/src/include +plugindir = $(libdir)/gnunet +pkgcfgdir = $(pkgdatadir)/config.d/ +pkgcfg_DATA = \ + datastore.conf + +@MINGW_TRUE@WINFLAGS = -Wl,--no-undefined -Wl,--export-all-symbols +@USE_COVERAGE_TRUE@AM_CFLAGS = --coverage -O0 +@USE_COVERAGE_TRUE@XLIBS = -lgcov +lib_LTLIBRARIES = \ + libgnunetdatastore.la + +libgnunetdatastore_la_SOURCES = \ + datastore_api.c datastore.h + +libgnunetdatastore_la_LIBADD = \ + $(top_builddir)/src/statistics/libgnunetstatistics.la \ + $(top_builddir)/src/util/libgnunetutil.la \ + $(GN_LIBINTL) + +libgnunetdatastore_la_LDFLAGS = \ + $(GN_LIB_LDFLAGS) $(WINFLAGS) \ + -version-info 1:0:0 + +gnunet_service_datastore_SOURCES = \ + gnunet-service-datastore.c + +gnunet_service_datastore_LDADD = \ + $(top_builddir)/src/statistics/libgnunetstatistics.la \ + $(top_builddir)/src/util/libgnunetutil.la \ + $(GN_LIBINTL) + +@HAVE_MYSQL_TRUE@MYSQL_PLUGIN = libgnunet_plugin_datastore_mysql.la +@HAVE_BENCHMARKS_TRUE@@HAVE_MYSQL_TRUE@MYSQL_BENCHMARKS = \ +@HAVE_BENCHMARKS_TRUE@@HAVE_MYSQL_TRUE@ perf_datastore_api_mysql \ +@HAVE_BENCHMARKS_TRUE@@HAVE_MYSQL_TRUE@ perf_plugin_datastore_mysql + +@HAVE_MYSQL_TRUE@MYSQL_TESTS = \ +@HAVE_MYSQL_TRUE@ test_datastore_api_mysql \ +@HAVE_MYSQL_TRUE@ test_datastore_api_management_mysql \ +@HAVE_MYSQL_TRUE@ test_plugin_datastore_mysql \ +@HAVE_MYSQL_TRUE@ $(MYSQL_BENCHMARKS) + +@HAVE_SQLITE_TRUE@SQLITE_PLUGIN = libgnunet_plugin_datastore_sqlite.la +@HAVE_BENCHMARKS_TRUE@@HAVE_SQLITE_TRUE@SQLITE_BENCHMARKS = \ +@HAVE_BENCHMARKS_TRUE@@HAVE_SQLITE_TRUE@ perf_datastore_api_sqlite \ +@HAVE_BENCHMARKS_TRUE@@HAVE_SQLITE_TRUE@ perf_plugin_datastore_sqlite + +@HAVE_SQLITE_TRUE@SQLITE_TESTS = \ +@HAVE_SQLITE_TRUE@ test_datastore_api_sqlite \ +@HAVE_SQLITE_TRUE@ test_datastore_api_management_sqlite \ +@HAVE_SQLITE_TRUE@ test_plugin_datastore_sqlite \ +@HAVE_SQLITE_TRUE@ $(SQLITE_BENCHMARKS) + +@HAVE_POSTGRES_TRUE@POSTGRES_PLUGIN = libgnunet_plugin_datastore_postgres.la +@HAVE_BENCHMARKS_TRUE@@HAVE_POSTGRES_TRUE@POSTGRES_BENCHMARKS = \ +@HAVE_BENCHMARKS_TRUE@@HAVE_POSTGRES_TRUE@ perf_datastore_api_postgres \ +@HAVE_BENCHMARKS_TRUE@@HAVE_POSTGRES_TRUE@ perf_plugin_datastore_postgres + +@HAVE_POSTGRES_TRUE@POSTGRES_TESTS = \ +@HAVE_POSTGRES_TRUE@ test_datastore_api_postgres \ +@HAVE_POSTGRES_TRUE@ test_datastore_api_management_postgres \ +@HAVE_POSTGRES_TRUE@ test_plugin_datastore_postgres \ +@HAVE_POSTGRES_TRUE@ $(POSTGRES_BENCHMARKS) + +plugin_LTLIBRARIES = \ + $(SQLITE_PLUGIN) \ + $(MYSQL_PLUGIN) \ + $(POSTGRES_PLUGIN) \ + libgnunet_plugin_datastore_template.la + +libgnunet_plugin_datastore_sqlite_la_SOURCES = \ + plugin_datastore_sqlite.c + +libgnunet_plugin_datastore_sqlite_la_LIBADD = \ + $(top_builddir)/src/statistics/libgnunetstatistics.la \ + $(top_builddir)/src/util/libgnunetutil.la $(XLIBS) -lsqlite3 + +libgnunet_plugin_datastore_sqlite_la_LDFLAGS = \ + $(GN_PLUGIN_LDFLAGS) + +libgnunet_plugin_datastore_mysql_la_SOURCES = \ + plugin_datastore_mysql.c + +libgnunet_plugin_datastore_mysql_la_LIBADD = \ + $(top_builddir)/src/statistics/libgnunetstatistics.la \ + $(top_builddir)/src/util/libgnunetutil.la $(XLIBS) -lz -lmysqlclient + +libgnunet_plugin_datastore_mysql_la_LDFLAGS = \ + $(GN_PLUGIN_LDFLAGS) $(MYSQL_LDFLAGS) -lmysqlclient + +libgnunet_plugin_datastore_mysql_la_CPPFLAGS = \ + $(MYSQL_CPPFLAGS) + +libgnunet_plugin_datastore_postgres_la_SOURCES = \ + plugin_datastore_postgres.c + +libgnunet_plugin_datastore_postgres_la_LIBADD = \ + $(top_builddir)/src/statistics/libgnunetstatistics.la \ + $(top_builddir)/src/util/libgnunetutil.la $(XLIBS) -lpq + +libgnunet_plugin_datastore_postgres_la_LDFLAGS = \ + $(GN_PLUGIN_LDFLAGS) $(POSTGRES_LDFLAGS) -lpq + +libgnunet_plugin_datastore_postgres_la_CPPFLAGS = \ + $(POSTGRES_CPPFLAGS) + +libgnunet_plugin_datastore_template_la_SOURCES = \ + plugin_datastore_template.c + +libgnunet_plugin_datastore_template_la_LIBADD = \ + $(top_builddir)/src/util/libgnunetutil.la $(XLIBS) + +libgnunet_plugin_datastore_template_la_LDFLAGS = \ + $(GN_PLUGIN_LDFLAGS) + +@ENABLE_TEST_RUN_TRUE@TESTS = $(check_PROGRAMS) +test_datastore_api_sqlite_SOURCES = \ + test_datastore_api.c + +test_datastore_api_sqlite_LDADD = \ + $(top_builddir)/src/datastore/libgnunetdatastore.la \ + $(top_builddir)/src/util/libgnunetutil.la + +test_datastore_api_management_sqlite_SOURCES = \ + test_datastore_api_management.c + +test_datastore_api_management_sqlite_LDADD = \ + $(top_builddir)/src/datastore/libgnunetdatastore.la \ + $(top_builddir)/src/util/libgnunetutil.la + +perf_datastore_api_sqlite_SOURCES = \ + perf_datastore_api.c + +perf_datastore_api_sqlite_LDADD = \ + $(top_builddir)/src/datastore/libgnunetdatastore.la \ + $(top_builddir)/src/util/libgnunetutil.la + +perf_plugin_datastore_sqlite_SOURCES = \ + perf_plugin_datastore.c + +perf_plugin_datastore_sqlite_LDADD = \ + $(top_builddir)/src/util/libgnunetutil.la + +test_plugin_datastore_sqlite_SOURCES = \ + test_plugin_datastore.c + +test_plugin_datastore_sqlite_LDADD = \ + $(top_builddir)/src/util/libgnunetutil.la + +test_datastore_api_mysql_SOURCES = \ + test_datastore_api.c + +test_datastore_api_mysql_LDADD = \ + $(top_builddir)/src/datastore/libgnunetdatastore.la \ + $(top_builddir)/src/util/libgnunetutil.la + +test_datastore_api_management_mysql_SOURCES = \ + test_datastore_api_management.c + +test_datastore_api_management_mysql_LDADD = \ + $(top_builddir)/src/datastore/libgnunetdatastore.la \ + $(top_builddir)/src/util/libgnunetutil.la + +perf_datastore_api_mysql_SOURCES = \ + perf_datastore_api.c + +perf_datastore_api_mysql_LDADD = \ + $(top_builddir)/src/datastore/libgnunetdatastore.la \ + $(top_builddir)/src/util/libgnunetutil.la + +test_plugin_datastore_mysql_SOURCES = \ + test_plugin_datastore.c + +test_plugin_datastore_mysql_LDADD = \ + $(top_builddir)/src/util/libgnunetutil.la + +perf_plugin_datastore_mysql_SOURCES = \ + perf_plugin_datastore.c + +perf_plugin_datastore_mysql_LDADD = \ + $(top_builddir)/src/util/libgnunetutil.la + +test_datastore_api_postgres_SOURCES = \ + test_datastore_api.c + +test_datastore_api_postgres_LDADD = \ + $(top_builddir)/src/datastore/libgnunetdatastore.la \ + $(top_builddir)/src/util/libgnunetutil.la + +test_datastore_api_management_postgres_SOURCES = \ + test_datastore_api_management.c + +test_datastore_api_management_postgres_LDADD = \ + $(top_builddir)/src/datastore/libgnunetdatastore.la \ + $(top_builddir)/src/util/libgnunetutil.la + +perf_datastore_api_postgres_SOURCES = \ + perf_datastore_api.c + +perf_datastore_api_postgres_LDADD = \ + $(top_builddir)/src/datastore/libgnunetdatastore.la \ + $(top_builddir)/src/util/libgnunetutil.la + +test_plugin_datastore_postgres_SOURCES = \ + test_plugin_datastore.c + +test_plugin_datastore_postgres_LDADD = \ + $(top_builddir)/src/util/libgnunetutil.la + +perf_plugin_datastore_postgres_SOURCES = \ + perf_plugin_datastore.c + +perf_plugin_datastore_postgres_LDADD = \ + $(top_builddir)/src/util/libgnunetutil.la + +EXTRA_DIST = \ + test_defaults.conf \ + test_datastore_api_data_sqlite.conf \ + perf_plugin_datastore_data_sqlite.conf \ + test_datastore_api_data_mysql.conf \ + perf_plugin_datastore_data_mysql.conf \ + test_datastore_api_data_postgres.conf \ + perf_plugin_datastore_data_postgres.conf \ + test_plugin_datastore_data_mysql.conf \ + test_plugin_datastore_data_postgres.conf \ + test_plugin_datastore_data_sqlite.conf + +all: all-am + +.SUFFIXES: +.SUFFIXES: .c .lo .o .obj +$(srcdir)/Makefile.in: $(srcdir)/Makefile.am $(am__configure_deps) + @for dep in $?; do \ + case '$(am__configure_deps)' in \ + *$$dep*) \ + ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \ + && { if test -f $@; then exit 0; else break; fi; }; \ + exit 1;; \ + esac; \ + done; \ + echo ' cd $(top_srcdir) && $(AUTOMAKE) --gnu src/datastore/Makefile'; \ + $(am__cd) $(top_srcdir) && \ + $(AUTOMAKE) --gnu src/datastore/Makefile +.PRECIOUS: Makefile +Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status + @case '$?' in \ + *config.status*) \ + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \ + *) \ + echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe)'; \ + cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe);; \ + esac; + +$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh + +$(top_srcdir)/configure: $(am__configure_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(ACLOCAL_M4): $(am__aclocal_m4_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(am__aclocal_m4_deps): +datastore.conf: $(top_builddir)/config.status $(srcdir)/datastore.conf.in + cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ +install-libLTLIBRARIES: $(lib_LTLIBRARIES) + @$(NORMAL_INSTALL) + test -z "$(libdir)" || $(MKDIR_P) "$(DESTDIR)$(libdir)" + @list='$(lib_LTLIBRARIES)'; test -n "$(libdir)" || list=; \ + list2=; for p in $$list; do \ + if test -f $$p; then \ + list2="$$list2 $$p"; \ + else :; fi; \ + done; \ + test -z "$$list2" || { \ + echo " $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL) $(INSTALL_STRIP_FLAG) $$list2 '$(DESTDIR)$(libdir)'"; \ + $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL) $(INSTALL_STRIP_FLAG) $$list2 "$(DESTDIR)$(libdir)"; \ + } + +uninstall-libLTLIBRARIES: + @$(NORMAL_UNINSTALL) + @list='$(lib_LTLIBRARIES)'; test -n "$(libdir)" || list=; \ + for p in $$list; do \ + $(am__strip_dir) \ + echo " $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=uninstall rm -f '$(DESTDIR)$(libdir)/$$f'"; \ + $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=uninstall rm -f "$(DESTDIR)$(libdir)/$$f"; \ + done + +clean-libLTLIBRARIES: + -test -z "$(lib_LTLIBRARIES)" || rm -f $(lib_LTLIBRARIES) + @list='$(lib_LTLIBRARIES)'; for p in $$list; do \ + dir="`echo $$p | sed -e 's|/[^/]*$$||'`"; \ + test "$$dir" != "$$p" || dir=.; \ + echo "rm -f \"$${dir}/so_locations\""; \ + rm -f "$${dir}/so_locations"; \ + done +install-pluginLTLIBRARIES: $(plugin_LTLIBRARIES) + @$(NORMAL_INSTALL) + test -z "$(plugindir)" || $(MKDIR_P) "$(DESTDIR)$(plugindir)" + @list='$(plugin_LTLIBRARIES)'; test -n "$(plugindir)" || list=; \ + list2=; for p in $$list; do \ + if test -f $$p; then \ + list2="$$list2 $$p"; \ + else :; fi; \ + done; \ + test -z "$$list2" || { \ + echo " $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL) $(INSTALL_STRIP_FLAG) $$list2 '$(DESTDIR)$(plugindir)'"; \ + $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL) $(INSTALL_STRIP_FLAG) $$list2 "$(DESTDIR)$(plugindir)"; \ + } + +uninstall-pluginLTLIBRARIES: + @$(NORMAL_UNINSTALL) + @list='$(plugin_LTLIBRARIES)'; test -n "$(plugindir)" || list=; \ + for p in $$list; do \ + $(am__strip_dir) \ + echo " $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=uninstall rm -f '$(DESTDIR)$(plugindir)/$$f'"; \ + $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=uninstall rm -f "$(DESTDIR)$(plugindir)/$$f"; \ + done + +clean-pluginLTLIBRARIES: + -test -z "$(plugin_LTLIBRARIES)" || rm -f $(plugin_LTLIBRARIES) + @list='$(plugin_LTLIBRARIES)'; for p in $$list; do \ + dir="`echo $$p | sed -e 's|/[^/]*$$||'`"; \ + test "$$dir" != "$$p" || dir=.; \ + echo "rm -f \"$${dir}/so_locations\""; \ + rm -f "$${dir}/so_locations"; \ + done +libgnunet_plugin_datastore_mysql.la: $(libgnunet_plugin_datastore_mysql_la_OBJECTS) $(libgnunet_plugin_datastore_mysql_la_DEPENDENCIES) + $(AM_V_CCLD)$(libgnunet_plugin_datastore_mysql_la_LINK) $(am_libgnunet_plugin_datastore_mysql_la_rpath) $(libgnunet_plugin_datastore_mysql_la_OBJECTS) $(libgnunet_plugin_datastore_mysql_la_LIBADD) $(LIBS) +libgnunet_plugin_datastore_postgres.la: $(libgnunet_plugin_datastore_postgres_la_OBJECTS) $(libgnunet_plugin_datastore_postgres_la_DEPENDENCIES) + $(AM_V_CCLD)$(libgnunet_plugin_datastore_postgres_la_LINK) $(am_libgnunet_plugin_datastore_postgres_la_rpath) $(libgnunet_plugin_datastore_postgres_la_OBJECTS) $(libgnunet_plugin_datastore_postgres_la_LIBADD) $(LIBS) +libgnunet_plugin_datastore_sqlite.la: $(libgnunet_plugin_datastore_sqlite_la_OBJECTS) $(libgnunet_plugin_datastore_sqlite_la_DEPENDENCIES) + $(AM_V_CCLD)$(libgnunet_plugin_datastore_sqlite_la_LINK) $(am_libgnunet_plugin_datastore_sqlite_la_rpath) $(libgnunet_plugin_datastore_sqlite_la_OBJECTS) $(libgnunet_plugin_datastore_sqlite_la_LIBADD) $(LIBS) +libgnunet_plugin_datastore_template.la: $(libgnunet_plugin_datastore_template_la_OBJECTS) $(libgnunet_plugin_datastore_template_la_DEPENDENCIES) + $(AM_V_CCLD)$(libgnunet_plugin_datastore_template_la_LINK) -rpath $(plugindir) $(libgnunet_plugin_datastore_template_la_OBJECTS) $(libgnunet_plugin_datastore_template_la_LIBADD) $(LIBS) +libgnunetdatastore.la: $(libgnunetdatastore_la_OBJECTS) $(libgnunetdatastore_la_DEPENDENCIES) + $(AM_V_CCLD)$(libgnunetdatastore_la_LINK) -rpath $(libdir) $(libgnunetdatastore_la_OBJECTS) $(libgnunetdatastore_la_LIBADD) $(LIBS) +install-binPROGRAMS: $(bin_PROGRAMS) + @$(NORMAL_INSTALL) + test -z "$(bindir)" || $(MKDIR_P) "$(DESTDIR)$(bindir)" + @list='$(bin_PROGRAMS)'; test -n "$(bindir)" || list=; \ + for p in $$list; do echo "$$p $$p"; done | \ + sed 's/$(EXEEXT)$$//' | \ + while read p p1; do if test -f $$p || test -f $$p1; \ + then echo "$$p"; echo "$$p"; else :; fi; \ + done | \ + sed -e 'p;s,.*/,,;n;h' -e 's|.*|.|' \ + -e 'p;x;s,.*/,,;s/$(EXEEXT)$$//;$(transform);s/$$/$(EXEEXT)/' | \ + sed 'N;N;N;s,\n, ,g' | \ + $(AWK) 'BEGIN { files["."] = ""; dirs["."] = 1 } \ + { d=$$3; if (dirs[d] != 1) { print "d", d; dirs[d] = 1 } \ + if ($$2 == $$4) files[d] = files[d] " " $$1; \ + else { print "f", $$3 "/" $$4, $$1; } } \ + END { for (d in files) print "f", d, files[d] }' | \ + while read type dir files; do \ + if test "$$dir" = .; then dir=; else dir=/$$dir; fi; \ + test -z "$$files" || { \ + echo " $(INSTALL_PROGRAM_ENV) $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL_PROGRAM) $$files '$(DESTDIR)$(bindir)$$dir'"; \ + $(INSTALL_PROGRAM_ENV) $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL_PROGRAM) $$files "$(DESTDIR)$(bindir)$$dir" || exit $$?; \ + } \ + ; done + +uninstall-binPROGRAMS: + @$(NORMAL_UNINSTALL) + @list='$(bin_PROGRAMS)'; test -n "$(bindir)" || list=; \ + files=`for p in $$list; do echo "$$p"; done | \ + sed -e 'h;s,^.*/,,;s/$(EXEEXT)$$//;$(transform)' \ + -e 's/$$/$(EXEEXT)/' `; \ + test -n "$$list" || exit 0; \ + echo " ( cd '$(DESTDIR)$(bindir)' && rm -f" $$files ")"; \ + cd "$(DESTDIR)$(bindir)" && rm -f $$files + +clean-binPROGRAMS: + @list='$(bin_PROGRAMS)'; test -n "$$list" || exit 0; \ + echo " rm -f" $$list; \ + rm -f $$list || exit $$?; \ + test -n "$(EXEEXT)" || exit 0; \ + list=`for p in $$list; do echo "$$p"; done | sed 's/$(EXEEXT)$$//'`; \ + echo " rm -f" $$list; \ + rm -f $$list + +clean-checkPROGRAMS: + @list='$(check_PROGRAMS)'; test -n "$$list" || exit 0; \ + echo " rm -f" $$list; \ + rm -f $$list || exit $$?; \ + test -n "$(EXEEXT)" || exit 0; \ + list=`for p in $$list; do echo "$$p"; done | sed 's/$(EXEEXT)$$//'`; \ + echo " rm -f" $$list; \ + rm -f $$list +gnunet-service-datastore$(EXEEXT): $(gnunet_service_datastore_OBJECTS) $(gnunet_service_datastore_DEPENDENCIES) + @rm -f gnunet-service-datastore$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(gnunet_service_datastore_OBJECTS) $(gnunet_service_datastore_LDADD) $(LIBS) +perf_datastore_api_mysql$(EXEEXT): $(perf_datastore_api_mysql_OBJECTS) $(perf_datastore_api_mysql_DEPENDENCIES) + @rm -f perf_datastore_api_mysql$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(perf_datastore_api_mysql_OBJECTS) $(perf_datastore_api_mysql_LDADD) $(LIBS) +perf_datastore_api_postgres$(EXEEXT): $(perf_datastore_api_postgres_OBJECTS) $(perf_datastore_api_postgres_DEPENDENCIES) + @rm -f perf_datastore_api_postgres$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(perf_datastore_api_postgres_OBJECTS) $(perf_datastore_api_postgres_LDADD) $(LIBS) +perf_datastore_api_sqlite$(EXEEXT): $(perf_datastore_api_sqlite_OBJECTS) $(perf_datastore_api_sqlite_DEPENDENCIES) + @rm -f perf_datastore_api_sqlite$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(perf_datastore_api_sqlite_OBJECTS) $(perf_datastore_api_sqlite_LDADD) $(LIBS) +perf_plugin_datastore_mysql$(EXEEXT): $(perf_plugin_datastore_mysql_OBJECTS) $(perf_plugin_datastore_mysql_DEPENDENCIES) + @rm -f perf_plugin_datastore_mysql$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(perf_plugin_datastore_mysql_OBJECTS) $(perf_plugin_datastore_mysql_LDADD) $(LIBS) +perf_plugin_datastore_postgres$(EXEEXT): $(perf_plugin_datastore_postgres_OBJECTS) $(perf_plugin_datastore_postgres_DEPENDENCIES) + @rm -f perf_plugin_datastore_postgres$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(perf_plugin_datastore_postgres_OBJECTS) $(perf_plugin_datastore_postgres_LDADD) $(LIBS) +perf_plugin_datastore_sqlite$(EXEEXT): $(perf_plugin_datastore_sqlite_OBJECTS) $(perf_plugin_datastore_sqlite_DEPENDENCIES) + @rm -f perf_plugin_datastore_sqlite$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(perf_plugin_datastore_sqlite_OBJECTS) $(perf_plugin_datastore_sqlite_LDADD) $(LIBS) +test_datastore_api_management_mysql$(EXEEXT): $(test_datastore_api_management_mysql_OBJECTS) $(test_datastore_api_management_mysql_DEPENDENCIES) + @rm -f test_datastore_api_management_mysql$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_datastore_api_management_mysql_OBJECTS) $(test_datastore_api_management_mysql_LDADD) $(LIBS) +test_datastore_api_management_postgres$(EXEEXT): $(test_datastore_api_management_postgres_OBJECTS) $(test_datastore_api_management_postgres_DEPENDENCIES) + @rm -f test_datastore_api_management_postgres$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_datastore_api_management_postgres_OBJECTS) $(test_datastore_api_management_postgres_LDADD) $(LIBS) +test_datastore_api_management_sqlite$(EXEEXT): $(test_datastore_api_management_sqlite_OBJECTS) $(test_datastore_api_management_sqlite_DEPENDENCIES) + @rm -f test_datastore_api_management_sqlite$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_datastore_api_management_sqlite_OBJECTS) $(test_datastore_api_management_sqlite_LDADD) $(LIBS) +test_datastore_api_mysql$(EXEEXT): $(test_datastore_api_mysql_OBJECTS) $(test_datastore_api_mysql_DEPENDENCIES) + @rm -f test_datastore_api_mysql$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_datastore_api_mysql_OBJECTS) $(test_datastore_api_mysql_LDADD) $(LIBS) +test_datastore_api_postgres$(EXEEXT): $(test_datastore_api_postgres_OBJECTS) $(test_datastore_api_postgres_DEPENDENCIES) + @rm -f test_datastore_api_postgres$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_datastore_api_postgres_OBJECTS) $(test_datastore_api_postgres_LDADD) $(LIBS) +test_datastore_api_sqlite$(EXEEXT): $(test_datastore_api_sqlite_OBJECTS) $(test_datastore_api_sqlite_DEPENDENCIES) + @rm -f test_datastore_api_sqlite$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_datastore_api_sqlite_OBJECTS) $(test_datastore_api_sqlite_LDADD) $(LIBS) +test_plugin_datastore_mysql$(EXEEXT): $(test_plugin_datastore_mysql_OBJECTS) $(test_plugin_datastore_mysql_DEPENDENCIES) + @rm -f test_plugin_datastore_mysql$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_plugin_datastore_mysql_OBJECTS) $(test_plugin_datastore_mysql_LDADD) $(LIBS) +test_plugin_datastore_postgres$(EXEEXT): $(test_plugin_datastore_postgres_OBJECTS) $(test_plugin_datastore_postgres_DEPENDENCIES) + @rm -f test_plugin_datastore_postgres$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_plugin_datastore_postgres_OBJECTS) $(test_plugin_datastore_postgres_LDADD) $(LIBS) +test_plugin_datastore_sqlite$(EXEEXT): $(test_plugin_datastore_sqlite_OBJECTS) $(test_plugin_datastore_sqlite_DEPENDENCIES) + @rm -f test_plugin_datastore_sqlite$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_plugin_datastore_sqlite_OBJECTS) $(test_plugin_datastore_sqlite_LDADD) $(LIBS) + +mostlyclean-compile: + -rm -f *.$(OBJEXT) + +distclean-compile: + -rm -f *.tab.c + +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/datastore_api.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/gnunet-service-datastore.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libgnunet_plugin_datastore_mysql_la-plugin_datastore_mysql.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libgnunet_plugin_datastore_postgres_la-plugin_datastore_postgres.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/perf_datastore_api.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/perf_plugin_datastore.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/plugin_datastore_sqlite.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/plugin_datastore_template.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test_datastore_api.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test_datastore_api_management.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test_plugin_datastore.Po@am__quote@ + +.c.o: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(COMPILE) -c $< + +.c.obj: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(COMPILE) -c `$(CYGPATH_W) '$<'` + +.c.lo: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LTCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo +@am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(LTCOMPILE) -c -o $@ $< + +libgnunet_plugin_datastore_mysql_la-plugin_datastore_mysql.lo: plugin_datastore_mysql.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnunet_plugin_datastore_mysql_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libgnunet_plugin_datastore_mysql_la-plugin_datastore_mysql.lo -MD -MP -MF $(DEPDIR)/libgnunet_plugin_datastore_mysql_la-plugin_datastore_mysql.Tpo -c -o libgnunet_plugin_datastore_mysql_la-plugin_datastore_mysql.lo `test -f 'plugin_datastore_mysql.c' || echo '$(srcdir)/'`plugin_datastore_mysql.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libgnunet_plugin_datastore_mysql_la-plugin_datastore_mysql.Tpo $(DEPDIR)/libgnunet_plugin_datastore_mysql_la-plugin_datastore_mysql.Plo +@am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='plugin_datastore_mysql.c' object='libgnunet_plugin_datastore_mysql_la-plugin_datastore_mysql.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnunet_plugin_datastore_mysql_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libgnunet_plugin_datastore_mysql_la-plugin_datastore_mysql.lo `test -f 'plugin_datastore_mysql.c' || echo '$(srcdir)/'`plugin_datastore_mysql.c + +libgnunet_plugin_datastore_postgres_la-plugin_datastore_postgres.lo: plugin_datastore_postgres.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnunet_plugin_datastore_postgres_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libgnunet_plugin_datastore_postgres_la-plugin_datastore_postgres.lo -MD -MP -MF $(DEPDIR)/libgnunet_plugin_datastore_postgres_la-plugin_datastore_postgres.Tpo -c -o libgnunet_plugin_datastore_postgres_la-plugin_datastore_postgres.lo `test -f 'plugin_datastore_postgres.c' || echo '$(srcdir)/'`plugin_datastore_postgres.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libgnunet_plugin_datastore_postgres_la-plugin_datastore_postgres.Tpo $(DEPDIR)/libgnunet_plugin_datastore_postgres_la-plugin_datastore_postgres.Plo +@am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='plugin_datastore_postgres.c' object='libgnunet_plugin_datastore_postgres_la-plugin_datastore_postgres.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libgnunet_plugin_datastore_postgres_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libgnunet_plugin_datastore_postgres_la-plugin_datastore_postgres.lo `test -f 'plugin_datastore_postgres.c' || echo '$(srcdir)/'`plugin_datastore_postgres.c + +mostlyclean-libtool: + -rm -f *.lo + +clean-libtool: + -rm -rf .libs _libs +install-pkgcfgDATA: $(pkgcfg_DATA) + @$(NORMAL_INSTALL) + test -z "$(pkgcfgdir)" || $(MKDIR_P) "$(DESTDIR)$(pkgcfgdir)" + @list='$(pkgcfg_DATA)'; test -n "$(pkgcfgdir)" || list=; \ + for p in $$list; do \ + if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \ + echo "$$d$$p"; \ + done | $(am__base_list) | \ + while read files; do \ + echo " $(INSTALL_DATA) $$files '$(DESTDIR)$(pkgcfgdir)'"; \ + $(INSTALL_DATA) $$files "$(DESTDIR)$(pkgcfgdir)" || exit $$?; \ + done + +uninstall-pkgcfgDATA: + @$(NORMAL_UNINSTALL) + @list='$(pkgcfg_DATA)'; test -n "$(pkgcfgdir)" || list=; \ + files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \ + test -n "$$files" || exit 0; \ + echo " ( cd '$(DESTDIR)$(pkgcfgdir)' && rm -f" $$files ")"; \ + cd "$(DESTDIR)$(pkgcfgdir)" && rm -f $$files + +ID: $(HEADERS) $(SOURCES) $(LISP) $(TAGS_FILES) + list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | \ + $(AWK) '{ files[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in files) print i; }; }'`; \ + mkid -fID $$unique +tags: TAGS + +TAGS: $(HEADERS) $(SOURCES) $(TAGS_DEPENDENCIES) \ + $(TAGS_FILES) $(LISP) + set x; \ + here=`pwd`; \ + list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | \ + $(AWK) '{ files[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in files) print i; }; }'`; \ + shift; \ + if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \ + test -n "$$unique" || unique=$$empty_fix; \ + if test $$# -gt 0; then \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + "$$@" $$unique; \ + else \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + $$unique; \ + fi; \ + fi +ctags: CTAGS +CTAGS: $(HEADERS) $(SOURCES) $(TAGS_DEPENDENCIES) \ + $(TAGS_FILES) $(LISP) + list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | \ + $(AWK) '{ files[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in files) print i; }; }'`; \ + test -z "$(CTAGS_ARGS)$$unique" \ + || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \ + $$unique + +GTAGS: + here=`$(am__cd) $(top_builddir) && pwd` \ + && $(am__cd) $(top_srcdir) \ + && gtags -i $(GTAGS_ARGS) "$$here" + +distclean-tags: + -rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags + +check-TESTS: $(TESTS) + @failed=0; all=0; xfail=0; xpass=0; skip=0; \ + srcdir=$(srcdir); export srcdir; \ + list=' $(TESTS) '; \ + $(am__tty_colors); \ + if test -n "$$list"; then \ + for tst in $$list; do \ + if test -f ./$$tst; then dir=./; \ + elif test -f $$tst; then dir=; \ + else dir="$(srcdir)/"; fi; \ + if $(TESTS_ENVIRONMENT) $${dir}$$tst; then \ + all=`expr $$all + 1`; \ + case " $(XFAIL_TESTS) " in \ + *[\ \ ]$$tst[\ \ ]*) \ + xpass=`expr $$xpass + 1`; \ + failed=`expr $$failed + 1`; \ + col=$$red; res=XPASS; \ + ;; \ + *) \ + col=$$grn; res=PASS; \ + ;; \ + esac; \ + elif test $$? -ne 77; then \ + all=`expr $$all + 1`; \ + case " $(XFAIL_TESTS) " in \ + *[\ \ ]$$tst[\ \ ]*) \ + xfail=`expr $$xfail + 1`; \ + col=$$lgn; res=XFAIL; \ + ;; \ + *) \ + failed=`expr $$failed + 1`; \ + col=$$red; res=FAIL; \ + ;; \ + esac; \ + else \ + skip=`expr $$skip + 1`; \ + col=$$blu; res=SKIP; \ + fi; \ + echo "$${col}$$res$${std}: $$tst"; \ + done; \ + if test "$$all" -eq 1; then \ + tests="test"; \ + All=""; \ + else \ + tests="tests"; \ + All="All "; \ + fi; \ + if test "$$failed" -eq 0; then \ + if test "$$xfail" -eq 0; then \ + banner="$$All$$all $$tests passed"; \ + else \ + if test "$$xfail" -eq 1; then failures=failure; else failures=failures; fi; \ + banner="$$All$$all $$tests behaved as expected ($$xfail expected $$failures)"; \ + fi; \ + else \ + if test "$$xpass" -eq 0; then \ + banner="$$failed of $$all $$tests failed"; \ + else \ + if test "$$xpass" -eq 1; then passes=pass; else passes=passes; fi; \ + banner="$$failed of $$all $$tests did not behave as expected ($$xpass unexpected $$passes)"; \ + fi; \ + fi; \ + dashes="$$banner"; \ + skipped=""; \ + if test "$$skip" -ne 0; then \ + if test "$$skip" -eq 1; then \ + skipped="($$skip test was not run)"; \ + else \ + skipped="($$skip tests were not run)"; \ + fi; \ + test `echo "$$skipped" | wc -c` -le `echo "$$banner" | wc -c` || \ + dashes="$$skipped"; \ + fi; \ + report=""; \ + if test "$$failed" -ne 0 && test -n "$(PACKAGE_BUGREPORT)"; then \ + report="Please report to $(PACKAGE_BUGREPORT)"; \ + test `echo "$$report" | wc -c` -le `echo "$$banner" | wc -c` || \ + dashes="$$report"; \ + fi; \ + dashes=`echo "$$dashes" | sed s/./=/g`; \ + if test "$$failed" -eq 0; then \ + echo "$$grn$$dashes"; \ + else \ + echo "$$red$$dashes"; \ + fi; \ + echo "$$banner"; \ + test -z "$$skipped" || echo "$$skipped"; \ + test -z "$$report" || echo "$$report"; \ + echo "$$dashes$$std"; \ + test "$$failed" -eq 0; \ + else :; fi + +distdir: $(DISTFILES) + @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + list='$(DISTFILES)'; \ + dist_files=`for file in $$list; do echo $$file; done | \ + sed -e "s|^$$srcdirstrip/||;t" \ + -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \ + case $$dist_files in \ + */*) $(MKDIR_P) `echo "$$dist_files" | \ + sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \ + sort -u` ;; \ + esac; \ + for file in $$dist_files; do \ + if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \ + if test -d $$d/$$file; then \ + dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \ + if test -d "$(distdir)/$$file"; then \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \ + cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \ + else \ + test -f "$(distdir)/$$file" \ + || cp -p $$d/$$file "$(distdir)/$$file" \ + || exit 1; \ + fi; \ + done +check-am: all-am + $(MAKE) $(AM_MAKEFLAGS) $(check_PROGRAMS) + $(MAKE) $(AM_MAKEFLAGS) check-TESTS +check: check-am +all-am: Makefile $(LTLIBRARIES) $(PROGRAMS) $(DATA) +install-binPROGRAMS: install-libLTLIBRARIES + +installdirs: + for dir in "$(DESTDIR)$(libdir)" "$(DESTDIR)$(plugindir)" "$(DESTDIR)$(bindir)" "$(DESTDIR)$(pkgcfgdir)"; do \ + test -z "$$dir" || $(MKDIR_P) "$$dir"; \ + done +install: install-am +install-exec: install-exec-am +install-data: install-data-am +uninstall: uninstall-am + +install-am: all-am + @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am + +installcheck: installcheck-am +install-strip: + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + `test -z '$(STRIP)' || \ + echo "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'"` install +mostlyclean-generic: + +clean-generic: + +distclean-generic: + -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES) + -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES) + +maintainer-clean-generic: + @echo "This command is intended for maintainers to use" + @echo "it deletes files that may require special tools to rebuild." +clean: clean-am + +clean-am: clean-binPROGRAMS clean-checkPROGRAMS clean-generic \ + clean-libLTLIBRARIES clean-libtool clean-pluginLTLIBRARIES \ + mostlyclean-am + +distclean: distclean-am + -rm -rf ./$(DEPDIR) + -rm -f Makefile +distclean-am: clean-am distclean-compile distclean-generic \ + distclean-tags + +dvi: dvi-am + +dvi-am: + +html: html-am + +html-am: + +info: info-am + +info-am: + +install-data-am: install-pkgcfgDATA install-pluginLTLIBRARIES + +install-dvi: install-dvi-am + +install-dvi-am: + +install-exec-am: install-binPROGRAMS install-libLTLIBRARIES + +install-html: install-html-am + +install-html-am: + +install-info: install-info-am + +install-info-am: + +install-man: + +install-pdf: install-pdf-am + +install-pdf-am: + +install-ps: install-ps-am + +install-ps-am: + +installcheck-am: + +maintainer-clean: maintainer-clean-am + -rm -rf ./$(DEPDIR) + -rm -f Makefile +maintainer-clean-am: distclean-am maintainer-clean-generic + +mostlyclean: mostlyclean-am + +mostlyclean-am: mostlyclean-compile mostlyclean-generic \ + mostlyclean-libtool + +pdf: pdf-am + +pdf-am: + +ps: ps-am + +ps-am: + +uninstall-am: uninstall-binPROGRAMS uninstall-libLTLIBRARIES \ + uninstall-pkgcfgDATA uninstall-pluginLTLIBRARIES + +.MAKE: check-am install-am install-strip + +.PHONY: CTAGS GTAGS all all-am check check-TESTS check-am clean \ + clean-binPROGRAMS clean-checkPROGRAMS clean-generic \ + clean-libLTLIBRARIES clean-libtool clean-pluginLTLIBRARIES \ + ctags distclean distclean-compile distclean-generic \ + distclean-libtool distclean-tags distdir dvi dvi-am html \ + html-am info info-am install install-am install-binPROGRAMS \ + install-data install-data-am install-dvi install-dvi-am \ + install-exec install-exec-am install-html install-html-am \ + install-info install-info-am install-libLTLIBRARIES \ + install-man install-pdf install-pdf-am install-pkgcfgDATA \ + install-pluginLTLIBRARIES install-ps install-ps-am \ + install-strip installcheck installcheck-am installdirs \ + maintainer-clean maintainer-clean-generic mostlyclean \ + mostlyclean-compile mostlyclean-generic mostlyclean-libtool \ + pdf pdf-am ps ps-am tags uninstall uninstall-am \ + uninstall-binPROGRAMS uninstall-libLTLIBRARIES \ + uninstall-pkgcfgDATA uninstall-pluginLTLIBRARIES + + +# Tell versions [3.59,3.63) of GNU make to not export all variables. +# Otherwise a system limit (for SysV at least) may be exceeded. +.NOEXPORT: diff --git a/src/datastore/datastore.conf.in b/src/datastore/datastore.conf.in new file mode 100644 index 0000000..837c619 --- /dev/null +++ b/src/datastore/datastore.conf.in @@ -0,0 +1,33 @@ +[datastore] +AUTOSTART = YES +UNIXPATH = /tmp/gnunet-service-datastore.sock +UNIX_MATCH_UID = YES +UNIX_MATCH_GID = YES +@UNIXONLY@ PORT = 2093 +HOSTNAME = localhost +HOME = $SERVICEHOME +CONFIG = $DEFAULTCONFIG +BINARY = gnunet-service-datastore +ACCEPT_FROM = 127.0.0.1; +ACCEPT_FROM6 = ::1; +QUOTA = 100 MB +BLOOMFILTER = $SERVICEHOME/datastore/bloomfilter +DATABASE = sqlite +# DISABLE_SOCKET_FORWARDING = NO + +[datastore-sqlite] +FILENAME = $SERVICEHOME/datastore/sqlite.db + +[datastore-postgres] +CONFIG = connect_timeout=10; dbname=gnunet + +[datastore-mysql] +DATABASE = gnunet +CONFIG = ~/.my.cnf +# USER = gnunet +# PASSWORD = +# HOST = localhost +# PORT = 3306 + + + diff --git a/src/datastore/datastore.h b/src/datastore/datastore.h new file mode 100644 index 0000000..1126027 --- /dev/null +++ b/src/datastore/datastore.h @@ -0,0 +1,263 @@ +/* + This file is part of GNUnet + (C) 2004, 2005, 2006, 2007, 2009 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 2, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file datastore/datastore.h + * @brief structs for communication between datastore service and API + * @author Christian Grothoff + */ + +#ifndef DATASTORE_H +#define DATASTORE_H + +#define DEBUG_DATASTORE GNUNET_EXTRA_LOGGING + +#include "gnunet_util_lib.h" + +GNUNET_NETWORK_STRUCT_BEGIN + +/** + * Message from datastore service informing client about + * the current size of the datastore. + */ +struct ReserveMessage +{ + /** + * Type is GNUNET_MESSAGE_TYPE_DATASTORE_RESERVE. + */ + struct GNUNET_MessageHeader header; + + /** + * Number of items to reserve. + */ + uint32_t entries GNUNET_PACKED; + + /** + * Number of bytes to reserve. + */ + uint64_t amount GNUNET_PACKED; +}; + + +/** + * Message from datastore service informing client about + * the success or failure of a requested operation. + * This header is optionally followed by a variable-size, + * 0-terminated error message. + */ +struct StatusMessage +{ + /** + * Type is GNUNET_MESSAGE_TYPE_DATASTORE_STATUS. + */ + struct GNUNET_MessageHeader header; + + /** + * Status code, -1 for errors. + */ + int32_t status GNUNET_PACKED; + + /** + * Minimum expiration time required for content to be stored + * by the datacache at this time, zero for unknown or no limit. + */ + struct GNUNET_TIME_AbsoluteNBO min_expiration; + +}; + + +/** + * Message from datastore client informing service that + * the remainder of the reserved bytes can now be released + * for other requests. + */ +struct ReleaseReserveMessage +{ + /** + * Type is GNUNET_MESSAGE_TYPE_DATASTORE_RELEASE_RESERVE. + */ + struct GNUNET_MessageHeader header; + + /** + * Reservation id. + */ + int32_t rid GNUNET_PACKED; + +}; + + +/** + * Message to the datastore service asking about specific + * content. + */ +struct GetMessage +{ + /** + * Type is GNUNET_MESSAGE_TYPE_DATASTORE_GET. Size + * can either be "sizeof(struct GetMessage)" or + * "sizeof(struct GetMessage) - sizeof(GNUNET_HashCode)"! + */ + struct GNUNET_MessageHeader header; + + /** + * Desired content type. (actually an enum GNUNET_BLOCK_Type) + */ + uint32_t type GNUNET_PACKED; + + /** + * Offset of the result. + */ + uint64_t offset GNUNET_PACKED; + + /** + * Desired key (optional). Check the "size" of the + * header to see if the key is actually present. + */ + GNUNET_HashCode key GNUNET_PACKED; + +}; + + +/** + * Message to the datastore service asking about zero + * anonymity content. + */ +struct GetZeroAnonymityMessage +{ + /** + * Type is GNUNET_MESSAGE_TYPE_DATASTORE_GET_ZERO_ANONYMITY. + */ + struct GNUNET_MessageHeader header; + + /** + * Desired content type (actually an enum GNUNET_BLOCK_Type) + */ + uint32_t type GNUNET_PACKED; + + /** + * Offset of the result. + */ + uint64_t offset GNUNET_PACKED; + +}; + + +/** + * Message to the datastore service requesting an update + * to the priority or expiration for some content. + */ +struct UpdateMessage +{ + /** + * Type is GNUNET_MESSAGE_TYPE_DATASTORE_UPDATE. + */ + struct GNUNET_MessageHeader header; + + /** + * Desired priority increase. + */ + int32_t priority GNUNET_PACKED; + + /** + * Desired new expiration time. + */ + struct GNUNET_TIME_AbsoluteNBO expiration; + + /** + * Unique ID for the content. + */ + uint64_t uid; + +}; + + +/** + * Message transmitting content from or to the datastore + * service. + */ +struct DataMessage +{ + /** + * Type is either GNUNET_MESSAGE_TYPE_DATASTORE_PUT, + * GNUNET_MESSAGE_TYPE_DATASTORE_REMOVE or + * GNUNET_MESSAGE_TYPE_DATASTORE_DATA. Depending on the message + * type, some fields may simply have values of zero. + */ + struct GNUNET_MessageHeader header; + + /** + * Reservation ID to use; use zero for none. + */ + uint32_t rid GNUNET_PACKED; + + /** + * Number of bytes in the item (NBO). + */ + uint32_t size GNUNET_PACKED; + + /** + * Type of the item (NBO), zero for remove, (actually an enum GNUNET_BLOCK_Type) + */ + uint32_t type GNUNET_PACKED; + + /** + * Priority of the item (NBO), zero for remove. + */ + uint32_t priority GNUNET_PACKED; + + /** + * Desired anonymity level (NBO), zero for remove. + */ + uint32_t anonymity GNUNET_PACKED; + + /** + * Desired replication level. 0 from service to API. + */ + uint32_t replication GNUNET_PACKED; + + /** + * For alignment. + */ + uint32_t reserved GNUNET_PACKED; + + /** + * Unique ID for the content (can be used for UPDATE); + * can be zero for remove (which indicates that + * the datastore should use whatever UID matches + * the key and content). + */ + uint64_t uid; + + /** + * Expiration time (NBO); zero for remove. + */ + struct GNUNET_TIME_AbsoluteNBO expiration; + + /** + * Key under which the item can be found. + */ + GNUNET_HashCode key GNUNET_PACKED; + +}; +GNUNET_NETWORK_STRUCT_END + + + +#endif diff --git a/src/datastore/datastore_api.c b/src/datastore/datastore_api.c new file mode 100644 index 0000000..4f406a2 --- /dev/null +++ b/src/datastore/datastore_api.c @@ -0,0 +1,1505 @@ +/* + This file is part of GNUnet + (C) 2004, 2005, 2006, 2007, 2009, 2010, 2011 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file datastore/datastore_api.c + * @brief Management for the datastore for files stored on a GNUnet node. Implements + * a priority queue for requests (with timeouts). + * @author Christian Grothoff + */ +#include "platform.h" +#include "gnunet_arm_service.h" +#include "gnunet_constants.h" +#include "gnunet_datastore_service.h" +#include "gnunet_statistics_service.h" +#include "datastore.h" + +#define LOG(kind,...) GNUNET_log_from (kind, "datastore-api",__VA_ARGS__) + +/** + * If a client stopped asking for more results, how many more do + * we receive from the DB before killing the connection? Trade-off + * between re-doing TCP handshakes and (needlessly) receiving + * useless results. + */ +#define MAX_EXCESS_RESULTS 8 + +/** + * Context for processing status messages. + */ +struct StatusContext +{ + /** + * Continuation to call with the status. + */ + GNUNET_DATASTORE_ContinuationWithStatus cont; + + /** + * Closure for cont. + */ + void *cont_cls; + +}; + + +/** + * Context for processing result messages. + */ +struct ResultContext +{ + /** + * Function to call with the result. + */ + GNUNET_DATASTORE_DatumProcessor proc; + + /** + * Closure for proc. + */ + void *proc_cls; + +}; + + +/** + * Context for a queue operation. + */ +union QueueContext +{ + + struct StatusContext sc; + + struct ResultContext rc; + +}; + + + +/** + * Entry in our priority queue. + */ +struct GNUNET_DATASTORE_QueueEntry +{ + + /** + * This is a linked list. + */ + struct GNUNET_DATASTORE_QueueEntry *next; + + /** + * This is a linked list. + */ + struct GNUNET_DATASTORE_QueueEntry *prev; + + /** + * Handle to the master context. + */ + struct GNUNET_DATASTORE_Handle *h; + + /** + * Response processor (NULL if we are not waiting for a response). + * This struct should be used for the closure, function-specific + * arguments can be passed via 'qc'. + */ + GNUNET_CLIENT_MessageHandler response_proc; + + /** + * Function to call after transmission of the request. + */ + GNUNET_DATASTORE_ContinuationWithStatus cont; + + /** + * Closure for 'cont'. + */ + void *cont_cls; + + /** + * Context for the operation. + */ + union QueueContext qc; + + /** + * Task for timeout signalling. + */ + GNUNET_SCHEDULER_TaskIdentifier task; + + /** + * Timeout for the current operation. + */ + struct GNUNET_TIME_Absolute timeout; + + /** + * Priority in the queue. + */ + unsigned int priority; + + /** + * Maximum allowed length of queue (otherwise + * this request should be discarded). + */ + unsigned int max_queue; + + /** + * Number of bytes in the request message following + * this struct. 32-bit value for nicer memory + * access (and overall struct alignment). + */ + uint32_t message_size; + + /** + * Has this message been transmitted to the service? + * Only ever GNUNET_YES for the head of the queue. + * Note that the overall struct should end at a + * multiple of 64 bits. + */ + int was_transmitted; + +}; + +/** + * Handle to the datastore service. + */ +struct GNUNET_DATASTORE_Handle +{ + + /** + * Our configuration. + */ + const struct GNUNET_CONFIGURATION_Handle *cfg; + + /** + * Current connection to the datastore service. + */ + struct GNUNET_CLIENT_Connection *client; + + /** + * Handle for statistics. + */ + struct GNUNET_STATISTICS_Handle *stats; + + /** + * Current transmit handle. + */ + struct GNUNET_CLIENT_TransmitHandle *th; + + /** + * Current head of priority queue. + */ + struct GNUNET_DATASTORE_QueueEntry *queue_head; + + /** + * Current tail of priority queue. + */ + struct GNUNET_DATASTORE_QueueEntry *queue_tail; + + /** + * Task for trying to reconnect. + */ + GNUNET_SCHEDULER_TaskIdentifier reconnect_task; + + /** + * How quickly should we retry? Used for exponential back-off on + * connect-errors. + */ + struct GNUNET_TIME_Relative retry_time; + + /** + * Number of entries in the queue. + */ + unsigned int queue_size; + + /** + * Number of results we're receiving for the current query + * after application stopped to care. Used to determine when + * to reset the connection. + */ + unsigned int result_count; + + /** + * Are we currently trying to receive from the service? + */ + int in_receive; + + /** + * We should ignore the next message(s) from the service. + */ + unsigned int skip_next_messages; + +}; + + + +/** + * Connect to the datastore service. + * + * @param cfg configuration to use + * @return handle to use to access the service + */ +struct GNUNET_DATASTORE_Handle * +GNUNET_DATASTORE_connect (const struct GNUNET_CONFIGURATION_Handle *cfg) +{ + struct GNUNET_CLIENT_Connection *c; + struct GNUNET_DATASTORE_Handle *h; + + c = GNUNET_CLIENT_connect ("datastore", cfg); + if (c == NULL) + return NULL; /* oops */ + h = GNUNET_malloc (sizeof (struct GNUNET_DATASTORE_Handle) + + GNUNET_SERVER_MAX_MESSAGE_SIZE - 1); + h->client = c; + h->cfg = cfg; + h->stats = GNUNET_STATISTICS_create ("datastore-api", cfg); + return h; +} + + +/** + * Transmit DROP message to datastore service. + * + * @param cls the 'struct GNUNET_DATASTORE_Handle' + * @param size number of bytes that can be copied to buf + * @param buf where to copy the drop message + * @return number of bytes written to buf + */ +static size_t +transmit_drop (void *cls, size_t size, void *buf) +{ + struct GNUNET_DATASTORE_Handle *h = cls; + struct GNUNET_MessageHeader *hdr; + + if (buf == NULL) + { + LOG (GNUNET_ERROR_TYPE_WARNING, + _("Failed to transmit request to drop database.\n")); + GNUNET_DATASTORE_disconnect (h, GNUNET_NO); + return 0; + } + GNUNET_assert (size >= sizeof (struct GNUNET_MessageHeader)); + hdr = buf; + hdr->size = htons (sizeof (struct GNUNET_MessageHeader)); + hdr->type = htons (GNUNET_MESSAGE_TYPE_DATASTORE_DROP); + GNUNET_DATASTORE_disconnect (h, GNUNET_NO); + return sizeof (struct GNUNET_MessageHeader); +} + + +/** + * Disconnect from the datastore service (and free + * associated resources). + * + * @param h handle to the datastore + * @param drop set to GNUNET_YES to delete all data in datastore (!) + */ +void +GNUNET_DATASTORE_disconnect (struct GNUNET_DATASTORE_Handle *h, int drop) +{ + struct GNUNET_DATASTORE_QueueEntry *qe; + +#if DEBUG_DATASTORE + LOG (GNUNET_ERROR_TYPE_DEBUG, "Datastore disconnect\n"); +#endif + if (NULL != h->th) + { + GNUNET_CLIENT_notify_transmit_ready_cancel (h->th); + h->th = NULL; + } + if (h->client != NULL) + { + GNUNET_CLIENT_disconnect (h->client, GNUNET_NO); + h->client = NULL; + } + if (h->reconnect_task != GNUNET_SCHEDULER_NO_TASK) + { + GNUNET_SCHEDULER_cancel (h->reconnect_task); + h->reconnect_task = GNUNET_SCHEDULER_NO_TASK; + } + while (NULL != (qe = h->queue_head)) + { + GNUNET_assert (NULL != qe->response_proc); + qe->response_proc (h, NULL); + } + if (GNUNET_YES == drop) + { + h->client = GNUNET_CLIENT_connect ("datastore", h->cfg); + if (h->client != NULL) + { + if (NULL != + GNUNET_CLIENT_notify_transmit_ready (h->client, + sizeof (struct + GNUNET_MessageHeader), + GNUNET_TIME_UNIT_MINUTES, + GNUNET_YES, &transmit_drop, h)) + return; + GNUNET_CLIENT_disconnect (h->client, GNUNET_NO); + h->client = NULL; + } + GNUNET_break (0); + } + GNUNET_STATISTICS_destroy (h->stats, GNUNET_NO); + h->stats = NULL; + GNUNET_free (h); +} + + +/** + * A request has timed out (before being transmitted to the service). + * + * @param cls the 'struct GNUNET_DATASTORE_QueueEntry' + * @param tc scheduler context + */ +static void +timeout_queue_entry (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct GNUNET_DATASTORE_QueueEntry *qe = cls; + + GNUNET_STATISTICS_update (qe->h->stats, + gettext_noop ("# queue entry timeouts"), 1, + GNUNET_NO); + qe->task = GNUNET_SCHEDULER_NO_TASK; + GNUNET_assert (qe->was_transmitted == GNUNET_NO); +#if DEBUG_DATASTORE + LOG (GNUNET_ERROR_TYPE_DEBUG, "Timeout of request in datastore queue\n"); +#endif + qe->response_proc (qe->h, NULL); +} + + +/** + * Create a new entry for our priority queue (and possibly discard other entires if + * the queue is getting too long). + * + * @param h handle to the datastore + * @param msize size of the message to queue + * @param queue_priority priority of the entry + * @param max_queue_size at what queue size should this request be dropped + * (if other requests of higher priority are in the queue) + * @param timeout timeout for the operation + * @param response_proc function to call with replies (can be NULL) + * @param qc client context (NOT a closure for response_proc) + * @return NULL if the queue is full + */ +static struct GNUNET_DATASTORE_QueueEntry * +make_queue_entry (struct GNUNET_DATASTORE_Handle *h, size_t msize, + unsigned int queue_priority, unsigned int max_queue_size, + struct GNUNET_TIME_Relative timeout, + GNUNET_CLIENT_MessageHandler response_proc, + const union QueueContext *qc) +{ + struct GNUNET_DATASTORE_QueueEntry *ret; + struct GNUNET_DATASTORE_QueueEntry *pos; + unsigned int c; + + c = 0; + pos = h->queue_head; + while ((pos != NULL) && (c < max_queue_size) && + (pos->priority >= queue_priority)) + { + c++; + pos = pos->next; + } + if (c >= max_queue_size) + { + GNUNET_STATISTICS_update (h->stats, gettext_noop ("# queue overflows"), 1, + GNUNET_NO); + return NULL; + } + ret = GNUNET_malloc (sizeof (struct GNUNET_DATASTORE_QueueEntry) + msize); + ret->h = h; + ret->response_proc = response_proc; + ret->qc = *qc; + ret->timeout = GNUNET_TIME_relative_to_absolute (timeout); + ret->priority = queue_priority; + ret->max_queue = max_queue_size; + ret->message_size = msize; + ret->was_transmitted = GNUNET_NO; + if (pos == NULL) + { + /* append at the tail */ + pos = h->queue_tail; + } + else + { + pos = pos->prev; + /* do not insert at HEAD if HEAD query was already + * transmitted and we are still receiving replies! */ + if ((pos == NULL) && (h->queue_head->was_transmitted)) + pos = h->queue_head; + } + c++; + GNUNET_STATISTICS_update (h->stats, gettext_noop ("# queue entries created"), + 1, GNUNET_NO); + GNUNET_CONTAINER_DLL_insert_after (h->queue_head, h->queue_tail, pos, ret); + h->queue_size++; + ret->task = GNUNET_SCHEDULER_add_delayed (timeout, &timeout_queue_entry, ret); + pos = ret->next; + while (pos != NULL) + { + if ((pos->max_queue < h->queue_size) && (pos->was_transmitted == GNUNET_NO)) + { + GNUNET_assert (pos->response_proc != NULL); + /* move 'pos' element to head so that it will be + * killed on 'NULL' call below */ +#if DEBUG_DATASTORE + LOG (GNUNET_ERROR_TYPE_DEBUG, "Dropping request from datastore queue\n"); +#endif + GNUNET_CONTAINER_DLL_remove (h->queue_head, h->queue_tail, pos); + GNUNET_CONTAINER_DLL_insert (h->queue_head, h->queue_tail, pos); + GNUNET_STATISTICS_update (h->stats, + gettext_noop + ("# Requests dropped from datastore queue"), 1, + GNUNET_NO); + GNUNET_assert (h->queue_head == pos); + pos->response_proc (h, NULL); + break; + } + pos = pos->next; + } + return ret; +} + + +/** + * Process entries in the queue (or do nothing if we are already + * doing so). + * + * @param h handle to the datastore + */ +static void +process_queue (struct GNUNET_DATASTORE_Handle *h); + + +/** + * Try reconnecting to the datastore service. + * + * @param cls the 'struct GNUNET_DATASTORE_Handle' + * @param tc scheduler context + */ +static void +try_reconnect (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct GNUNET_DATASTORE_Handle *h = cls; + + if (h->retry_time.rel_value < GNUNET_CONSTANTS_SERVICE_RETRY.rel_value) + h->retry_time = GNUNET_CONSTANTS_SERVICE_RETRY; + else + h->retry_time = GNUNET_TIME_relative_multiply (h->retry_time, 2); + if (h->retry_time.rel_value > GNUNET_CONSTANTS_SERVICE_TIMEOUT.rel_value) + h->retry_time = GNUNET_CONSTANTS_SERVICE_TIMEOUT; + h->reconnect_task = GNUNET_SCHEDULER_NO_TASK; + h->client = GNUNET_CLIENT_connect ("datastore", h->cfg); + if (h->client == NULL) + { + LOG (GNUNET_ERROR_TYPE_ERROR, "DATASTORE reconnect failed (fatally)\n"); + return; + } + GNUNET_STATISTICS_update (h->stats, + gettext_noop + ("# datastore connections (re)created"), 1, + GNUNET_NO); +#if DEBUG_DATASTORE + LOG (GNUNET_ERROR_TYPE_DEBUG, "Reconnected to DATASTORE\n"); +#endif + process_queue (h); +} + + +/** + * Disconnect from the service and then try reconnecting to the datastore service + * after some delay. + * + * @param h handle to datastore to disconnect and reconnect + */ +static void +do_disconnect (struct GNUNET_DATASTORE_Handle *h) +{ + if (h->client == NULL) + { +#if DEBUG_DATASTORE + LOG (GNUNET_ERROR_TYPE_DEBUG, + "client NULL in disconnect, will not try to reconnect\n"); +#endif + return; + } +#if 0 + GNUNET_STATISTICS_update (stats, gettext_noop ("# reconnected to DATASTORE"), + 1, GNUNET_NO); +#endif + GNUNET_CLIENT_disconnect (h->client, GNUNET_NO); + h->skip_next_messages = 0; + h->client = NULL; + h->reconnect_task = + GNUNET_SCHEDULER_add_delayed (h->retry_time, &try_reconnect, h); +} + + +/** + * Function called whenever we receive a message from + * the service. Calls the appropriate handler. + * + * @param cls the 'struct GNUNET_DATASTORE_Handle' + * @param msg the received message + */ +static void +receive_cb (void *cls, const struct GNUNET_MessageHeader *msg) +{ + struct GNUNET_DATASTORE_Handle *h = cls; + struct GNUNET_DATASTORE_QueueEntry *qe; + + h->in_receive = GNUNET_NO; +#if DEBUG_DATASTORE + LOG (GNUNET_ERROR_TYPE_DEBUG, "Receiving reply from datastore\n"); +#endif + if (h->skip_next_messages > 0) + { + h->skip_next_messages--; + process_queue (h); + return; + } + if (NULL == (qe = h->queue_head)) + { + GNUNET_break (0); + process_queue (h); + return; + } + qe->response_proc (h, msg); +} + + +/** + * Transmit request from queue to datastore service. + * + * @param cls the 'struct GNUNET_DATASTORE_Handle' + * @param size number of bytes that can be copied to buf + * @param buf where to copy the drop message + * @return number of bytes written to buf + */ +static size_t +transmit_request (void *cls, size_t size, void *buf) +{ + struct GNUNET_DATASTORE_Handle *h = cls; + struct GNUNET_DATASTORE_QueueEntry *qe; + size_t msize; + + h->th = NULL; + if (NULL == (qe = h->queue_head)) + return 0; /* no entry in queue */ + if (buf == NULL) + { +#if DEBUG_DATASTORE + LOG (GNUNET_ERROR_TYPE_DEBUG, "Failed to transmit request to DATASTORE.\n"); +#endif + GNUNET_STATISTICS_update (h->stats, + gettext_noop ("# transmission request failures"), + 1, GNUNET_NO); + do_disconnect (h); + return 0; + } + if (size < (msize = qe->message_size)) + { + process_queue (h); + return 0; + } +#if DEBUG_DATASTORE + LOG (GNUNET_ERROR_TYPE_DEBUG, "Transmitting %u byte request to DATASTORE\n", + msize); +#endif + memcpy (buf, &qe[1], msize); + qe->was_transmitted = GNUNET_YES; + GNUNET_SCHEDULER_cancel (qe->task); + qe->task = GNUNET_SCHEDULER_NO_TASK; + GNUNET_assert (GNUNET_NO == h->in_receive); + h->in_receive = GNUNET_YES; + GNUNET_CLIENT_receive (h->client, &receive_cb, h, + GNUNET_TIME_absolute_get_remaining (qe->timeout)); + GNUNET_STATISTICS_update (h->stats, + gettext_noop ("# bytes sent to datastore"), 1, + GNUNET_NO); + return msize; +} + + +/** + * Process entries in the queue (or do nothing if we are already + * doing so). + * + * @param h handle to the datastore + */ +static void +process_queue (struct GNUNET_DATASTORE_Handle *h) +{ + struct GNUNET_DATASTORE_QueueEntry *qe; + + if (NULL == (qe = h->queue_head)) + { +#if DEBUG_DATASTORE > 1 + LOG (GNUNET_ERROR_TYPE_DEBUG, "Queue empty\n"); +#endif + return; /* no entry in queue */ + } + if (qe->was_transmitted == GNUNET_YES) + { +#if DEBUG_DATASTORE > 1 + LOG (GNUNET_ERROR_TYPE_DEBUG, "Head request already transmitted\n"); +#endif + return; /* waiting for replies */ + } + if (h->th != NULL) + { +#if DEBUG_DATASTORE > 1 + LOG (GNUNET_ERROR_TYPE_DEBUG, "Pending transmission request\n"); +#endif + return; /* request pending */ + } + if (h->client == NULL) + { +#if DEBUG_DATASTORE > 1 + LOG (GNUNET_ERROR_TYPE_DEBUG, "Not connected\n"); +#endif + return; /* waiting for reconnect */ + } + if (GNUNET_YES == h->in_receive) + { + /* wait for response to previous query */ + return; + } +#if DEBUG_DATASTORE + LOG (GNUNET_ERROR_TYPE_DEBUG, "Queueing %u byte request to DATASTORE\n", + qe->message_size); +#endif + h->th = + GNUNET_CLIENT_notify_transmit_ready (h->client, qe->message_size, + GNUNET_TIME_absolute_get_remaining + (qe->timeout), GNUNET_YES, + &transmit_request, h); + GNUNET_assert (GNUNET_NO == h->in_receive); + GNUNET_break (NULL != h->th); +} + + +/** + * Dummy continuation used to do nothing (but be non-zero). + * + * @param cls closure + * @param result result + * @param min_expiration expiration time + * @param emsg error message + */ +static void +drop_status_cont (void *cls, int32_t result, + struct GNUNET_TIME_Absolute min_expiration, + const char *emsg) +{ + /* do nothing */ +} + + +/** + * Free a queue entry. Removes the given entry from the + * queue and releases associated resources. Does NOT + * call the callback. + * + * @param qe entry to free. + */ +static void +free_queue_entry (struct GNUNET_DATASTORE_QueueEntry *qe) +{ + struct GNUNET_DATASTORE_Handle *h = qe->h; + + GNUNET_CONTAINER_DLL_remove (h->queue_head, h->queue_tail, qe); + if (qe->task != GNUNET_SCHEDULER_NO_TASK) + { + GNUNET_SCHEDULER_cancel (qe->task); + qe->task = GNUNET_SCHEDULER_NO_TASK; + } + h->queue_size--; + qe->was_transmitted = GNUNET_SYSERR; /* use-after-free warning */ + GNUNET_free (qe); +} + + +/** + * Type of a function to call when we receive a message + * from the service. + * + * @param cls closure + * @param msg message received, NULL on timeout or fatal error + */ +static void +process_status_message (void *cls, const struct GNUNET_MessageHeader *msg) +{ + struct GNUNET_DATASTORE_Handle *h = cls; + struct GNUNET_DATASTORE_QueueEntry *qe; + struct StatusContext rc; + const struct StatusMessage *sm; + const char *emsg; + int32_t status; + int was_transmitted; + + if (NULL == (qe = h->queue_head)) + { + GNUNET_break (0); + do_disconnect (h); + return; + } + rc = qe->qc.sc; + if (msg == NULL) + { + was_transmitted = qe->was_transmitted; + free_queue_entry (qe); + if (was_transmitted == GNUNET_YES) + do_disconnect (h); + else + process_queue (h); + if (rc.cont != NULL) + rc.cont (rc.cont_cls, GNUNET_SYSERR, + GNUNET_TIME_UNIT_ZERO_ABS, + _("Failed to receive status response from database.")); + return; + } + GNUNET_assert (GNUNET_YES == qe->was_transmitted); + free_queue_entry (qe); + if ((ntohs (msg->size) < sizeof (struct StatusMessage)) || + (ntohs (msg->type) != GNUNET_MESSAGE_TYPE_DATASTORE_STATUS)) + { + GNUNET_break (0); + h->retry_time = GNUNET_TIME_UNIT_ZERO; + do_disconnect (h); + if (rc.cont != NULL) + rc.cont (rc.cont_cls, GNUNET_SYSERR, + GNUNET_TIME_UNIT_ZERO_ABS, + _("Error reading response from datastore service")); + return; + } + sm = (const struct StatusMessage *) msg; + status = ntohl (sm->status); + emsg = NULL; + if (ntohs (msg->size) > sizeof (struct StatusMessage)) + { + emsg = (const char *) &sm[1]; + if (emsg[ntohs (msg->size) - sizeof (struct StatusMessage) - 1] != '\0') + { + GNUNET_break (0); + emsg = _("Invalid error message received from datastore service"); + } + } + if ((status == GNUNET_SYSERR) && (emsg == NULL)) + { + GNUNET_break (0); + emsg = _("Invalid error message received from datastore service"); + } +#if DEBUG_DATASTORE + LOG (GNUNET_ERROR_TYPE_DEBUG, "Received status %d/%s\n", (int) status, emsg); +#endif + GNUNET_STATISTICS_update (h->stats, + gettext_noop ("# status messages received"), 1, + GNUNET_NO); + h->retry_time.rel_value = 0; + process_queue (h); + if (rc.cont != NULL) + rc.cont (rc.cont_cls, status, + GNUNET_TIME_absolute_ntoh (sm->min_expiration), + emsg); +} + + +/** + * Store an item in the datastore. If the item is already present, + * the priorities are summed up and the higher expiration time and + * lower anonymity level is used. + * + * @param h handle to the datastore + * @param rid reservation ID to use (from "reserve"); use 0 if no + * prior reservation was made + * @param key key for the value + * @param size number of bytes in data + * @param data content stored + * @param type type of the content + * @param priority priority of the content + * @param anonymity anonymity-level for the content + * @param replication how often should the content be replicated to other peers? + * @param expiration expiration time for the content + * @param queue_priority ranking of this request in the priority queue + * @param max_queue_size at what queue size should this request be dropped + * (if other requests of higher priority are in the queue) + * @param timeout timeout for the operation + * @param cont continuation to call when done + * @param cont_cls closure for cont + * @return NULL if the entry was not queued, otherwise a handle that can be used to + * cancel; note that even if NULL is returned, the callback will be invoked + * (or rather, will already have been invoked) + */ +struct GNUNET_DATASTORE_QueueEntry * +GNUNET_DATASTORE_put (struct GNUNET_DATASTORE_Handle *h, uint32_t rid, + const GNUNET_HashCode * key, size_t size, + const void *data, enum GNUNET_BLOCK_Type type, + uint32_t priority, uint32_t anonymity, + uint32_t replication, + struct GNUNET_TIME_Absolute expiration, + unsigned int queue_priority, unsigned int max_queue_size, + struct GNUNET_TIME_Relative timeout, + GNUNET_DATASTORE_ContinuationWithStatus cont, + void *cont_cls) +{ + struct GNUNET_DATASTORE_QueueEntry *qe; + struct DataMessage *dm; + size_t msize; + union QueueContext qc; + +#if DEBUG_DATASTORE + LOG (GNUNET_ERROR_TYPE_DEBUG, + "Asked to put %u bytes of data under key `%s' for %llu ms\n", size, + GNUNET_h2s (key), + GNUNET_TIME_absolute_get_remaining (expiration).rel_value); +#endif + msize = sizeof (struct DataMessage) + size; + GNUNET_assert (msize < GNUNET_SERVER_MAX_MESSAGE_SIZE); + qc.sc.cont = cont; + qc.sc.cont_cls = cont_cls; + qe = make_queue_entry (h, msize, queue_priority, max_queue_size, timeout, + &process_status_message, &qc); + if (qe == NULL) + { +#if DEBUG_DATASTORE + LOG (GNUNET_ERROR_TYPE_DEBUG, "Could not create queue entry for PUT\n"); +#endif + return NULL; + } + GNUNET_STATISTICS_update (h->stats, gettext_noop ("# PUT requests executed"), + 1, GNUNET_NO); + dm = (struct DataMessage *) &qe[1]; + dm->header.type = htons (GNUNET_MESSAGE_TYPE_DATASTORE_PUT); + dm->header.size = htons (msize); + dm->rid = htonl (rid); + dm->size = htonl ((uint32_t) size); + dm->type = htonl (type); + dm->priority = htonl (priority); + dm->anonymity = htonl (anonymity); + dm->replication = htonl (replication); + dm->reserved = htonl (0); + dm->uid = GNUNET_htonll (0); + dm->expiration = GNUNET_TIME_absolute_hton (expiration); + dm->key = *key; + memcpy (&dm[1], data, size); + process_queue (h); + return qe; +} + + +/** + * Reserve space in the datastore. This function should be used + * to avoid "out of space" failures during a longer sequence of "put" + * operations (for example, when a file is being inserted). + * + * @param h handle to the datastore + * @param amount how much space (in bytes) should be reserved (for content only) + * @param entries how many entries will be created (to calculate per-entry overhead) + * @param queue_priority ranking of this request in the priority queue + * @param max_queue_size at what queue size should this request be dropped + * (if other requests of higher priority are in the queue) + * @param timeout how long to wait at most for a response (or before dying in queue) + * @param cont continuation to call when done; "success" will be set to + * a positive reservation value if space could be reserved. + * @param cont_cls closure for cont + * @return NULL if the entry was not queued, otherwise a handle that can be used to + * cancel; note that even if NULL is returned, the callback will be invoked + * (or rather, will already have been invoked) + */ +struct GNUNET_DATASTORE_QueueEntry * +GNUNET_DATASTORE_reserve (struct GNUNET_DATASTORE_Handle *h, uint64_t amount, + uint32_t entries, unsigned int queue_priority, + unsigned int max_queue_size, + struct GNUNET_TIME_Relative timeout, + GNUNET_DATASTORE_ContinuationWithStatus cont, + void *cont_cls) +{ + struct GNUNET_DATASTORE_QueueEntry *qe; + struct ReserveMessage *rm; + union QueueContext qc; + + if (cont == NULL) + cont = &drop_status_cont; +#if DEBUG_DATASTORE + LOG (GNUNET_ERROR_TYPE_DEBUG, + "Asked to reserve %llu bytes of data and %u entries\n", + (unsigned long long) amount, (unsigned int) entries); +#endif + qc.sc.cont = cont; + qc.sc.cont_cls = cont_cls; + qe = make_queue_entry (h, sizeof (struct ReserveMessage), queue_priority, + max_queue_size, timeout, &process_status_message, &qc); + if (qe == NULL) + { +#if DEBUG_DATASTORE + LOG (GNUNET_ERROR_TYPE_DEBUG, "Could not create queue entry to reserve\n"); +#endif + return NULL; + } + GNUNET_STATISTICS_update (h->stats, + gettext_noop ("# RESERVE requests executed"), 1, + GNUNET_NO); + rm = (struct ReserveMessage *) &qe[1]; + rm->header.type = htons (GNUNET_MESSAGE_TYPE_DATASTORE_RESERVE); + rm->header.size = htons (sizeof (struct ReserveMessage)); + rm->entries = htonl (entries); + rm->amount = GNUNET_htonll (amount); + process_queue (h); + return qe; +} + + +/** + * Signal that all of the data for which a reservation was made has + * been stored and that whatever excess space might have been reserved + * can now be released. + * + * @param h handle to the datastore + * @param rid reservation ID (value of "success" in original continuation + * from the "reserve" function). + * @param queue_priority ranking of this request in the priority queue + * @param max_queue_size at what queue size should this request be dropped + * (if other requests of higher priority are in the queue) + * @param queue_priority ranking of this request in the priority queue + * @param max_queue_size at what queue size should this request be dropped + * (if other requests of higher priority are in the queue) + * @param timeout how long to wait at most for a response + * @param cont continuation to call when done + * @param cont_cls closure for cont + * @return NULL if the entry was not queued, otherwise a handle that can be used to + * cancel; note that even if NULL is returned, the callback will be invoked + * (or rather, will already have been invoked) + */ +struct GNUNET_DATASTORE_QueueEntry * +GNUNET_DATASTORE_release_reserve (struct GNUNET_DATASTORE_Handle *h, + uint32_t rid, unsigned int queue_priority, + unsigned int max_queue_size, + struct GNUNET_TIME_Relative timeout, + GNUNET_DATASTORE_ContinuationWithStatus cont, + void *cont_cls) +{ + struct GNUNET_DATASTORE_QueueEntry *qe; + struct ReleaseReserveMessage *rrm; + union QueueContext qc; + + if (cont == NULL) + cont = &drop_status_cont; +#if DEBUG_DATASTORE + LOG (GNUNET_ERROR_TYPE_DEBUG, "Asked to release reserve %d\n", rid); +#endif + qc.sc.cont = cont; + qc.sc.cont_cls = cont_cls; + qe = make_queue_entry (h, sizeof (struct ReleaseReserveMessage), + queue_priority, max_queue_size, timeout, + &process_status_message, &qc); + if (qe == NULL) + { +#if DEBUG_DATASTORE + LOG (GNUNET_ERROR_TYPE_DEBUG, + "Could not create queue entry to release reserve\n"); +#endif + return NULL; + } + GNUNET_STATISTICS_update (h->stats, + gettext_noop + ("# RELEASE RESERVE requests executed"), 1, + GNUNET_NO); + rrm = (struct ReleaseReserveMessage *) &qe[1]; + rrm->header.type = htons (GNUNET_MESSAGE_TYPE_DATASTORE_RELEASE_RESERVE); + rrm->header.size = htons (sizeof (struct ReleaseReserveMessage)); + rrm->rid = htonl (rid); + process_queue (h); + return qe; +} + + +/** + * Update a value in the datastore. + * + * @param h handle to the datastore + * @param uid identifier for the value + * @param priority how much to increase the priority of the value + * @param expiration new expiration value should be MAX of existing and this argument + * @param queue_priority ranking of this request in the priority queue + * @param max_queue_size at what queue size should this request be dropped + * (if other requests of higher priority are in the queue) + * @param timeout how long to wait at most for a response + * @param cont continuation to call when done + * @param cont_cls closure for cont + * @return NULL if the entry was not queued, otherwise a handle that can be used to + * cancel; note that even if NULL is returned, the callback will be invoked + * (or rather, will already have been invoked) + */ +struct GNUNET_DATASTORE_QueueEntry * +GNUNET_DATASTORE_update (struct GNUNET_DATASTORE_Handle *h, uint64_t uid, + uint32_t priority, + struct GNUNET_TIME_Absolute expiration, + unsigned int queue_priority, + unsigned int max_queue_size, + struct GNUNET_TIME_Relative timeout, + GNUNET_DATASTORE_ContinuationWithStatus cont, + void *cont_cls) +{ + struct GNUNET_DATASTORE_QueueEntry *qe; + struct UpdateMessage *um; + union QueueContext qc; + + if (cont == NULL) + cont = &drop_status_cont; +#if DEBUG_DATASTORE + LOG (GNUNET_ERROR_TYPE_DEBUG, + "Asked to update entry %llu raising priority by %u and expiration to %llu\n", + uid, (unsigned int) priority, (unsigned long long) expiration.abs_value); +#endif + qc.sc.cont = cont; + qc.sc.cont_cls = cont_cls; + qe = make_queue_entry (h, sizeof (struct UpdateMessage), queue_priority, + max_queue_size, timeout, &process_status_message, &qc); + if (qe == NULL) + { +#if DEBUG_DATASTORE + LOG (GNUNET_ERROR_TYPE_DEBUG, "Could not create queue entry for UPDATE\n"); +#endif + return NULL; + } + GNUNET_STATISTICS_update (h->stats, + gettext_noop ("# UPDATE requests executed"), 1, + GNUNET_NO); + um = (struct UpdateMessage *) &qe[1]; + um->header.type = htons (GNUNET_MESSAGE_TYPE_DATASTORE_UPDATE); + um->header.size = htons (sizeof (struct UpdateMessage)); + um->priority = htonl (priority); + um->expiration = GNUNET_TIME_absolute_hton (expiration); + um->uid = GNUNET_htonll (uid); + process_queue (h); + return qe; +} + + +/** + * Explicitly remove some content from the database. + * The "cont"inuation will be called with status + * "GNUNET_OK" if content was removed, "GNUNET_NO" + * if no matching entry was found and "GNUNET_SYSERR" + * on all other types of errors. + * + * @param h handle to the datastore + * @param key key for the value + * @param size number of bytes in data + * @param data content stored + * @param queue_priority ranking of this request in the priority queue + * @param max_queue_size at what queue size should this request be dropped + * (if other requests of higher priority are in the queue) + * @param timeout how long to wait at most for a response + * @param cont continuation to call when done + * @param cont_cls closure for cont + * @return NULL if the entry was not queued, otherwise a handle that can be used to + * cancel; note that even if NULL is returned, the callback will be invoked + * (or rather, will already have been invoked) + */ +struct GNUNET_DATASTORE_QueueEntry * +GNUNET_DATASTORE_remove (struct GNUNET_DATASTORE_Handle *h, + const GNUNET_HashCode * key, size_t size, + const void *data, unsigned int queue_priority, + unsigned int max_queue_size, + struct GNUNET_TIME_Relative timeout, + GNUNET_DATASTORE_ContinuationWithStatus cont, + void *cont_cls) +{ + struct GNUNET_DATASTORE_QueueEntry *qe; + struct DataMessage *dm; + size_t msize; + union QueueContext qc; + + if (cont == NULL) + cont = &drop_status_cont; +#if DEBUG_DATASTORE + LOG (GNUNET_ERROR_TYPE_DEBUG, "Asked to remove %u bytes under key `%s'\n", + size, GNUNET_h2s (key)); +#endif + qc.sc.cont = cont; + qc.sc.cont_cls = cont_cls; + msize = sizeof (struct DataMessage) + size; + GNUNET_assert (msize < GNUNET_SERVER_MAX_MESSAGE_SIZE); + qe = make_queue_entry (h, msize, queue_priority, max_queue_size, timeout, + &process_status_message, &qc); + if (qe == NULL) + { +#if DEBUG_DATASTORE + LOG (GNUNET_ERROR_TYPE_DEBUG, "Could not create queue entry for REMOVE\n"); +#endif + return NULL; + } + GNUNET_STATISTICS_update (h->stats, + gettext_noop ("# REMOVE requests executed"), 1, + GNUNET_NO); + dm = (struct DataMessage *) &qe[1]; + dm->header.type = htons (GNUNET_MESSAGE_TYPE_DATASTORE_REMOVE); + dm->header.size = htons (msize); + dm->rid = htonl (0); + dm->size = htonl (size); + dm->type = htonl (0); + dm->priority = htonl (0); + dm->anonymity = htonl (0); + dm->uid = GNUNET_htonll (0); + dm->expiration = GNUNET_TIME_absolute_hton (GNUNET_TIME_UNIT_ZERO_ABS); + dm->key = *key; + memcpy (&dm[1], data, size); + process_queue (h); + return qe; +} + + +/** + * Type of a function to call when we receive a message + * from the service. + * + * @param cls closure + * @param msg message received, NULL on timeout or fatal error + */ +static void +process_result_message (void *cls, const struct GNUNET_MessageHeader *msg) +{ + struct GNUNET_DATASTORE_Handle *h = cls; + struct GNUNET_DATASTORE_QueueEntry *qe; + struct ResultContext rc; + const struct DataMessage *dm; + int was_transmitted; + + if (msg == NULL) + { + qe = h->queue_head; + GNUNET_assert (NULL != qe); + rc = qe->qc.rc; + was_transmitted = qe->was_transmitted; + free_queue_entry (qe); + if (was_transmitted == GNUNET_YES) + { + LOG (GNUNET_ERROR_TYPE_WARNING, + _("Failed to receive response from database.\n")); + do_disconnect (h); + } + else + { + process_queue (h); + } + if (rc.proc != NULL) + rc.proc (rc.proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, + 0); + return; + } + if (ntohs (msg->type) == GNUNET_MESSAGE_TYPE_DATASTORE_DATA_END) + { + GNUNET_break (ntohs (msg->size) == sizeof (struct GNUNET_MessageHeader)); + qe = h->queue_head; + rc = qe->qc.rc; + GNUNET_assert (GNUNET_YES == qe->was_transmitted); + free_queue_entry (qe); +#if DEBUG_DATASTORE + LOG (GNUNET_ERROR_TYPE_DEBUG, + "Received end of result set, new queue size is %u\n", h->queue_size); +#endif + h->retry_time.rel_value = 0; + h->result_count = 0; + process_queue (h); + if (rc.proc != NULL) + rc.proc (rc.proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, + 0); + return; + } + qe = h->queue_head; + GNUNET_assert (NULL != qe); + rc = qe->qc.rc; + if (GNUNET_YES != qe->was_transmitted) + { + GNUNET_break (0); + free_queue_entry (qe); + h->retry_time = GNUNET_TIME_UNIT_ZERO; + do_disconnect (h); + if (rc.proc != NULL) + rc.proc (rc.proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, + 0); + return; + } + if ((ntohs (msg->size) < sizeof (struct DataMessage)) || + (ntohs (msg->type) != GNUNET_MESSAGE_TYPE_DATASTORE_DATA) || + (ntohs (msg->size) != + sizeof (struct DataMessage) + + ntohl (((const struct DataMessage *) msg)->size))) + { + GNUNET_break (0); + free_queue_entry (qe); + h->retry_time = GNUNET_TIME_UNIT_ZERO; + do_disconnect (h); + if (rc.proc != NULL) + rc.proc (rc.proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, + 0); + return; + } + GNUNET_STATISTICS_update (h->stats, gettext_noop ("# Results received"), 1, + GNUNET_NO); + dm = (const struct DataMessage *) msg; +#if DEBUG_DATASTORE + LOG (GNUNET_ERROR_TYPE_DEBUG, + "Received result %llu with type %u and size %u with key %s\n", + (unsigned long long) GNUNET_ntohll (dm->uid), ntohl (dm->type), + ntohl (dm->size), GNUNET_h2s (&dm->key)); +#endif + free_queue_entry (qe); + h->retry_time.rel_value = 0; + process_queue (h); + if (rc.proc != NULL) + rc.proc (rc.proc_cls, &dm->key, ntohl (dm->size), &dm[1], ntohl (dm->type), + ntohl (dm->priority), ntohl (dm->anonymity), + GNUNET_TIME_absolute_ntoh (dm->expiration), + GNUNET_ntohll (dm->uid)); +} + + +/** + * Get a random value from the datastore for content replication. + * Returns a single, random value among those with the highest + * replication score, lowering positive replication scores by one for + * the chosen value (if only content with a replication score exists, + * a random value is returned and replication scores are not changed). + * + * @param h handle to the datastore + * @param queue_priority ranking of this request in the priority queue + * @param max_queue_size at what queue size should this request be dropped + * (if other requests of higher priority are in the queue) + * @param timeout how long to wait at most for a response + * @param proc function to call on a random value; it + * will be called once with a value (if available) + * and always once with a value of NULL. + * @param proc_cls closure for proc + * @return NULL if the entry was not queued, otherwise a handle that can be used to + * cancel + */ +struct GNUNET_DATASTORE_QueueEntry * +GNUNET_DATASTORE_get_for_replication (struct GNUNET_DATASTORE_Handle *h, + unsigned int queue_priority, + unsigned int max_queue_size, + struct GNUNET_TIME_Relative timeout, + GNUNET_DATASTORE_DatumProcessor proc, + void *proc_cls) +{ + struct GNUNET_DATASTORE_QueueEntry *qe; + struct GNUNET_MessageHeader *m; + union QueueContext qc; + + GNUNET_assert (NULL != proc); +#if DEBUG_DATASTORE + LOG (GNUNET_ERROR_TYPE_DEBUG, "Asked to get replication entry in %llu ms\n", + (unsigned long long) timeout.rel_value); +#endif + qc.rc.proc = proc; + qc.rc.proc_cls = proc_cls; + qe = make_queue_entry (h, sizeof (struct GNUNET_MessageHeader), + queue_priority, max_queue_size, timeout, + &process_result_message, &qc); + if (qe == NULL) + { +#if DEBUG_DATASTORE + LOG (GNUNET_ERROR_TYPE_DEBUG, + "Could not create queue entry for GET REPLICATION\n"); +#endif + return NULL; + } + GNUNET_STATISTICS_update (h->stats, + gettext_noop + ("# GET REPLICATION requests executed"), 1, + GNUNET_NO); + m = (struct GNUNET_MessageHeader *) &qe[1]; + m->type = htons (GNUNET_MESSAGE_TYPE_DATASTORE_GET_REPLICATION); + m->size = htons (sizeof (struct GNUNET_MessageHeader)); + process_queue (h); + return qe; +} + + +/** + * Get a single zero-anonymity value from the datastore. + * + * @param h handle to the datastore + * @param offset offset of the result (modulo num-results); set to + * a random 64-bit value initially; then increment by + * one each time; detect that all results have been found by uid + * being again the first uid ever returned. + * @param queue_priority ranking of this request in the priority queue + * @param max_queue_size at what queue size should this request be dropped + * (if other requests of higher priority are in the queue) + * @param timeout how long to wait at most for a response + * @param type allowed type for the operation (never zero) + * @param proc function to call on a random value; it + * will be called once with a value (if available) + * or with NULL if none value exists. + * @param proc_cls closure for proc + * @return NULL if the entry was not queued, otherwise a handle that can be used to + * cancel + */ +struct GNUNET_DATASTORE_QueueEntry * +GNUNET_DATASTORE_get_zero_anonymity (struct GNUNET_DATASTORE_Handle *h, + uint64_t offset, + unsigned int queue_priority, + unsigned int max_queue_size, + struct GNUNET_TIME_Relative timeout, + enum GNUNET_BLOCK_Type type, + GNUNET_DATASTORE_DatumProcessor proc, + void *proc_cls) +{ + struct GNUNET_DATASTORE_QueueEntry *qe; + struct GetZeroAnonymityMessage *m; + union QueueContext qc; + + GNUNET_assert (NULL != proc); + GNUNET_assert (type != GNUNET_BLOCK_TYPE_ANY); +#if DEBUG_DATASTORE + LOG (GNUNET_ERROR_TYPE_DEBUG, + "Asked to get %llu-th zero-anonymity entry of type %d in %llu ms\n", + (unsigned long long) offset, type, + (unsigned long long) timeout.rel_value); +#endif + qc.rc.proc = proc; + qc.rc.proc_cls = proc_cls; + qe = make_queue_entry (h, sizeof (struct GetZeroAnonymityMessage), + queue_priority, max_queue_size, timeout, + &process_result_message, &qc); + if (qe == NULL) + { +#if DEBUG_DATASTORE + LOG (GNUNET_ERROR_TYPE_DEBUG, + "Could not create queue entry for zero-anonymity procation\n"); +#endif + return NULL; + } + GNUNET_STATISTICS_update (h->stats, + gettext_noop + ("# GET ZERO ANONYMITY requests executed"), 1, + GNUNET_NO); + m = (struct GetZeroAnonymityMessage *) &qe[1]; + m->header.type = htons (GNUNET_MESSAGE_TYPE_DATASTORE_GET_ZERO_ANONYMITY); + m->header.size = htons (sizeof (struct GetZeroAnonymityMessage)); + m->type = htonl ((uint32_t) type); + m->offset = GNUNET_htonll (offset); + process_queue (h); + return qe; +} + + +/** + * Get a result for a particular key from the datastore. The processor + * will only be called once. + * + * @param h handle to the datastore + * @param offset offset of the result (modulo num-results); set to + * a random 64-bit value initially; then increment by + * one each time; detect that all results have been found by uid + * being again the first uid ever returned. + * @param key maybe NULL (to match all entries) + * @param type desired type, 0 for any + * @param queue_priority ranking of this request in the priority queue + * @param max_queue_size at what queue size should this request be dropped + * (if other requests of higher priority are in the queue) + * @param timeout how long to wait at most for a response + * @param proc function to call on each matching value; + * will be called once with a NULL value at the end + * @param proc_cls closure for proc + * @return NULL if the entry was not queued, otherwise a handle that can be used to + * cancel + */ +struct GNUNET_DATASTORE_QueueEntry * +GNUNET_DATASTORE_get_key (struct GNUNET_DATASTORE_Handle *h, uint64_t offset, + const GNUNET_HashCode * key, + enum GNUNET_BLOCK_Type type, + unsigned int queue_priority, + unsigned int max_queue_size, + struct GNUNET_TIME_Relative timeout, + GNUNET_DATASTORE_DatumProcessor proc, void *proc_cls) +{ + struct GNUNET_DATASTORE_QueueEntry *qe; + struct GetMessage *gm; + union QueueContext qc; + + GNUNET_assert (NULL != proc); +#if DEBUG_DATASTORE + LOG (GNUNET_ERROR_TYPE_DEBUG, + "Asked to look for data of type %u under key `%s'\n", + (unsigned int) type, GNUNET_h2s (key)); +#endif + qc.rc.proc = proc; + qc.rc.proc_cls = proc_cls; + qe = make_queue_entry (h, sizeof (struct GetMessage), queue_priority, + max_queue_size, timeout, &process_result_message, &qc); + if (qe == NULL) + { +#if DEBUG_DATASTORE + LOG (GNUNET_ERROR_TYPE_DEBUG, "Could not queue request for `%s'\n", + GNUNET_h2s (key)); +#endif + return NULL; + } + GNUNET_STATISTICS_update (h->stats, gettext_noop ("# GET requests executed"), + 1, GNUNET_NO); + gm = (struct GetMessage *) &qe[1]; + gm->header.type = htons (GNUNET_MESSAGE_TYPE_DATASTORE_GET); + gm->type = htonl (type); + gm->offset = GNUNET_htonll (offset); + if (key != NULL) + { + gm->header.size = htons (sizeof (struct GetMessage)); + gm->key = *key; + } + else + { + gm->header.size = + htons (sizeof (struct GetMessage) - sizeof (GNUNET_HashCode)); + } + process_queue (h); + return qe; +} + + +/** + * Cancel a datastore operation. The final callback from the + * operation must not have been done yet. + * + * @param qe operation to cancel + */ +void +GNUNET_DATASTORE_cancel (struct GNUNET_DATASTORE_QueueEntry *qe) +{ + struct GNUNET_DATASTORE_Handle *h; + + GNUNET_assert (GNUNET_SYSERR != qe->was_transmitted); + h = qe->h; +#if DEBUG_DATASTORE + LOG (GNUNET_ERROR_TYPE_DEBUG, + "Pending DATASTORE request %p cancelled (%d, %d)\n", qe, + qe->was_transmitted, h->queue_head == qe); +#endif + if (GNUNET_YES == qe->was_transmitted) + { + free_queue_entry (qe); + h->skip_next_messages++; + return; + } + free_queue_entry (qe); + process_queue (h); +} + + +/* end of datastore_api.c */ diff --git a/src/datastore/gnunet-service-datastore.c b/src/datastore/gnunet-service-datastore.c new file mode 100644 index 0000000..1d7e8cd --- /dev/null +++ b/src/datastore/gnunet-service-datastore.c @@ -0,0 +1,1693 @@ +/* + This file is part of GNUnet + (C) 2004, 2005, 2006, 2007, 2009 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 2, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file datastore/gnunet-service-datastore.c + * @brief Management for the datastore for files stored on a GNUnet node + * @author Christian Grothoff + */ + +#include "platform.h" +#include "gnunet_util_lib.h" +#include "gnunet_protocols.h" +#include "gnunet_statistics_service.h" +#include "gnunet_datastore_plugin.h" +#include "datastore.h" + +/** + * How many messages do we queue at most per client? + */ +#define MAX_PENDING 1024 + +/** + * How long are we at most keeping "expired" content + * past the expiration date in the database? + */ +#define MAX_EXPIRE_DELAY GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15) + +/** + * How fast are we allowed to query the database for deleting + * expired content? (1 item per second). + */ +#define MIN_EXPIRE_DELAY GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 1) + +/** + * Name under which we store current space consumption. + */ +static char *quota_stat_name; + +/** + * After how many payload-changing operations + * do we sync our statistics? + */ +#define MAX_STAT_SYNC_LAG 50 + + +/** + * Our datastore plugin. + */ +struct DatastorePlugin +{ + + /** + * API of the transport as returned by the plugin's + * initialization function. + */ + struct GNUNET_DATASTORE_PluginFunctions *api; + + /** + * Short name for the plugin (i.e. "sqlite"). + */ + char *short_name; + + /** + * Name of the library (i.e. "gnunet_plugin_datastore_sqlite"). + */ + char *lib_name; + + /** + * Environment this transport service is using + * for this plugin. + */ + struct GNUNET_DATASTORE_PluginEnvironment env; + +}; + + +/** + * Linked list of active reservations. + */ +struct ReservationList +{ + + /** + * This is a linked list. + */ + struct ReservationList *next; + + /** + * Client that made the reservation. + */ + struct GNUNET_SERVER_Client *client; + + /** + * Number of bytes (still) reserved. + */ + uint64_t amount; + + /** + * Number of items (still) reserved. + */ + uint64_t entries; + + /** + * Reservation identifier. + */ + int32_t rid; + +}; + + + +/** + * Our datastore plugin (NULL if not available). + */ +static struct DatastorePlugin *plugin; + +/** + * Linked list of space reservations made by clients. + */ +static struct ReservationList *reservations; + +/** + * Bloomfilter to quickly tell if we don't have the content. + */ +static struct GNUNET_CONTAINER_BloomFilter *filter; + +/** + * How much space are we allowed to use? + */ +static unsigned long long quota; + +/** + * Should the database be dropped on exit? + */ +static int do_drop; + +/** + * Name of our plugin. + */ +static char *plugin_name; + +/** + * How much space are we using for the cache? (space available for + * insertions that will be instantly reclaimed by discarding less + * important content --- or possibly whatever we just inserted into + * the "cache"). + */ +static unsigned long long cache_size; + +/** + * How much space have we currently reserved? + */ +static unsigned long long reserved; + +/** + * How much data are we currently storing + * in the database? + */ +static unsigned long long payload; + +/** + * Number of updates that were made to the + * payload value since we last synchronized + * it with the statistics service. + */ +static unsigned int lastSync; + +/** + * Did we get an answer from statistics? + */ +static int stats_worked; + +/** + * Identity of the task that is used to delete + * expired content. + */ +static GNUNET_SCHEDULER_TaskIdentifier expired_kill_task; + +/** + * Our configuration. + */ +const struct GNUNET_CONFIGURATION_Handle *cfg; + +/** + * Minimum time that content should have to not be discarded instantly + * (time stamp of any content that we've been discarding recently to + * stay below the quota). FOREVER if we had to expire content with + * non-zero priority. + */ +static struct GNUNET_TIME_Absolute min_expiration; + +/** + * Handle for reporting statistics. + */ +static struct GNUNET_STATISTICS_Handle *stats; + + +/** + * Synchronize our utilization statistics with the + * statistics service. + */ +static void +sync_stats () +{ + GNUNET_STATISTICS_set (stats, quota_stat_name, payload, GNUNET_YES); + GNUNET_STATISTICS_set (stats, "# utilization by current datastore", payload, GNUNET_NO); + lastSync = 0; +} + + + +/** + * Context for transmitting replies to clients. + */ +struct TransmitCallbackContext +{ + + /** + * We keep these in a doubly-linked list (for cleanup). + */ + struct TransmitCallbackContext *next; + + /** + * We keep these in a doubly-linked list (for cleanup). + */ + struct TransmitCallbackContext *prev; + + /** + * The message that we're asked to transmit. + */ + struct GNUNET_MessageHeader *msg; + + /** + * Handle for the transmission request. + */ + struct GNUNET_CONNECTION_TransmitHandle *th; + + /** + * Client that we are transmitting to. + */ + struct GNUNET_SERVER_Client *client; + +}; + + +/** + * Head of the doubly-linked list (for cleanup). + */ +static struct TransmitCallbackContext *tcc_head; + +/** + * Tail of the doubly-linked list (for cleanup). + */ +static struct TransmitCallbackContext *tcc_tail; + +/** + * Have we already cleaned up the TCCs and are hence no longer + * willing (or able) to transmit anything to anyone? + */ +static int cleaning_done; + +/** + * Handle for pending get request. + */ +static struct GNUNET_STATISTICS_GetHandle *stat_get; + + +/** + * Task that is used to remove expired entries from + * the datastore. This task will schedule itself + * again automatically to always delete all expired + * content quickly. + * + * @param cls not used + * @param tc task context + */ +static void +delete_expired (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc); + + +/** + * Iterate over the expired items stored in the datastore. + * Delete all expired items; once we have processed all + * expired items, re-schedule the "delete_expired" task. + * + * @param cls not used + * @param key key for the content + * @param size number of bytes in data + * @param data content stored + * @param type type of the content + * @param priority priority of the content + * @param anonymity anonymity-level for the content + * @param expiration expiration time for the content + * @param uid unique identifier for the datum; + * maybe 0 if no unique identifier is available + * + * @return GNUNET_SYSERR to abort the iteration, GNUNET_OK to continue + * (continue on call to "next", of course), + * GNUNET_NO to delete the item and continue (if supported) + */ +static int +expired_processor (void *cls, const GNUNET_HashCode * key, uint32_t size, + const void *data, enum GNUNET_BLOCK_Type type, + uint32_t priority, uint32_t anonymity, + struct GNUNET_TIME_Absolute expiration, uint64_t uid) +{ + struct GNUNET_TIME_Absolute now; + + if (key == NULL) + { + expired_kill_task = + GNUNET_SCHEDULER_add_delayed_with_priority (MAX_EXPIRE_DELAY, + GNUNET_SCHEDULER_PRIORITY_IDLE, + &delete_expired, NULL); + return GNUNET_SYSERR; + } + now = GNUNET_TIME_absolute_get (); + if (expiration.abs_value > now.abs_value) + { + /* finished processing */ + expired_kill_task = + GNUNET_SCHEDULER_add_delayed_with_priority (MAX_EXPIRE_DELAY, + GNUNET_SCHEDULER_PRIORITY_IDLE, + &delete_expired, NULL); + return GNUNET_SYSERR; + } +#if DEBUG_DATASTORE + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Deleting content `%s' of type %u that expired %llu ms ago\n", + GNUNET_h2s (key), type, + (unsigned long long) (now.abs_value - expiration.abs_value)); +#endif + min_expiration = now; + GNUNET_STATISTICS_update (stats, gettext_noop ("# bytes expired"), size, + GNUNET_YES); + GNUNET_CONTAINER_bloomfilter_remove (filter, key); + expired_kill_task = + GNUNET_SCHEDULER_add_delayed_with_priority (MIN_EXPIRE_DELAY, + GNUNET_SCHEDULER_PRIORITY_IDLE, + &delete_expired, NULL); + return GNUNET_NO; +} + + +/** + * Task that is used to remove expired entries from + * the datastore. This task will schedule itself + * again automatically to always delete all expired + * content quickly. + * + * @param cls not used + * @param tc task context + */ +static void +delete_expired (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + expired_kill_task = GNUNET_SCHEDULER_NO_TASK; + plugin->api->get_expiration (plugin->api->cls, &expired_processor, NULL); +} + + +/** + * An iterator over a set of items stored in the datastore + * that deletes until we're happy with respect to our quota. + * + * @param cls closure + * @param key key for the content + * @param size number of bytes in data + * @param data content stored + * @param type type of the content + * @param priority priority of the content + * @param anonymity anonymity-level for the content + * @param expiration expiration time for the content + * @param uid unique identifier for the datum; + * maybe 0 if no unique identifier is available + * + * @return GNUNET_SYSERR to abort the iteration, GNUNET_OK to continue + * (continue on call to "next", of course), + * GNUNET_NO to delete the item and continue (if supported) + */ +static int +quota_processor (void *cls, const GNUNET_HashCode * key, uint32_t size, + const void *data, enum GNUNET_BLOCK_Type type, + uint32_t priority, uint32_t anonymity, + struct GNUNET_TIME_Absolute expiration, uint64_t uid) +{ + unsigned long long *need = cls; + + if (NULL == key) + return GNUNET_SYSERR; +#if DEBUG_DATASTORE + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Deleting %llu bytes of low-priority (%u) content `%s' of type %u at %llu ms prior to expiration (still trying to free another %llu bytes)\n", + (unsigned long long) (size + GNUNET_DATASTORE_ENTRY_OVERHEAD), + (unsigned int) priority, + GNUNET_h2s (key), type, + (unsigned long long) GNUNET_TIME_absolute_get_remaining (expiration).rel_value, + *need); +#endif + if (size + GNUNET_DATASTORE_ENTRY_OVERHEAD > *need) + *need = 0; + else + *need -= size + GNUNET_DATASTORE_ENTRY_OVERHEAD; + if (priority > 0) + min_expiration = GNUNET_TIME_UNIT_FOREVER_ABS; + else + min_expiration = expiration; + GNUNET_STATISTICS_update (stats, + gettext_noop ("# bytes purged (low-priority)"), + size, GNUNET_YES); + GNUNET_CONTAINER_bloomfilter_remove (filter, key); + return GNUNET_NO; +} + + +/** + * Manage available disk space by running tasks + * that will discard content if necessary. This + * function will be run whenever a request for + * "need" bytes of storage could only be satisfied + * by eating into the "cache" (and we want our cache + * space back). + * + * @param need number of bytes of content that were + * placed into the "cache" (and hence the + * number of bytes that should be removed). + */ +static void +manage_space (unsigned long long need) +{ + unsigned long long last; + +#if DEBUG_DATASTORE + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Asked to free up %llu bytes of cache space\n", need); +#endif + last = 0; + while ((need > 0) && (last != need)) + { + last = need; + plugin->api->get_expiration (plugin->api->cls, "a_processor, &need); + } +} + + +/** + * Function called to notify a client about the socket + * begin ready to queue more data. "buf" will be + * NULL and "size" zero if the socket was closed for + * writing in the meantime. + * + * @param cls closure + * @param size number of bytes available in buf + * @param buf where the callee should write the message + * @return number of bytes written to buf + */ +static size_t +transmit_callback (void *cls, size_t size, void *buf) +{ + struct TransmitCallbackContext *tcc = cls; + size_t msize; + + tcc->th = NULL; + GNUNET_CONTAINER_DLL_remove (tcc_head, tcc_tail, tcc); + msize = ntohs (tcc->msg->size); + if (size == 0) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _("Transmission to client failed!\n")); + GNUNET_SERVER_receive_done (tcc->client, GNUNET_SYSERR); + GNUNET_SERVER_client_drop (tcc->client); + GNUNET_free (tcc->msg); + GNUNET_free (tcc); + return 0; + } + GNUNET_assert (size >= msize); + memcpy (buf, tcc->msg, msize); + GNUNET_SERVER_receive_done (tcc->client, GNUNET_OK); + GNUNET_SERVER_client_drop (tcc->client); + GNUNET_free (tcc->msg); + GNUNET_free (tcc); + return msize; +} + + +/** + * Transmit the given message to the client. + * + * @param client target of the message + * @param msg message to transmit, will be freed! + */ +static void +transmit (struct GNUNET_SERVER_Client *client, struct GNUNET_MessageHeader *msg) +{ + struct TransmitCallbackContext *tcc; + + if (GNUNET_YES == cleaning_done) + { +#if DEBUG_DATASTORE + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + "Shutdown in progress, aborting transmission.\n"); +#endif + GNUNET_SERVER_receive_done (client, GNUNET_SYSERR); + GNUNET_free (msg); + return; + } + tcc = GNUNET_malloc (sizeof (struct TransmitCallbackContext)); + tcc->msg = msg; + tcc->client = client; + if (NULL == + (tcc->th = + GNUNET_SERVER_notify_transmit_ready (client, ntohs (msg->size), + GNUNET_TIME_UNIT_FOREVER_REL, + &transmit_callback, tcc))) + { + GNUNET_break (0); + GNUNET_SERVER_receive_done (client, GNUNET_SYSERR); + GNUNET_free (msg); + GNUNET_free (tcc); + return; + } + GNUNET_SERVER_client_keep (client); + GNUNET_CONTAINER_DLL_insert (tcc_head, tcc_tail, tcc); +} + + +/** + * Transmit a status code to the client. + * + * @param client receiver of the response + * @param code status code + * @param msg optional error message (can be NULL) + */ +static void +transmit_status (struct GNUNET_SERVER_Client *client, int code, const char *msg) +{ + struct StatusMessage *sm; + size_t slen; + +#if DEBUG_DATASTORE + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Transmitting `%s' message with value %d and message `%s'\n", + "STATUS", code, msg != NULL ? msg : "(none)"); +#endif + slen = (msg == NULL) ? 0 : strlen (msg) + 1; + sm = GNUNET_malloc (sizeof (struct StatusMessage) + slen); + sm->header.size = htons (sizeof (struct StatusMessage) + slen); + sm->header.type = htons (GNUNET_MESSAGE_TYPE_DATASTORE_STATUS); + sm->status = htonl (code); + sm->min_expiration = GNUNET_TIME_absolute_hton (min_expiration); + if (slen > 0) + memcpy (&sm[1], msg, slen); + transmit (client, &sm->header); +} + + + +/** + * Function that will transmit the given datastore entry + * to the client. + * + * @param cls closure, pointer to the client (of type GNUNET_SERVER_Client). + * @param key key for the content + * @param size number of bytes in data + * @param data content stored + * @param type type of the content + * @param priority priority of the content + * @param anonymity anonymity-level for the content + * @param expiration expiration time for the content + * @param uid unique identifier for the datum; + * maybe 0 if no unique identifier is available + * + * @return GNUNET_SYSERR to abort the iteration, GNUNET_OK to continue, + * GNUNET_NO to delete the item and continue (if supported) + */ +static int +transmit_item (void *cls, const GNUNET_HashCode * key, uint32_t size, + const void *data, enum GNUNET_BLOCK_Type type, uint32_t priority, + uint32_t anonymity, struct GNUNET_TIME_Absolute expiration, + uint64_t uid) +{ + struct GNUNET_SERVER_Client *client = cls; + struct GNUNET_MessageHeader *end; + struct DataMessage *dm; + + if (key == NULL) + { + /* transmit 'DATA_END' */ +#if DEBUG_DATASTORE + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Transmitting `%s' message\n", + "DATA_END"); +#endif + end = GNUNET_malloc (sizeof (struct GNUNET_MessageHeader)); + end->size = htons (sizeof (struct GNUNET_MessageHeader)); + end->type = htons (GNUNET_MESSAGE_TYPE_DATASTORE_DATA_END); + transmit (client, end); + GNUNET_SERVER_client_drop (client); + return GNUNET_OK; + } + GNUNET_assert (sizeof (struct DataMessage) + size < + GNUNET_SERVER_MAX_MESSAGE_SIZE); + dm = GNUNET_malloc (sizeof (struct DataMessage) + size); + dm->header.size = htons (sizeof (struct DataMessage) + size); + dm->header.type = htons (GNUNET_MESSAGE_TYPE_DATASTORE_DATA); + dm->rid = htonl (0); + dm->size = htonl (size); + dm->type = htonl (type); + dm->priority = htonl (priority); + dm->anonymity = htonl (anonymity); + dm->replication = htonl (0); + dm->reserved = htonl (0); + dm->expiration = GNUNET_TIME_absolute_hton (expiration); + dm->uid = GNUNET_htonll (uid); + dm->key = *key; + memcpy (&dm[1], data, size); +#if DEBUG_DATASTORE + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Transmitting `%s' message for `%s' of type %u with expiration %llu (now: %llu)\n", + "DATA", GNUNET_h2s (key), type, + (unsigned long long) expiration.abs_value, + (unsigned long long) GNUNET_TIME_absolute_get ().abs_value); +#endif + GNUNET_STATISTICS_update (stats, gettext_noop ("# results found"), 1, + GNUNET_NO); + transmit (client, &dm->header); + GNUNET_SERVER_client_drop (client); + return GNUNET_OK; +} + + +/** + * Handle RESERVE-message. + * + * @param cls closure + * @param client identification of the client + * @param message the actual message + */ +static void +handle_reserve (void *cls, struct GNUNET_SERVER_Client *client, + const struct GNUNET_MessageHeader *message) +{ + /** + * Static counter to produce reservation identifiers. + */ + static int reservation_gen; + + const struct ReserveMessage *msg = (const struct ReserveMessage *) message; + struct ReservationList *e; + unsigned long long used; + unsigned long long req; + uint64_t amount; + uint32_t entries; + +#if DEBUG_DATASTORE + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Processing `%s' request\n", "RESERVE"); +#endif + amount = GNUNET_ntohll (msg->amount); + entries = ntohl (msg->entries); + used = payload + reserved; + req = + amount + ((unsigned long long) GNUNET_DATASTORE_ENTRY_OVERHEAD) * entries; + if (used + req > quota) + { + if (quota < used) + used = quota; /* cheat a bit for error message (to avoid negative numbers) */ + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _ + ("Insufficient space (%llu bytes are available) to satisfy `%s' request for %llu bytes\n"), + quota - used, "RESERVE", req); + if (cache_size < req) + { + /* TODO: document this in the FAQ; essentially, if this + * message happens, the insertion request could be blocked + * by less-important content from migration because it is + * larger than 1/8th of the overall available space, and + * we only reserve 1/8th for "fresh" insertions */ + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _ + ("The requested amount (%llu bytes) is larger than the cache size (%llu bytes)\n"), + req, cache_size); + transmit_status (client, 0, + gettext_noop + ("Insufficient space to satisfy request and " + "requested amount is larger than cache size")); + } + else + { + transmit_status (client, 0, + gettext_noop ("Insufficient space to satisfy request")); + } + return; + } + reserved += req; + GNUNET_STATISTICS_set (stats, gettext_noop ("# reserved"), reserved, + GNUNET_NO); + e = GNUNET_malloc (sizeof (struct ReservationList)); + e->next = reservations; + reservations = e; + e->client = client; + e->amount = amount; + e->entries = entries; + e->rid = ++reservation_gen; + if (reservation_gen < 0) + reservation_gen = 0; /* wrap around */ + transmit_status (client, e->rid, NULL); +} + + +/** + * Handle RELEASE_RESERVE-message. + * + * @param cls closure + * @param client identification of the client + * @param message the actual message + */ +static void +handle_release_reserve (void *cls, struct GNUNET_SERVER_Client *client, + const struct GNUNET_MessageHeader *message) +{ + const struct ReleaseReserveMessage *msg = + (const struct ReleaseReserveMessage *) message; + struct ReservationList *pos; + struct ReservationList *prev; + struct ReservationList *next; + int rid = ntohl (msg->rid); + unsigned long long rem; + +#if DEBUG_DATASTORE + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Processing `%s' request\n", + "RELEASE_RESERVE"); +#endif + next = reservations; + prev = NULL; + while (NULL != (pos = next)) + { + next = pos->next; + if (rid == pos->rid) + { + if (prev == NULL) + reservations = next; + else + prev->next = next; + rem = + pos->amount + + ((unsigned long long) GNUNET_DATASTORE_ENTRY_OVERHEAD) * pos->entries; + GNUNET_assert (reserved >= rem); + reserved -= rem; + GNUNET_STATISTICS_set (stats, gettext_noop ("# reserved"), reserved, + GNUNET_NO); +#if DEBUG_DATASTORE + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Returning %llu remaining reserved bytes to storage pool\n", + rem); +#endif + GNUNET_free (pos); + transmit_status (client, GNUNET_OK, NULL); + return; + } + prev = pos; + } + GNUNET_break (0); + transmit_status (client, GNUNET_SYSERR, + gettext_noop ("Could not find matching reservation")); +} + + +/** + * Check that the given message is a valid data message. + * + * @return NULL if the message is not well-formed, otherwise the message + */ +static const struct DataMessage * +check_data (const struct GNUNET_MessageHeader *message) +{ + uint16_t size; + uint32_t dsize; + const struct DataMessage *dm; + + size = ntohs (message->size); + if (size < sizeof (struct DataMessage)) + { + GNUNET_break (0); + return NULL; + } + dm = (const struct DataMessage *) message; + dsize = ntohl (dm->size); + if (size != dsize + sizeof (struct DataMessage)) + { + GNUNET_break (0); + return NULL; + } + return dm; +} + + +/** + * Context for a PUT request used to see if the content is + * already present. + */ +struct PutContext +{ + /** + * Client to notify on completion. + */ + struct GNUNET_SERVER_Client *client; + +#if ! HAVE_UNALIGNED_64_ACCESS + void *reserved; +#endif + + /* followed by the 'struct DataMessage' */ +}; + + +/** + * Actually put the data message. + * + * @param client sender of the message + * @param dm message with the data to store + */ +static void +execute_put (struct GNUNET_SERVER_Client *client, const struct DataMessage *dm) +{ + uint32_t size; + char *msg; + int ret; + + size = ntohl (dm->size); + msg = NULL; + ret = + plugin->api->put (plugin->api->cls, &dm->key, size, &dm[1], + ntohl (dm->type), ntohl (dm->priority), + ntohl (dm->anonymity), ntohl (dm->replication), + GNUNET_TIME_absolute_ntoh (dm->expiration), &msg); + if (GNUNET_OK == ret) + { + GNUNET_STATISTICS_update (stats, gettext_noop ("# bytes stored"), size, + GNUNET_YES); + GNUNET_CONTAINER_bloomfilter_add (filter, &dm->key); +#if DEBUG_DATASTORE + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Successfully stored %u bytes of type %u under key `%s'\n", + size, ntohl (dm->type), GNUNET_h2s (&dm->key)); +#endif + } + transmit_status (client, ret, msg); + GNUNET_free_non_null (msg); + if (quota - reserved - cache_size < payload) + { + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + _("Need %llu bytes more space (%llu allowed, using %llu)\n"), + (unsigned long long) size + GNUNET_DATASTORE_ENTRY_OVERHEAD, + (unsigned long long) (quota - reserved - cache_size), + (unsigned long long) payload); + manage_space (size + GNUNET_DATASTORE_ENTRY_OVERHEAD); + } +} + + +/** + * Function that will check if the given datastore entry + * matches the put and if none match executes the put. + * + * @param cls closure, pointer to the client (of type 'struct PutContext'). + * @param key key for the content + * @param size number of bytes in data + * @param data content stored + * @param type type of the content + * @param priority priority of the content + * @param anonymity anonymity-level for the content + * @param expiration expiration time for the content + * @param uid unique identifier for the datum; + * maybe 0 if no unique identifier is available + * + * @return GNUNET_OK usually + * GNUNET_NO to delete the item + */ +static int +check_present (void *cls, const GNUNET_HashCode * key, uint32_t size, + const void *data, enum GNUNET_BLOCK_Type type, uint32_t priority, + uint32_t anonymity, struct GNUNET_TIME_Absolute expiration, + uint64_t uid) +{ + struct PutContext *pc = cls; + const struct DataMessage *dm; + + dm = (const struct DataMessage *) &pc[1]; + if (key == NULL) + { + execute_put (pc->client, dm); + GNUNET_SERVER_client_drop (pc->client); + GNUNET_free (pc); + return GNUNET_OK; + } + if ((GNUNET_BLOCK_TYPE_FS_DBLOCK == type) || + (GNUNET_BLOCK_TYPE_FS_IBLOCK == type) || ((size == ntohl (dm->size)) && + (0 == + memcmp (&dm[1], data, size)))) + { +#if DEBUG_MYSQL + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Result already present in datastore\n"); +#endif + /* FIXME: change API to allow increasing 'replication' counter */ + if ((ntohl (dm->priority) > 0) || + (GNUNET_TIME_absolute_ntoh (dm->expiration).abs_value > + expiration.abs_value)) + plugin->api->update (plugin->api->cls, uid, + (int32_t) ntohl (dm->priority), + GNUNET_TIME_absolute_ntoh (dm->expiration), NULL); + transmit_status (pc->client, GNUNET_NO, NULL); + GNUNET_SERVER_client_drop (pc->client); + GNUNET_free (pc); + } + else + { + execute_put (pc->client, dm); + GNUNET_SERVER_client_drop (pc->client); + GNUNET_free (pc); + } + return GNUNET_OK; +} + + +/** + * Handle PUT-message. + * + * @param cls closure + * @param client identification of the client + * @param message the actual message + */ +static void +handle_put (void *cls, struct GNUNET_SERVER_Client *client, + const struct GNUNET_MessageHeader *message) +{ + const struct DataMessage *dm = check_data (message); + int rid; + struct ReservationList *pos; + struct PutContext *pc; + GNUNET_HashCode vhash; + uint32_t size; + + if ((dm == NULL) || (ntohl (dm->type) == 0)) + { + GNUNET_break (0); + GNUNET_SERVER_receive_done (client, GNUNET_SYSERR); + return; + } +#if DEBUG_DATASTORE + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Processing `%s' request for `%s' of type %u\n", "PUT", + GNUNET_h2s (&dm->key), ntohl (dm->type)); +#endif + rid = ntohl (dm->rid); + size = ntohl (dm->size); + if (rid > 0) + { + pos = reservations; + while ((NULL != pos) && (rid != pos->rid)) + pos = pos->next; + GNUNET_break (pos != NULL); + if (NULL != pos) + { + GNUNET_break (pos->entries > 0); + GNUNET_break (pos->amount >= size); + pos->entries--; + pos->amount -= size; + reserved -= (size + GNUNET_DATASTORE_ENTRY_OVERHEAD); + GNUNET_STATISTICS_set (stats, gettext_noop ("# reserved"), reserved, + GNUNET_NO); + } + } + if (GNUNET_YES == GNUNET_CONTAINER_bloomfilter_test (filter, &dm->key)) + { + GNUNET_CRYPTO_hash (&dm[1], size, &vhash); + pc = GNUNET_malloc (sizeof (struct PutContext) + size + + sizeof (struct DataMessage)); + pc->client = client; + GNUNET_SERVER_client_keep (client); + memcpy (&pc[1], dm, size + sizeof (struct DataMessage)); + plugin->api->get_key (plugin->api->cls, 0, &dm->key, &vhash, + ntohl (dm->type), &check_present, pc); + return; + } + execute_put (client, dm); +} + + +/** + * Handle GET-message. + * + * @param cls closure + * @param client identification of the client + * @param message the actual message + */ +static void +handle_get (void *cls, struct GNUNET_SERVER_Client *client, + const struct GNUNET_MessageHeader *message) +{ + const struct GetMessage *msg; + uint16_t size; + + size = ntohs (message->size); + if ((size != sizeof (struct GetMessage)) && + (size != sizeof (struct GetMessage) - sizeof (GNUNET_HashCode))) + { + GNUNET_break (0); + GNUNET_SERVER_receive_done (client, GNUNET_SYSERR); + return; + } + msg = (const struct GetMessage *) message; +#if DEBUG_DATASTORE + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Processing `%s' request for `%s' of type %u\n", "GET", + GNUNET_h2s (&msg->key), ntohl (msg->type)); +#endif + GNUNET_STATISTICS_update (stats, gettext_noop ("# GET requests received"), 1, + GNUNET_NO); + GNUNET_SERVER_client_keep (client); + if ((size == sizeof (struct GetMessage)) && + (GNUNET_YES != GNUNET_CONTAINER_bloomfilter_test (filter, &msg->key))) + { + /* don't bother database... */ +#if DEBUG_DATASTORE + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Empty result set for `%s' request for `%s' (bloomfilter).\n", + "GET", GNUNET_h2s (&msg->key)); +#endif + GNUNET_STATISTICS_update (stats, + gettext_noop + ("# requests filtered by bloomfilter"), 1, + GNUNET_NO); + transmit_item (client, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, + 0); + return; + } + plugin->api->get_key (plugin->api->cls, GNUNET_ntohll (msg->offset), + ((size == + sizeof (struct GetMessage)) ? &msg->key : NULL), NULL, + ntohl (msg->type), &transmit_item, client); +} + + +/** + * Handle UPDATE-message. + * + * @param cls closure + * @param client identification of the client + * @param message the actual message + */ +static void +handle_update (void *cls, struct GNUNET_SERVER_Client *client, + const struct GNUNET_MessageHeader *message) +{ + const struct UpdateMessage *msg; + int ret; + char *emsg; + + GNUNET_STATISTICS_update (stats, gettext_noop ("# UPDATE requests received"), + 1, GNUNET_NO); + msg = (const struct UpdateMessage *) message; + emsg = NULL; +#if DEBUG_DATASTORE + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Processing `%s' request for %llu\n", + "UPDATE", (unsigned long long) GNUNET_ntohll (msg->uid)); +#endif + ret = + plugin->api->update (plugin->api->cls, GNUNET_ntohll (msg->uid), + (int32_t) ntohl (msg->priority), + GNUNET_TIME_absolute_ntoh (msg->expiration), &emsg); + transmit_status (client, ret, emsg); + GNUNET_free_non_null (emsg); +} + + +/** + * Handle GET_REPLICATION-message. + * + * @param cls closure + * @param client identification of the client + * @param message the actual message + */ +static void +handle_get_replication (void *cls, struct GNUNET_SERVER_Client *client, + const struct GNUNET_MessageHeader *message) +{ +#if DEBUG_DATASTORE + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Processing `%s' request\n", + "GET_REPLICATION"); +#endif + GNUNET_STATISTICS_update (stats, + gettext_noop + ("# GET REPLICATION requests received"), 1, + GNUNET_NO); + GNUNET_SERVER_client_keep (client); + plugin->api->get_replication (plugin->api->cls, &transmit_item, client); +} + + +/** + * Handle GET_ZERO_ANONYMITY-message. + * + * @param cls closure + * @param client identification of the client + * @param message the actual message + */ +static void +handle_get_zero_anonymity (void *cls, struct GNUNET_SERVER_Client *client, + const struct GNUNET_MessageHeader *message) +{ + const struct GetZeroAnonymityMessage *msg = + (const struct GetZeroAnonymityMessage *) message; + enum GNUNET_BLOCK_Type type; + + type = (enum GNUNET_BLOCK_Type) ntohl (msg->type); + if (type == GNUNET_BLOCK_TYPE_ANY) + { + GNUNET_break (0); + GNUNET_SERVER_receive_done (client, GNUNET_SYSERR); + return; + } +#if DEBUG_DATASTORE + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Processing `%s' request\n", + "GET_ZERO_ANONYMITY"); +#endif + GNUNET_STATISTICS_update (stats, + gettext_noop + ("# GET ZERO ANONYMITY requests received"), 1, + GNUNET_NO); + GNUNET_SERVER_client_keep (client); + plugin->api->get_zero_anonymity (plugin->api->cls, + GNUNET_ntohll (msg->offset), type, + &transmit_item, client); +} + + +/** + * Callback function that will cause the item that is passed + * in to be deleted (by returning GNUNET_NO). + */ +static int +remove_callback (void *cls, const GNUNET_HashCode * key, uint32_t size, + const void *data, enum GNUNET_BLOCK_Type type, + uint32_t priority, uint32_t anonymity, + struct GNUNET_TIME_Absolute expiration, uint64_t uid) +{ + struct GNUNET_SERVER_Client *client = cls; + + if (key == NULL) + { +#if DEBUG_DATASTORE + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "No further matches for `%s' request.\n", "REMOVE"); +#endif + transmit_status (client, GNUNET_NO, _("Content not found")); + GNUNET_SERVER_client_drop (client); + return GNUNET_OK; /* last item */ + } +#if DEBUG_DATASTORE + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Item %llu matches `%s' request for key `%s' and type %u.\n", + (unsigned long long) uid, "REMOVE", GNUNET_h2s (key), type); +#endif + GNUNET_STATISTICS_update (stats, + gettext_noop ("# bytes removed (explicit request)"), + size, GNUNET_YES); + GNUNET_CONTAINER_bloomfilter_remove (filter, key); + transmit_status (client, GNUNET_OK, NULL); + GNUNET_SERVER_client_drop (client); + return GNUNET_NO; +} + + +/** + * Handle REMOVE-message. + * + * @param cls closure + * @param client identification of the client + * @param message the actual message + */ +static void +handle_remove (void *cls, struct GNUNET_SERVER_Client *client, + const struct GNUNET_MessageHeader *message) +{ + const struct DataMessage *dm = check_data (message); + GNUNET_HashCode vhash; + + if (dm == NULL) + { + GNUNET_break (0); + GNUNET_SERVER_receive_done (client, GNUNET_SYSERR); + return; + } +#if DEBUG_DATASTORE + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Processing `%s' request for `%s' of type %u\n", "REMOVE", + GNUNET_h2s (&dm->key), ntohl (dm->type)); +#endif + GNUNET_STATISTICS_update (stats, gettext_noop ("# REMOVE requests received"), + 1, GNUNET_NO); + GNUNET_SERVER_client_keep (client); + GNUNET_CRYPTO_hash (&dm[1], ntohl (dm->size), &vhash); + plugin->api->get_key (plugin->api->cls, 0, &dm->key, &vhash, + (enum GNUNET_BLOCK_Type) ntohl (dm->type), + &remove_callback, client); +} + + +/** + * Handle DROP-message. + * + * @param cls closure + * @param client identification of the client + * @param message the actual message + */ +static void +handle_drop (void *cls, struct GNUNET_SERVER_Client *client, + const struct GNUNET_MessageHeader *message) +{ +#if DEBUG_DATASTORE + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Processing `%s' request\n", "DROP"); +#endif + do_drop = GNUNET_YES; + GNUNET_SERVER_receive_done (client, GNUNET_OK); +} + + +/** + * Function called by plugins to notify us about a + * change in their disk utilization. + * + * @param cls closure (NULL) + * @param delta change in disk utilization, + * 0 for "reset to empty" + */ +static void +disk_utilization_change_cb (void *cls, int delta) +{ + if ((delta < 0) && (payload < -delta)) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _ + ("Datastore payload inaccurate (%lld < %lld). Trying to fix.\n"), + (long long) payload, (long long) -delta); + payload = plugin->api->estimate_size (plugin->api->cls); + sync_stats (); + return; + } + payload += delta; + lastSync++; + if (lastSync >= MAX_STAT_SYNC_LAG) + sync_stats (); +} + + +/** + * Callback function to process statistic values. + * + * @param cls closure (struct Plugin*) + * @param subsystem name of subsystem that created the statistic + * @param name the name of the datum + * @param value the current value + * @param is_persistent GNUNET_YES if the value is persistent, GNUNET_NO if not + * @return GNUNET_OK to continue, GNUNET_SYSERR to abort iteration + */ +static int +process_stat_in (void *cls, const char *subsystem, const char *name, + uint64_t value, int is_persistent) +{ + GNUNET_assert (stats_worked == GNUNET_NO); + stats_worked = GNUNET_YES; + payload += value; +#if DEBUG_SQLITE + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Notification from statistics about existing payload (%llu), new payload is %llu\n", + abs_value, payload); +#endif + return GNUNET_OK; +} + + +static void +process_stat_done (void *cls, int success) +{ + struct DatastorePlugin *plugin = cls; + + stat_get = NULL; + if (stats_worked == GNUNET_NO) + payload = plugin->api->estimate_size (plugin->api->cls); +} + + +/** + * Load the datastore plugin. + */ +static struct DatastorePlugin * +load_plugin () +{ + struct DatastorePlugin *ret; + char *libname; + + ret = GNUNET_malloc (sizeof (struct DatastorePlugin)); + ret->env.cfg = cfg; + ret->env.duc = &disk_utilization_change_cb; + ret->env.cls = NULL; + GNUNET_log (GNUNET_ERROR_TYPE_INFO, _("Loading `%s' datastore plugin\n"), + plugin_name); + GNUNET_asprintf (&libname, "libgnunet_plugin_datastore_%s", plugin_name); + ret->short_name = GNUNET_strdup (plugin_name); + ret->lib_name = libname; + ret->api = GNUNET_PLUGIN_load (libname, &ret->env); + if (ret->api == NULL) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + _("Failed to load datastore plugin for `%s'\n"), plugin_name); + GNUNET_free (ret->short_name); + GNUNET_free (libname); + GNUNET_free (ret); + return NULL; + } + return ret; +} + + +/** + * Function called when the service shuts + * down. Unloads our datastore plugin. + * + * @param plug plugin to unload + */ +static void +unload_plugin (struct DatastorePlugin *plug) +{ +#if DEBUG_DATASTORE + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Datastore service is unloading plugin...\n"); +#endif + GNUNET_break (NULL == GNUNET_PLUGIN_unload (plug->lib_name, plug->api)); + GNUNET_free (plug->lib_name); + GNUNET_free (plug->short_name); + GNUNET_free (plug); + GNUNET_free (quota_stat_name); + quota_stat_name = NULL; +} + + +/** + * Final task run after shutdown. Unloads plugins and disconnects us from + * statistics. + */ +static void +unload_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + if (lastSync > 0) + sync_stats (); + if (GNUNET_YES == do_drop) + plugin->api->drop (plugin->api->cls); + unload_plugin (plugin); + plugin = NULL; + if (filter != NULL) + { + GNUNET_CONTAINER_bloomfilter_free (filter); + filter = NULL; + } + if (stat_get != NULL) + { + GNUNET_STATISTICS_get_cancel (stat_get); + stat_get = NULL; + } + if (stats != NULL) + { + GNUNET_STATISTICS_destroy (stats, GNUNET_YES); + stats = NULL; + } + GNUNET_free_non_null (plugin_name); + plugin_name = NULL; +} + + +/** + * Last task run during shutdown. Disconnects us from + * the transport and core. + */ +static void +cleaning_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct TransmitCallbackContext *tcc; + + cleaning_done = GNUNET_YES; + while (NULL != (tcc = tcc_head)) + { + GNUNET_CONTAINER_DLL_remove (tcc_head, tcc_tail, tcc); + if (tcc->th != NULL) + { + GNUNET_CONNECTION_notify_transmit_ready_cancel (tcc->th); + GNUNET_SERVER_client_drop (tcc->client); + } + GNUNET_free (tcc->msg); + GNUNET_free (tcc); + } + if (expired_kill_task != GNUNET_SCHEDULER_NO_TASK) + { + GNUNET_SCHEDULER_cancel (expired_kill_task); + expired_kill_task = GNUNET_SCHEDULER_NO_TASK; + } + GNUNET_SCHEDULER_add_continuation (&unload_task, NULL, + GNUNET_SCHEDULER_REASON_PREREQ_DONE); +} + + +/** + * Function that removes all active reservations made + * by the given client and releases the space for other + * requests. + * + * @param cls closure + * @param client identification of the client + */ +static void +cleanup_reservations (void *cls, struct GNUNET_SERVER_Client *client) +{ + struct ReservationList *pos; + struct ReservationList *prev; + struct ReservationList *next; + + if (client == NULL) + return; + prev = NULL; + pos = reservations; + while (NULL != pos) + { + next = pos->next; + if (pos->client == client) + { + if (prev == NULL) + reservations = next; + else + prev->next = next; + reserved -= pos->amount + pos->entries * GNUNET_DATASTORE_ENTRY_OVERHEAD; + GNUNET_free (pos); + } + else + { + prev = pos; + } + pos = next; + } + GNUNET_STATISTICS_set (stats, gettext_noop ("# reserved"), reserved, + GNUNET_NO); +} + + +/** + * Adds a given key to the bloomfilter 'count' times. + * + * @param cls the bloomfilter + * @param key key to add + * @param count number of times to add key + */ +static void +add_key_to_bloomfilter (void *cls, + const GNUNET_HashCode *key, + unsigned int count) +{ + struct GNUNET_CONTAINER_BloomFilter *bf = cls; + while (0 < count--) + GNUNET_CONTAINER_bloomfilter_add (bf, key); +} + + +/** + * Process datastore requests. + * + * @param cls closure + * @param server the initialized server + * @param c configuration to use + */ +static void +run (void *cls, struct GNUNET_SERVER_Handle *server, + const struct GNUNET_CONFIGURATION_Handle *c) +{ + static const struct GNUNET_SERVER_MessageHandler handlers[] = { + {&handle_reserve, NULL, GNUNET_MESSAGE_TYPE_DATASTORE_RESERVE, + sizeof (struct ReserveMessage)}, + {&handle_release_reserve, NULL, + GNUNET_MESSAGE_TYPE_DATASTORE_RELEASE_RESERVE, + sizeof (struct ReleaseReserveMessage)}, + {&handle_put, NULL, GNUNET_MESSAGE_TYPE_DATASTORE_PUT, 0}, + {&handle_update, NULL, GNUNET_MESSAGE_TYPE_DATASTORE_UPDATE, + sizeof (struct UpdateMessage)}, + {&handle_get, NULL, GNUNET_MESSAGE_TYPE_DATASTORE_GET, 0}, + {&handle_get_replication, NULL, + GNUNET_MESSAGE_TYPE_DATASTORE_GET_REPLICATION, + sizeof (struct GNUNET_MessageHeader)}, + {&handle_get_zero_anonymity, NULL, + GNUNET_MESSAGE_TYPE_DATASTORE_GET_ZERO_ANONYMITY, + sizeof (struct GetZeroAnonymityMessage)}, + {&handle_remove, NULL, GNUNET_MESSAGE_TYPE_DATASTORE_REMOVE, 0}, + {&handle_drop, NULL, GNUNET_MESSAGE_TYPE_DATASTORE_DROP, + sizeof (struct GNUNET_MessageHeader)}, + {NULL, NULL, 0, 0} + }; + char *fn; + char *pfn; + unsigned int bf_size; + int refresh_bf; + + cfg = c; + if (GNUNET_OK != + GNUNET_CONFIGURATION_get_value_string (cfg, "DATASTORE", "DATABASE", + &plugin_name)) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + _("No `%s' specified for `%s' in configuration!\n"), "DATABASE", + "DATASTORE"); + return; + } + GNUNET_asprintf ("a_stat_name, + _("# bytes used in file-sharing datastore `%s'"), + plugin_name); + if (GNUNET_OK != + GNUNET_CONFIGURATION_get_value_size (cfg, "DATASTORE", "QUOTA", "a)) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + _("No `%s' specified for `%s' in configuration!\n"), "QUOTA", + "DATASTORE"); + return; + } + stats = GNUNET_STATISTICS_create ("datastore", cfg); + GNUNET_STATISTICS_set (stats, gettext_noop ("# quota"), quota, GNUNET_NO); + cache_size = quota / 8; /* Or should we make this an option? */ + GNUNET_STATISTICS_set (stats, gettext_noop ("# cache size"), cache_size, + GNUNET_NO); + if (quota / (32 * 1024LL) > (1 << 31)) + bf_size = (1 << 31); /* absolute limit: ~2 GB, beyond that BF just won't help anyway */ + else + bf_size = quota / (32 * 1024LL); /* 8 bit per entry, 1 bit per 32 kb in DB */ + fn = NULL; + if ((GNUNET_OK != + GNUNET_CONFIGURATION_get_value_filename (cfg, "DATASTORE", "BLOOMFILTER", + &fn)) || + (GNUNET_OK != GNUNET_DISK_directory_create_for_file (fn))) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _("Could not use specified filename `%s' for bloomfilter.\n"), + fn != NULL ? fn : ""); + GNUNET_free_non_null (fn); + fn = NULL; + } + if (fn != NULL) + { + GNUNET_asprintf (&pfn, "%s.%s", fn, plugin_name); + if (GNUNET_YES == GNUNET_DISK_file_test (pfn)) + { + filter = GNUNET_CONTAINER_bloomfilter_load (pfn, bf_size, 5); /* approx. 3% false positives at max use */ + if (NULL == filter) + { + /* file exists but not valid, remove and try again, but refresh */ + if (0 != UNLINK (pfn)) + { + /* failed to remove, run without file */ + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + _("Failed to remove bogus bloomfilter file `%s'\n"), + pfn); + GNUNET_free (pfn); + pfn = NULL; + filter = GNUNET_CONTAINER_bloomfilter_load (NULL, bf_size, 5); /* approx. 3% false positives at max use */ + refresh_bf = GNUNET_YES; + } + else + { + /* try again after remove */ + filter = GNUNET_CONTAINER_bloomfilter_load (pfn, bf_size, 5); /* approx. 3% false positives at max use */ + refresh_bf = GNUNET_YES; + if (NULL == filter) + { + /* failed yet again, give up on using file */ + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + _("Failed to remove bogus bloomfilter file `%s'\n"), + pfn); + GNUNET_free (pfn); + pfn = NULL; + filter = GNUNET_CONTAINER_bloomfilter_load (NULL, bf_size, 5); /* approx. 3% false positives at max use */ + } + } + } + else + { + /* normal case: have an existing valid bf file, no need to refresh */ + refresh_bf = GNUNET_NO; + } + } + else + { + filter = GNUNET_CONTAINER_bloomfilter_load (pfn, bf_size, 5); /* approx. 3% false positives at max use */ + refresh_bf = GNUNET_YES; + } + GNUNET_free (pfn); + } + else + { + filter = GNUNET_CONTAINER_bloomfilter_init (NULL, bf_size, 5); /* approx. 3% false positives at max use */ + refresh_bf = GNUNET_YES; + } + GNUNET_free_non_null (fn); + if (filter == NULL) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + _("Failed to initialize bloomfilter.\n")); + if (stats != NULL) + { + GNUNET_STATISTICS_destroy (stats, GNUNET_YES); + stats = NULL; + } + return; + } + plugin = load_plugin (); + if (NULL == plugin) + { + GNUNET_CONTAINER_bloomfilter_free (filter); + filter = NULL; + if (stats != NULL) + { + GNUNET_STATISTICS_destroy (stats, GNUNET_YES); + stats = NULL; + } + return; + } + stat_get = + GNUNET_STATISTICS_get (stats, "datastore", quota_stat_name, + GNUNET_TIME_UNIT_SECONDS, &process_stat_done, + &process_stat_in, plugin); + GNUNET_SERVER_disconnect_notify (server, &cleanup_reservations, NULL); + GNUNET_SERVER_add_handlers (server, handlers); + if (GNUNET_YES == refresh_bf) + { + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + _("Rebuilding bloomfilter. Please be patient.\n")); + if (NULL != plugin->api->get_keys) + plugin->api->get_keys (plugin->api->cls, &add_key_to_bloomfilter, filter); + else + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + _("Plugin does not support get_keys function. Please fix!\n")); + + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + _("Bloomfilter construction complete.\n")); + } + expired_kill_task = + GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_IDLE, + &delete_expired, NULL); + GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_FOREVER_REL, &cleaning_task, + NULL); +} + + +/** + * The main function for the datastore service. + * + * @param argc number of arguments from the command line + * @param argv command line arguments + * @return 0 ok, 1 on error + */ +int +main (int argc, char *const *argv) +{ + int ret; + + ret = + (GNUNET_OK == + GNUNET_SERVICE_run (argc, argv, "datastore", GNUNET_SERVICE_OPTION_NONE, + &run, NULL)) ? 0 : 1; + return ret; +} + + +/* end of gnunet-service-datastore.c */ diff --git a/src/datastore/perf_datastore_api.c b/src/datastore/perf_datastore_api.c new file mode 100644 index 0000000..aae152d --- /dev/null +++ b/src/datastore/perf_datastore_api.c @@ -0,0 +1,405 @@ +/* + This file is part of GNUnet. + (C) 2004, 2005, 2006, 2007, 2009, 2011 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ +/* + * @file datastore/perf_datastore_api.c + * @brief performance measurement for the datastore implementation + * @author Christian Grothoff + * + * This testcase inserts a bunch of (variable size) data and then + * deletes data until the (reported) database size drops below a given + * threshold. This is iterated 10 times, with the actual size of the + * content stored and the number of operations performed being printed + * for each iteration. The code also prints a "I" for every 40 blocks + * inserted and a "D" for every 40 blocks deleted. The deletion + * strategy uses the "random" iterator. Priorities and expiration + * dates are set using a pseudo-random value within a realistic range. + */ + +#include "platform.h" +#include "gnunet_util_lib.h" +#include "gnunet_protocols.h" +#include "gnunet_datastore_service.h" +#include <gauger.h> + +#define VERBOSE GNUNET_NO + +/** + * How long until we give up on transmitting the message? + */ +#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 15) + +static const char *plugin_name; + +static struct GNUNET_DATASTORE_Handle *datastore; + +/** + * Target datastore size (in bytes). + */ +#define MAX_SIZE 1024LL * 1024 * 4 + +/** + * Report progress outside of major reports? Should probably be GNUNET_YES if + * size is > 16 MB. + */ +#define REPORT_ID GNUNET_YES + +/** + * Number of put operations equivalent to 1/3rd of MAX_SIZE + */ +#define PUT_10 MAX_SIZE / 32 / 1024 / 3 + +/** + * Total number of iterations (each iteration doing + * PUT_10 put operations); we report full status every + * 10 iterations. Abort with CTRL-C. + */ +#define ITERATIONS 8 + + +static unsigned long long stored_bytes; + +static unsigned long long stored_entries; + +static unsigned long long stored_ops; + +static struct GNUNET_TIME_Absolute start_time; + +static int ok; + +enum RunPhase +{ + RP_DONE = 0, + RP_PUT, + RP_CUT, + RP_REPORT, + RP_ERROR +}; + + +struct CpsRunContext +{ + const struct GNUNET_CONFIGURATION_Handle *cfg; + enum RunPhase phase; + int j; + unsigned long long size; + int i; +}; + + + +static void +run_continuation (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc); + + + + +static void +check_success (void *cls, int success, struct GNUNET_TIME_Absolute min_expiration, const char *msg) +{ + struct CpsRunContext *crc = cls; + + if (GNUNET_OK != success) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Check success failed: `%s'\n", msg); + crc->phase = RP_ERROR; + GNUNET_SCHEDULER_add_now (&run_continuation, crc); + return; + } +#if REPORT_ID + FPRINTF (stderr, "%s", "I"); +#endif + stored_bytes += crc->size; + stored_ops++; + stored_entries++; + crc->j++; + if (crc->j >= PUT_10) + { + crc->j = 0; + crc->i++; + if (crc->i == ITERATIONS) + crc->phase = RP_DONE; + else + crc->phase = RP_CUT; + } + GNUNET_SCHEDULER_add_continuation (&run_continuation, crc, + GNUNET_SCHEDULER_REASON_PREREQ_DONE); +} + + +/** + * Continuation called to notify client about result of the + * operation. + * + * @param cls closure + * @param success GNUNET_SYSERR on failure + * @param min_expiration minimum expiration time required for content to be stored + * by the datacache at this time, zero for unknown + * @param msg NULL on success, otherwise an error message + */ +static void +remove_next (void *cls, int success, struct GNUNET_TIME_Absolute min_expiration, const char *msg) +{ + struct CpsRunContext *crc = cls; + + if (GNUNET_OK != success) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "remove_next failed: `%s'\n", msg); + crc->phase = RP_ERROR; + GNUNET_SCHEDULER_add_now (&run_continuation, crc); + return; + } +#if REPORT_ID + FPRINTF (stderr, "%s", "D"); +#endif + GNUNET_assert (GNUNET_OK == success); + GNUNET_SCHEDULER_add_now (&run_continuation, crc); +} + + +static void +delete_value (void *cls, const GNUNET_HashCode * key, size_t size, + const void *data, enum GNUNET_BLOCK_Type type, uint32_t priority, + uint32_t anonymity, struct GNUNET_TIME_Absolute expiration, + uint64_t uid) +{ + struct CpsRunContext *crc = cls; + + GNUNET_assert (NULL != key); + stored_ops++; + stored_bytes -= size; + stored_entries--; + stored_ops++; + if (stored_bytes < MAX_SIZE) + crc->phase = RP_PUT; + GNUNET_assert (NULL != + GNUNET_DATASTORE_remove (datastore, key, size, data, 1, 1, + TIMEOUT, &remove_next, crc)); +} + + +static void +run_continuation (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct CpsRunContext *crc = cls; + size_t size; + static GNUNET_HashCode key; + static char data[65536]; + int i; + int k; + char gstr[128]; + + ok = (int) crc->phase; + switch (crc->phase) + { + case RP_PUT: + memset (&key, 256 - crc->i, sizeof (GNUNET_HashCode)); + i = crc->j; + k = crc->i; + /* most content is 32k */ + size = 32 * 1024; + if (GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 16) == 0) /* but some of it is less! */ + size = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 32 * 1024); + crc->size = size = size - (size & 7); /* always multiple of 8 */ + GNUNET_CRYPTO_hash (&key, sizeof (GNUNET_HashCode), &key); + memset (data, i, size); + if (i > 255) + memset (data, i - 255, size / 2); + data[0] = k; + GNUNET_assert (NULL != + GNUNET_DATASTORE_put (datastore, 0, &key, size, data, i + 1, + GNUNET_CRYPTO_random_u32 + (GNUNET_CRYPTO_QUALITY_WEAK, 100), i, + 0, + GNUNET_TIME_relative_to_absolute + (GNUNET_TIME_relative_multiply + (GNUNET_TIME_UNIT_SECONDS, + GNUNET_CRYPTO_random_u32 + (GNUNET_CRYPTO_QUALITY_WEAK, 1000))), + 1, 1, TIMEOUT, &check_success, crc)); + break; + case RP_CUT: + /* trim down below MAX_SIZE again */ + GNUNET_assert (NULL != + GNUNET_DATASTORE_get_for_replication (datastore, 1, 1, + TIMEOUT, &delete_value, + crc)); + break; + case RP_REPORT: + printf ( +#if REPORT_ID + "\n" +#endif + "Stored %llu kB / %lluk ops / %llu ops/s\n", stored_bytes / 1024, /* used size in k */ + stored_ops / 1024, /* total operations (in k) */ + 1000 * stored_ops / (1 + + GNUNET_TIME_absolute_get_duration + (start_time).rel_value)); + crc->phase = RP_PUT; + crc->j = 0; + GNUNET_SCHEDULER_add_continuation (&run_continuation, crc, + GNUNET_SCHEDULER_REASON_PREREQ_DONE); + break; + case RP_DONE: + GNUNET_snprintf (gstr, sizeof (gstr), "DATASTORE-%s", plugin_name); + if ((crc->i == ITERATIONS) && (stored_ops > 0)) + GAUGER (gstr, "PUT operation duration", + GNUNET_TIME_absolute_get_duration (start_time).rel_value / + stored_ops, "ms/operation"); + GNUNET_DATASTORE_disconnect (datastore, GNUNET_YES); + GNUNET_free (crc); + ok = 0; + break; + case RP_ERROR: + GNUNET_DATASTORE_disconnect (datastore, GNUNET_YES); + GNUNET_free (crc); + ok = 1; + break; + default: + GNUNET_assert (0); + } +} + + +static void +run_tests (void *cls, int success, struct GNUNET_TIME_Absolute min_expiration, const char *msg) +{ + struct CpsRunContext *crc = cls; + + if (success != GNUNET_YES) + { + FPRINTF (stderr, + "Test 'put' operation failed with error `%s' database likely not setup, skipping test.", + msg); + GNUNET_free (crc); + return; + } + GNUNET_SCHEDULER_add_continuation (&run_continuation, crc, + GNUNET_SCHEDULER_REASON_PREREQ_DONE); +} + + +static void +run (void *cls, char *const *args, const char *cfgfile, + const struct GNUNET_CONFIGURATION_Handle *cfg) +{ + struct CpsRunContext *crc; + static GNUNET_HashCode zkey; + + datastore = GNUNET_DATASTORE_connect (cfg); + start_time = GNUNET_TIME_absolute_get (); + crc = GNUNET_malloc (sizeof (struct CpsRunContext)); + crc->cfg = cfg; + crc->phase = RP_PUT; + if (NULL == + GNUNET_DATASTORE_put (datastore, 0, &zkey, 4, "TEST", + GNUNET_BLOCK_TYPE_TEST, 0, 0, 0, + GNUNET_TIME_relative_to_absolute + (GNUNET_TIME_UNIT_SECONDS), 0, 1, + GNUNET_TIME_UNIT_MINUTES, &run_tests, crc)) + { + FPRINTF (stderr, "%s", "Test 'put' operation failed.\n"); + ok = 1; + GNUNET_free (crc); + } +} + + +static int +check () +{ + struct GNUNET_OS_Process *proc; + char cfg_name[128]; + + char *const argv[] = { + "perf-datastore-api", + "-c", + cfg_name, +#if VERBOSE + "-L", "DEBUG", +#endif + NULL + }; + struct GNUNET_GETOPT_CommandLineOption options[] = { + GNUNET_GETOPT_OPTION_END + }; + + GNUNET_snprintf (cfg_name, sizeof (cfg_name), + "test_datastore_api_data_%s.conf", plugin_name); + proc = + GNUNET_OS_start_process (GNUNET_YES, NULL, NULL, "gnunet-service-arm", + "gnunet-service-arm", +#if VERBOSE + "-L", "DEBUG", +#endif + "-c", cfg_name, NULL); + GNUNET_assert (NULL != proc); + GNUNET_PROGRAM_run ((sizeof (argv) / sizeof (char *)) - 1, argv, + "perf-datastore-api", "nohelp", options, &run, NULL); + sleep (1); /* give datastore chance to process 'DROP' */ + if (0 != GNUNET_OS_process_kill (proc, SIGTERM)) + { + GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING, "kill"); + ok = 1; + } + GNUNET_OS_process_wait (proc); + GNUNET_OS_process_close (proc); + proc = NULL; + return ok; +} + + +int +main (int argc, char *argv[]) +{ + int ret; + char *pos; + char dir_name[128]; + + sleep (1); + /* determine name of plugin to use */ + plugin_name = argv[0]; + while (NULL != (pos = strstr (plugin_name, "_"))) + plugin_name = pos + 1; + if (NULL != (pos = strstr (plugin_name, "."))) + pos[0] = 0; + else + pos = (char *) plugin_name; + + GNUNET_snprintf (dir_name, sizeof (dir_name), "/tmp/test-gnunet-datastore-%s", + plugin_name); + GNUNET_DISK_directory_remove (dir_name); + GNUNET_log_setup ("perf-datastore-api", +#if VERBOSE + "DEBUG", +#else + "WARNING", +#endif + NULL); + ret = check (); + if (pos != plugin_name) + pos[0] = '.'; +#if REPORT_ID + FPRINTF (stderr, "%s", "\n"); +#endif + GNUNET_DISK_directory_remove (dir_name); + return ret; +} + +/* end of perf_datastore_api.c */ diff --git a/src/datastore/perf_plugin_datastore.c b/src/datastore/perf_plugin_datastore.c new file mode 100644 index 0000000..1860374 --- /dev/null +++ b/src/datastore/perf_plugin_datastore.c @@ -0,0 +1,521 @@ +/* + This file is part of GNUnet. + (C) 2004, 2005, 2006, 2007, 2009, 2011 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ +/* + * @file perf_plugin_datastore.c + * @brief Profile database plugin directly, focusing on iterators. + * @author Christian Grothoff + */ + +#include "platform.h" +#include "gnunet_util_lib.h" +#include "gnunet_protocols.h" +#include "gnunet_datastore_plugin.h" +#include <gauger.h> + +#define VERBOSE GNUNET_NO + +/** + * Target datastore size (in bytes). Realistic sizes are + * more like 16 GB (not the default of 16 MB); however, + * those take too long to run them in the usual "make check" + * sequence. Hence the value used for shipping is tiny. + */ +#define MAX_SIZE 1024LL * 1024 * 16 * 1 + +#define ITERATIONS 2 + +/** + * Number of put operations equivalent to 1/10th of MAX_SIZE + */ +#define PUT_10 (MAX_SIZE / 32 / 1024 / ITERATIONS) + +static char category[256]; + +static unsigned int hits[PUT_10 / 8 + 1]; + +static unsigned long long stored_bytes; + +static unsigned long long stored_entries; + +static unsigned long long stored_ops; + +static const char *plugin_name; + +static int ok; + +enum RunPhase +{ + RP_ERROR = 0, + RP_PUT, + RP_REP_GET, + RP_ZA_GET, + RP_EXP_GET, + RP_DONE +}; + + +struct CpsRunContext +{ + unsigned int i; + struct GNUNET_TIME_Absolute start; + struct GNUNET_TIME_Absolute end; + const struct GNUNET_CONFIGURATION_Handle *cfg; + struct GNUNET_DATASTORE_PluginFunctions *api; + enum RunPhase phase; + unsigned int cnt; + unsigned int iter; + uint64_t offset; +}; + + +/** + * Function called by plugins to notify us about a + * change in their disk utilization. + * + * @param cls closure (NULL) + * @param delta change in disk utilization, + * 0 for "reset to empty" + */ +static void +disk_utilization_change_cb (void *cls, int delta) +{ +} + + +static void +putValue (struct GNUNET_DATASTORE_PluginFunctions *api, int i, int k) +{ + char value[65536]; + size_t size; + static GNUNET_HashCode key; + static int ic; + char *msg; + unsigned int prio; + + /* most content is 32k */ + size = 32 * 1024; + if (GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 16) == 0) /* but some of it is less! */ + size = 8 + GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 32 * 1024); + size = size - (size & 7); /* always multiple of 8 */ + + /* generate random key */ + key.bits[0] = (unsigned int) GNUNET_TIME_absolute_get ().abs_value; + GNUNET_CRYPTO_hash (&key, sizeof (GNUNET_HashCode), &key); + memset (value, i, size); + if (i > 255) + memset (value, i - 255, size / 2); + value[0] = k; + memcpy (&value[4], &i, sizeof (i)); + msg = NULL; + prio = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 100); + if (GNUNET_OK != api->put (api->cls, &key, size, value, 1 + i % 4 /* type */ , + prio, i % 4 /* anonymity */ , + 0 /* replication */ , + GNUNET_TIME_relative_to_absolute + (GNUNET_TIME_relative_multiply + (GNUNET_TIME_UNIT_MILLISECONDS, + 60 * 60 * 60 * 1000 + + GNUNET_CRYPTO_random_u32 + (GNUNET_CRYPTO_QUALITY_WEAK, 1000))), &msg)) + { + FPRINTF (stderr, "ERROR: `%s'\n", msg); + GNUNET_free_non_null (msg); + return; + } + ic++; + stored_bytes += size; + stored_ops++; + stored_entries++; +} + +static void +test (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc); + + +static int +iterate_zeros (void *cls, const GNUNET_HashCode * key, uint32_t size, + const void *data, enum GNUNET_BLOCK_Type type, uint32_t priority, + uint32_t anonymity, struct GNUNET_TIME_Absolute expiration, + uint64_t uid) +{ + struct CpsRunContext *crc = cls; + int i; + const char *cdata = data; + + GNUNET_assert (key != NULL); + GNUNET_assert (size >= 8); + memcpy (&i, &cdata[4], sizeof (i)); + hits[i / 8] |= (1 << (i % 8)); + +#if VERBOSE + FPRINTF (stderr, "Found result type=%u, priority=%u, size=%u, expire=%llu\n", + type, priority, size, (unsigned long long) expiration.abs_value); +#endif + crc->cnt++; + if (crc->cnt == PUT_10 / 4 - 1) + { + unsigned int bc; + + bc = 0; + for (i = 0; i < PUT_10; i++) + if (0 != (hits[i / 8] & (1 << (i % 8)))) + bc++; + + crc->end = GNUNET_TIME_absolute_get (); + printf ("%s took %llu ms yielding %u/%u items\n", + "Select random zero-anonymity item", + (unsigned long long) (crc->end.abs_value - crc->start.abs_value), + bc, crc->cnt); + if (crc->cnt > 0) + GAUGER (category, "Select random zero-anonymity item", + (crc->end.abs_value - crc->start.abs_value) / crc->cnt, + "ms/item"); + memset (hits, 0, sizeof (hits)); + crc->phase++; + crc->cnt = 0; + crc->start = GNUNET_TIME_absolute_get (); + } + GNUNET_SCHEDULER_add_now (&test, crc); + return GNUNET_OK; +} + + +static int +expiration_get (void *cls, const GNUNET_HashCode * key, uint32_t size, + const void *data, enum GNUNET_BLOCK_Type type, + uint32_t priority, uint32_t anonymity, + struct GNUNET_TIME_Absolute expiration, uint64_t uid) +{ + struct CpsRunContext *crc = cls; + int i; + const char *cdata = data; + + GNUNET_assert (size >= 8); + memcpy (&i, &cdata[4], sizeof (i)); + hits[i / 8] |= (1 << (i % 8)); + crc->cnt++; + if (PUT_10 <= crc->cnt) + { + unsigned int bc; + + bc = 0; + for (i = 0; i < PUT_10; i++) + if (0 != (hits[i / 8] & (1 << (i % 8)))) + bc++; + + crc->end = GNUNET_TIME_absolute_get (); + printf ("%s took %llu ms yielding %u/%u items\n", + "Selecting and deleting by expiration", + (unsigned long long) (crc->end.abs_value - crc->start.abs_value), + bc, (unsigned int) PUT_10); + if (crc->cnt > 0) + GAUGER (category, "Selecting and deleting by expiration", + (crc->end.abs_value - crc->start.abs_value) / crc->cnt, + "ms/item"); + memset (hits, 0, sizeof (hits)); + if (++crc->iter == ITERATIONS) + crc->phase++; + else + crc->phase = RP_PUT; + crc->cnt = 0; + crc->start = GNUNET_TIME_absolute_get (); + } + GNUNET_SCHEDULER_add_now (&test, crc); + return GNUNET_NO; +} + + +static int +replication_get (void *cls, const GNUNET_HashCode * key, uint32_t size, + const void *data, enum GNUNET_BLOCK_Type type, + uint32_t priority, uint32_t anonymity, + struct GNUNET_TIME_Absolute expiration, uint64_t uid) +{ + struct CpsRunContext *crc = cls; + int i; + const char *cdata = data; + + GNUNET_assert (NULL != key); + GNUNET_assert (size >= 8); + memcpy (&i, &cdata[4], sizeof (i)); + hits[i / 8] |= (1 << (i % 8)); + crc->cnt++; + if (PUT_10 <= crc->cnt) + { + unsigned int bc; + + bc = 0; + for (i = 0; i < PUT_10; i++) + if (0 != (hits[i / 8] & (1 << (i % 8)))) + bc++; + + crc->end = GNUNET_TIME_absolute_get (); + printf ("%s took %llu ms yielding %u/%u items\n", + "Selecting random item for replication", + (unsigned long long) (crc->end.abs_value - crc->start.abs_value), + bc, (unsigned int) PUT_10); + if (crc->cnt > 0) + GAUGER (category, "Selecting random item for replication", + (crc->end.abs_value - crc->start.abs_value) / crc->cnt, + "ms/item"); + memset (hits, 0, sizeof (hits)); + crc->phase++; + crc->offset = 0; + crc->cnt = 0; + crc->start = GNUNET_TIME_absolute_get (); + } + + GNUNET_SCHEDULER_add_now (&test, crc); + return GNUNET_OK; +} + + +/** + * Function called when the service shuts + * down. Unloads our datastore plugin. + * + * @param api api to unload + * @param cfg configuration to use + */ +static void +unload_plugin (struct GNUNET_DATASTORE_PluginFunctions *api, + const struct GNUNET_CONFIGURATION_Handle *cfg) +{ + char *name; + char *libname; + + if (GNUNET_OK != + GNUNET_CONFIGURATION_get_value_string (cfg, "DATASTORE", "DATABASE", + &name)) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + _("No `%s' specified for `%s' in configuration!\n"), "DATABASE", + "DATASTORE"); + return; + } + GNUNET_asprintf (&libname, "libgnunet_plugin_datastore_%s", name); + GNUNET_break (NULL == GNUNET_PLUGIN_unload (libname, api)); + GNUNET_free (libname); + GNUNET_free (name); +} + + + +/** + * Last task run during shutdown. Disconnects us from + * the transport and core. + */ +static void +cleaning_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct CpsRunContext *crc = cls; + + unload_plugin (crc->api, crc->cfg); + GNUNET_free (crc); +} + + +static void +test (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct CpsRunContext *crc = cls; + int j; + + if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN)) + { + GNUNET_break (0); + crc->phase = RP_ERROR; + } +#if VERBOSE + FPRINTF (stderr, "In phase %d, iteration %u\n", crc->phase, crc->cnt); +#endif + switch (crc->phase) + { + case RP_ERROR: + GNUNET_break (0); + crc->api->drop (crc->api->cls); + ok = 1; + GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_IDLE, + &cleaning_task, crc); + break; + case RP_PUT: + crc->start = GNUNET_TIME_absolute_get (); + for (j = 0; j < PUT_10; j++) + putValue (crc->api, j, crc->i); + crc->end = GNUNET_TIME_absolute_get (); + { + printf ("%s took %llu ms for %llu items\n", "Storing an item", + (unsigned long long) (crc->end.abs_value - crc->start.abs_value), + PUT_10); + if (PUT_10 > 0) + GAUGER (category, "Storing an item", + (crc->end.abs_value - crc->start.abs_value) / PUT_10, + "ms/item"); + } + crc->i++; + crc->start = GNUNET_TIME_absolute_get (); + crc->phase++; + GNUNET_SCHEDULER_add_now (&test, crc); + break; + case RP_REP_GET: + crc->api->get_replication (crc->api->cls, &replication_get, crc); + break; + case RP_ZA_GET: + crc->api->get_zero_anonymity (crc->api->cls, crc->offset++, 1, + &iterate_zeros, crc); + break; + case RP_EXP_GET: + crc->api->get_expiration (crc->api->cls, &expiration_get, crc); + break; + case RP_DONE: + crc->api->drop (crc->api->cls); + ok = 0; + GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_IDLE, + &cleaning_task, crc); + break; + } +} + + +/** + * Load the datastore plugin. + */ +static struct GNUNET_DATASTORE_PluginFunctions * +load_plugin (const struct GNUNET_CONFIGURATION_Handle *cfg) +{ + static struct GNUNET_DATASTORE_PluginEnvironment env; + struct GNUNET_DATASTORE_PluginFunctions *ret; + char *name; + char *libname; + + if (GNUNET_OK != + GNUNET_CONFIGURATION_get_value_string (cfg, "DATASTORE", "DATABASE", + &name)) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + _("No `%s' specified for `%s' in configuration!\n"), "DATABASE", + "DATASTORE"); + return NULL; + } + env.cfg = cfg; + env.duc = &disk_utilization_change_cb; + env.cls = NULL; + GNUNET_log (GNUNET_ERROR_TYPE_INFO, _("Loading `%s' datastore plugin\n"), + name); + GNUNET_asprintf (&libname, "libgnunet_plugin_datastore_%s", name); + if (NULL == (ret = GNUNET_PLUGIN_load (libname, &env))) + { + FPRINTF (stderr, "Failed to load plugin `%s'!\n", name); + return NULL; + } + GNUNET_free (libname); + GNUNET_free (name); + return ret; +} + + +static void +run (void *cls, char *const *args, const char *cfgfile, + const struct GNUNET_CONFIGURATION_Handle *c) +{ + struct GNUNET_DATASTORE_PluginFunctions *api; + struct CpsRunContext *crc; + + api = load_plugin (c); + if (api == NULL) + { + FPRINTF (stderr, + "%s", "Could not initialize plugin, assuming database not configured. Test not run!\n"); + return; + } + crc = GNUNET_malloc (sizeof (struct CpsRunContext)); + crc->api = api; + crc->cfg = c; + crc->phase = RP_PUT; + ok = 2; + GNUNET_SCHEDULER_add_now (&test, crc); +} + + +static int +check () +{ + char cfg_name[128]; + + char *const argv[] = { + "perf-plugin-datastore", + "-c", + cfg_name, +#if VERBOSE + "-L", "DEBUG", +#endif + NULL + }; + struct GNUNET_GETOPT_CommandLineOption options[] = { + GNUNET_GETOPT_OPTION_END + }; + + GNUNET_snprintf (category, sizeof (category), "DATASTORE-%s", plugin_name); + GNUNET_snprintf (cfg_name, sizeof (cfg_name), + "perf_plugin_datastore_data_%s.conf", plugin_name); + GNUNET_PROGRAM_run ((sizeof (argv) / sizeof (char *)) - 1, argv, + "perf-plugin-datastore", "nohelp", options, &run, NULL); + if (ok != 0) + FPRINTF (stderr, "Missed some testcases: %u\n", ok); + return ok; +} + + +int +main (int argc, char *argv[]) +{ + int ret; + char *pos; + char dir_name[128]; + + sleep (1); + /* determine name of plugin to use */ + plugin_name = argv[0]; + while (NULL != (pos = strstr (plugin_name, "_"))) + plugin_name = pos + 1; + if (NULL != (pos = strstr (plugin_name, "."))) + pos[0] = 0; + else + pos = (char *) plugin_name; + + GNUNET_snprintf (dir_name, sizeof (dir_name), "/tmp/perf-gnunet-datastore-%s", + plugin_name); + GNUNET_DISK_directory_remove (dir_name); + GNUNET_log_setup ("perf-plugin-datastore", +#if VERBOSE + "DEBUG", +#else + "WARNING", +#endif + NULL); + ret = check (); + if (pos != plugin_name) + pos[0] = '.'; + GNUNET_DISK_directory_remove (dir_name); + + return ret; +} + +/* end of perf_plugin_datastore.c */ diff --git a/src/datastore/perf_plugin_datastore_data_mysql.conf b/src/datastore/perf_plugin_datastore_data_mysql.conf new file mode 100644 index 0000000..dd26512 --- /dev/null +++ b/src/datastore/perf_plugin_datastore_data_mysql.conf @@ -0,0 +1,11 @@ +@INLINE@ test_defaults.conf +[PATHS] +SERVICEHOME = /tmp/perf-gnunet-datastore-mysql/ +DEFAULTCONFIG = perf_plugin_datastore_data_mysql.conf + +[datastore] +DATABASE = mysql + +[datastore-mysql] +DATABASE = gnunetcheck + diff --git a/src/datastore/perf_plugin_datastore_data_postgres.conf b/src/datastore/perf_plugin_datastore_data_postgres.conf new file mode 100644 index 0000000..53ce6cf --- /dev/null +++ b/src/datastore/perf_plugin_datastore_data_postgres.conf @@ -0,0 +1,11 @@ +@INLINE@ test_defaults.conf +[PATHS] +SERVICEHOME = /tmp/perf-gnunet-datastore-postgres/ +DEFAULTCONFIG = perf_plugin_datastore_data_postgres.conf + +[datastore] +DATABASE = postgres + +[datastore-postgres] +CONFIG = dbname=gnunetcheck + diff --git a/src/datastore/perf_plugin_datastore_data_sqlite.conf b/src/datastore/perf_plugin_datastore_data_sqlite.conf new file mode 100644 index 0000000..241d85e --- /dev/null +++ b/src/datastore/perf_plugin_datastore_data_sqlite.conf @@ -0,0 +1,5 @@ +@INLINE@ test_defaults.conf +[PATHS] +SERVICEHOME = /tmp/perf-gnunet-datastore-sqlite/ +DEFAULTCONFIG = perf_plugin_datastore_data_sqlite.conf + diff --git a/src/datastore/plugin_datastore_mysql.c b/src/datastore/plugin_datastore_mysql.c new file mode 100644 index 0000000..76d6ad7 --- /dev/null +++ b/src/datastore/plugin_datastore_mysql.c @@ -0,0 +1,1612 @@ +/* + This file is part of GNUnet + (C) 2009, 2010, 2011 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file datastore/plugin_datastore_mysql.c + * @brief mysql-based datastore backend + * @author Igor Wronsky + * @author Christian Grothoff + * + * NOTE: This db module does NOT work with mysql prior to 4.1 since + * it uses prepared statements. MySQL 5.0.46 promises to fix a bug + * in MyISAM that is causing us grief. At the time of this writing, + * that version is yet to be released. In anticipation, the code + * will use MyISAM with 5.0.46 (and higher). If you run such a + * version, please run "make check" to verify that the MySQL bug + * was actually fixed in your version (and if not, change the + * code below to use MyISAM for gn071). + * + * HIGHLIGHTS + * + * Pros + * + On up-to-date hardware where mysql can be used comfortably, this + * module will have better performance than the other db choices + * (according to our tests). + * + Its often possible to recover the mysql database from internal + * inconsistencies. The other db choices do not support repair! + * Cons + * - Memory usage (Comment: "I have 1G and it never caused me trouble") + * - Manual setup + * + * MANUAL SETUP INSTRUCTIONS + * + * 1) in /etc/gnunet.conf, set + * @verbatim + [datastore] + DATABASE = "mysql" + @endverbatim + * 2) Then access mysql as root, + * @verbatim + $ mysql -u root -p + @endverbatim + * and do the following. [You should replace $USER with the username + * that will be running the gnunetd process]. + * @verbatim + CREATE DATABASE gnunet; + GRANT select,insert,update,delete,create,alter,drop,create temporary tables + ON gnunet.* TO $USER@localhost; + SET PASSWORD FOR $USER@localhost=PASSWORD('$the_password_you_like'); + FLUSH PRIVILEGES; + @endverbatim + * 3) In the $HOME directory of $USER, create a ".my.cnf" file + * with the following lines + * @verbatim + [client] + user=$USER + password=$the_password_you_like + @endverbatim + * + * Thats it. Note that .my.cnf file is a security risk unless its on + * a safe partition etc. The $HOME/.my.cnf can of course be a symbolic + * link. Even greater security risk can be achieved by setting no + * password for $USER. Luckily $USER has only priviledges to mess + * up GNUnet's tables, nothing else (unless you give him more, + * of course).<p> + * + * 4) Still, perhaps you should briefly try if the DB connection + * works. First, login as $USER. Then use, + * + * @verbatim + $ mysql -u $USER -p $the_password_you_like + mysql> use gnunet; + @endverbatim + * + * If you get the message "Database changed" it probably works. + * + * [If you get "ERROR 2002: Can't connect to local MySQL server + * through socket '/tmp/mysql.sock' (2)" it may be resolvable by + * "ln -s /var/run/mysqld/mysqld.sock /tmp/mysql.sock" + * so there may be some additional trouble depending on your mysql setup.] + * + * REPAIRING TABLES + * + * - Its probably healthy to check your tables for inconsistencies + * every now and then. + * - If you get odd SEGVs on gnunetd startup, it might be that the mysql + * databases have been corrupted. + * - The tables can be verified/fixed in two ways; + * 1) by running mysqlcheck -A, or + * 2) by executing (inside of mysql using the GNUnet database): + * @verbatim + mysql> REPAIR TABLE gn090; + @endverbatim + * + * PROBLEMS? + * + * If you have problems related to the mysql module, your best + * friend is probably the mysql manual. The first thing to check + * is that mysql is basically operational, that you can connect + * to it, create tables, issue queries etc. + */ + +#include "platform.h" +#include "gnunet_datastore_plugin.h" +#include "gnunet_util_lib.h" +#include <mysql/mysql.h> + +#define DEBUG_MYSQL GNUNET_EXTRA_LOGGING + +#define MAX_DATUM_SIZE 65536 + +/** + * Maximum number of supported parameters for a prepared + * statement. Increase if needed. + */ +#define MAX_PARAM 16 + +/** + * Die with an error message that indicates + * a failure of the command 'cmd' with the message given + * by strerror(errno). + */ +#define DIE_MYSQL(cmd, dbh) do { GNUNET_log(GNUNET_ERROR_TYPE_ERROR, _("`%s' failed at %s:%d with error: %s\n"), cmd, __FILE__, __LINE__, mysql_error((dbh)->dbf)); GNUNET_abort(); } while(0); + +/** + * Log an error message at log-level 'level' that indicates + * a failure of the command 'cmd' on file 'filename' + * with the message given by strerror(errno). + */ +#define LOG_MYSQL(level, cmd, dbh) do { GNUNET_log(level, _("`%s' failed at %s:%d with error: %s\n"), cmd, __FILE__, __LINE__, mysql_error((dbh)->dbf)); } while(0); + + +struct GNUNET_MysqlStatementHandle +{ + struct GNUNET_MysqlStatementHandle *next; + + struct GNUNET_MysqlStatementHandle *prev; + + char *query; + + MYSQL_STMT *statement; + + int valid; + +}; + + +/** + * Context for all functions in this plugin. + */ +struct Plugin +{ + /** + * Our execution environment. + */ + struct GNUNET_DATASTORE_PluginEnvironment *env; + + /** + * Handle to talk to MySQL. + */ + MYSQL *dbf; + + /** + * We keep all prepared statements in a DLL. This is the head. + */ + struct GNUNET_MysqlStatementHandle *shead; + + /** + * We keep all prepared statements in a DLL. This is the tail. + */ + struct GNUNET_MysqlStatementHandle *stail; + + /** + * Filename of "my.cnf" (msyql configuration). + */ + char *cnffile; + + /** + * Prepared statements. + */ +#define INSERT_ENTRY "INSERT INTO gn090 (repl,type,prio,anonLevel,expire,rvalue,hash,vhash,value) VALUES (?,?,?,?,?,?,?,?,?)" + struct GNUNET_MysqlStatementHandle *insert_entry; + +#define DELETE_ENTRY_BY_UID "DELETE FROM gn090 WHERE uid=?" + struct GNUNET_MysqlStatementHandle *delete_entry_by_uid; + +#define COUNT_ENTRY_BY_HASH "SELECT count(*) FROM gn090 FORCE INDEX (idx_hash) WHERE hash=?" + struct GNUNET_MysqlStatementHandle *count_entry_by_hash; + +#define SELECT_ENTRY_BY_HASH "SELECT type,prio,anonLevel,expire,hash,value,uid FROM gn090 FORCE INDEX (idx_hash) WHERE hash=? ORDER BY uid LIMIT 1 OFFSET ?" + struct GNUNET_MysqlStatementHandle *select_entry_by_hash; + +#define COUNT_ENTRY_BY_HASH_AND_VHASH "SELECT count(*) FROM gn090 FORCE INDEX (idx_hash_vhash) WHERE hash=? AND vhash=?" + struct GNUNET_MysqlStatementHandle *count_entry_by_hash_and_vhash; + +#define SELECT_ENTRY_BY_HASH_AND_VHASH "SELECT type,prio,anonLevel,expire,hash,value,uid FROM gn090 FORCE INDEX (idx_hash_vhash) WHERE hash=? AND vhash=? ORDER BY uid LIMIT 1 OFFSET ?" + struct GNUNET_MysqlStatementHandle *select_entry_by_hash_and_vhash; + +#define COUNT_ENTRY_BY_HASH_AND_TYPE "SELECT count(*) FROM gn090 FORCE INDEX (idx_hash_type_uid) WHERE hash=? AND type=?" + struct GNUNET_MysqlStatementHandle *count_entry_by_hash_and_type; + +#define SELECT_ENTRY_BY_HASH_AND_TYPE "SELECT type,prio,anonLevel,expire,hash,value,uid FROM gn090 FORCE INDEX (idx_hash_type_uid) WHERE hash=? AND type=? ORDER BY uid LIMIT 1 OFFSET ?" + struct GNUNET_MysqlStatementHandle *select_entry_by_hash_and_type; + +#define COUNT_ENTRY_BY_HASH_VHASH_AND_TYPE "SELECT count(*) FROM gn090 FORCE INDEX (idx_hash_vhash) WHERE hash=? AND vhash=? AND type=?" + struct GNUNET_MysqlStatementHandle *count_entry_by_hash_vhash_and_type; + +#define SELECT_ENTRY_BY_HASH_VHASH_AND_TYPE "SELECT type,prio,anonLevel,expire,hash,value,uid FROM gn090 FORCE INDEX (idx_hash_vhash) WHERE hash=? AND vhash=? AND type=? ORDER BY uid ASC LIMIT 1 OFFSET ?" + struct GNUNET_MysqlStatementHandle *select_entry_by_hash_vhash_and_type; + +#define UPDATE_ENTRY "UPDATE gn090 SET prio=prio+?,expire=IF(expire>=?,expire,?) WHERE uid=?" + struct GNUNET_MysqlStatementHandle *update_entry; + +#define DEC_REPL "UPDATE gn090 SET repl=GREATEST (0, repl - 1) WHERE uid=?" + struct GNUNET_MysqlStatementHandle *dec_repl; + +#define SELECT_SIZE "SELECT SUM(BIT_LENGTH(value) DIV 8) FROM gn090" + struct GNUNET_MysqlStatementHandle *get_size; + +#define SELECT_IT_NON_ANONYMOUS "SELECT type,prio,anonLevel,expire,hash,value,uid "\ + "FROM gn090 FORCE INDEX (idx_anonLevel_type_rvalue) "\ + "WHERE anonLevel=0 AND type=? AND "\ + "(rvalue >= ? OR"\ + " NOT EXISTS (SELECT 1 FROM gn090 FORCE INDEX (idx_anonLevel_type_rvalue) WHERE anonLevel=0 AND type=? AND rvalue>=?)) "\ + "ORDER BY rvalue ASC LIMIT 1" + struct GNUNET_MysqlStatementHandle *zero_iter; + +#define SELECT_IT_EXPIRATION "SELECT type,prio,anonLevel,expire,hash,value,uid FROM gn090 FORCE INDEX (idx_expire) WHERE expire < ? ORDER BY expire ASC LIMIT 1" + struct GNUNET_MysqlStatementHandle *select_expiration; + +#define SELECT_IT_PRIORITY "SELECT type,prio,anonLevel,expire,hash,value,uid FROM gn090 FORCE INDEX (idx_prio) ORDER BY prio ASC LIMIT 1" + struct GNUNET_MysqlStatementHandle *select_priority; + +#define SELECT_IT_REPLICATION "SELECT type,prio,anonLevel,expire,hash,value,uid "\ + "FROM gn090 FORCE INDEX (idx_repl_rvalue) "\ + "WHERE repl=? AND "\ + " (rvalue>=? OR"\ + " NOT EXISTS (SELECT 1 FROM gn090 FORCE INDEX (idx_repl_rvalue) WHERE repl=? AND rvalue>=?)) "\ + "ORDER BY rvalue ASC "\ + "LIMIT 1" + struct GNUNET_MysqlStatementHandle *select_replication; + +#define SELECT_MAX_REPL "SELECT MAX(repl) FROM gn090" + struct GNUNET_MysqlStatementHandle *max_repl; + +}; + + +/** + * Obtain the location of ".my.cnf". + * + * @param cfg our configuration + * @return NULL on error + */ +static char * +get_my_cnf_path (const struct GNUNET_CONFIGURATION_Handle *cfg) +{ + char *cnffile; + char *home_dir; + struct stat st; + +#ifndef WINDOWS + struct passwd *pw; +#endif + int configured; + +#ifndef WINDOWS + pw = getpwuid (getuid ()); + if (!pw) + { + GNUNET_log_strerror (GNUNET_ERROR_TYPE_ERROR, "getpwuid"); + return NULL; + } + if (GNUNET_YES == + GNUNET_CONFIGURATION_have_value (cfg, "datastore-mysql", "CONFIG")) + { + GNUNET_assert (GNUNET_OK == + GNUNET_CONFIGURATION_get_value_filename (cfg, + "datastore-mysql", + "CONFIG", + &cnffile)); + configured = GNUNET_YES; + } + else + { + home_dir = GNUNET_strdup (pw->pw_dir); + GNUNET_asprintf (&cnffile, "%s/.my.cnf", home_dir); + GNUNET_free (home_dir); + configured = GNUNET_NO; + } +#else + home_dir = (char *) GNUNET_malloc (_MAX_PATH + 1); + plibc_conv_to_win_path ("~/", home_dir); + GNUNET_asprintf (&cnffile, "%s/.my.cnf", home_dir); + GNUNET_free (home_dir); + configured = GNUNET_NO; +#endif + + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + _("Trying to use file `%s' for MySQL configuration.\n"), cnffile); + if ((0 != STAT (cnffile, &st)) || (0 != ACCESS (cnffile, R_OK)) || + (!S_ISREG (st.st_mode))) + { + if (configured == GNUNET_YES) + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + _("Could not access file `%s': %s\n"), cnffile, + STRERROR (errno)); + GNUNET_free (cnffile); + return NULL; + } + return cnffile; +} + + +/** + * Close database connection and all prepared statements (we got a DB + * disconnect error). + * + * @param plugin plugin context + */ +static int +iclose (struct Plugin *plugin) +{ + struct GNUNET_MysqlStatementHandle *s; + + for (s = plugin->shead; s != NULL; s = s->next) + { + if (s->valid) + { + mysql_stmt_close (s->statement); + s->valid = GNUNET_NO; + } + } + if (plugin->dbf != NULL) + { + mysql_close (plugin->dbf); + plugin->dbf = NULL; + } + return GNUNET_OK; +} + + +/** + * Open the connection with the database (and initialize + * our default options). + * + * @param plugin plugin context + * @return GNUNET_OK on success + */ +static int +iopen (struct Plugin *plugin) +{ + char *mysql_dbname; + char *mysql_server; + char *mysql_user; + char *mysql_password; + unsigned long long mysql_port; + my_bool reconnect; + unsigned int timeout; + + plugin->dbf = mysql_init (NULL); + if (plugin->dbf == NULL) + return GNUNET_SYSERR; + if (plugin->cnffile != NULL) + mysql_options (plugin->dbf, MYSQL_READ_DEFAULT_FILE, plugin->cnffile); + mysql_options (plugin->dbf, MYSQL_READ_DEFAULT_GROUP, "client"); + reconnect = 0; + mysql_options (plugin->dbf, MYSQL_OPT_RECONNECT, &reconnect); + timeout = 120; /* in seconds */ + mysql_options (plugin->dbf, MYSQL_OPT_CONNECT_TIMEOUT, + (const void *) &timeout); + mysql_options (plugin->dbf, MYSQL_SET_CHARSET_NAME, "UTF8"); + timeout = 60; /* in seconds */ + mysql_options (plugin->dbf, MYSQL_OPT_READ_TIMEOUT, (const void *) &timeout); + mysql_options (plugin->dbf, MYSQL_OPT_WRITE_TIMEOUT, (const void *) &timeout); + mysql_dbname = NULL; + if (GNUNET_YES == + GNUNET_CONFIGURATION_have_value (plugin->env->cfg, "datastore-mysql", + "DATABASE")) + GNUNET_assert (GNUNET_OK == + GNUNET_CONFIGURATION_get_value_string (plugin->env->cfg, + "datastore-mysql", + "DATABASE", + &mysql_dbname)); + else + mysql_dbname = GNUNET_strdup ("gnunet"); + mysql_user = NULL; + if (GNUNET_YES == + GNUNET_CONFIGURATION_have_value (plugin->env->cfg, "datastore-mysql", + "USER")) + { + GNUNET_assert (GNUNET_OK == + GNUNET_CONFIGURATION_get_value_string (plugin->env->cfg, + "datastore-mysql", + "USER", &mysql_user)); + } + mysql_password = NULL; + if (GNUNET_YES == + GNUNET_CONFIGURATION_have_value (plugin->env->cfg, "datastore-mysql", + "PASSWORD")) + { + GNUNET_assert (GNUNET_OK == + GNUNET_CONFIGURATION_get_value_string (plugin->env->cfg, + "datastore-mysql", + "PASSWORD", + &mysql_password)); + } + mysql_server = NULL; + if (GNUNET_YES == + GNUNET_CONFIGURATION_have_value (plugin->env->cfg, "datastore-mysql", + "HOST")) + { + GNUNET_assert (GNUNET_OK == + GNUNET_CONFIGURATION_get_value_string (plugin->env->cfg, + "datastore-mysql", + "HOST", + &mysql_server)); + } + mysql_port = 0; + if (GNUNET_YES == + GNUNET_CONFIGURATION_have_value (plugin->env->cfg, "datastore-mysql", + "PORT")) + { + GNUNET_assert (GNUNET_OK == + GNUNET_CONFIGURATION_get_value_number (plugin->env->cfg, + "datastore-mysql", + "PORT", &mysql_port)); + } + + GNUNET_assert (mysql_dbname != NULL); + mysql_real_connect (plugin->dbf, mysql_server, mysql_user, mysql_password, + mysql_dbname, (unsigned int) mysql_port, NULL, + CLIENT_IGNORE_SIGPIPE); + GNUNET_free_non_null (mysql_server); + GNUNET_free_non_null (mysql_user); + GNUNET_free_non_null (mysql_password); + GNUNET_free (mysql_dbname); + if (mysql_error (plugin->dbf)[0]) + { + LOG_MYSQL (GNUNET_ERROR_TYPE_ERROR, "mysql_real_connect", plugin); + return GNUNET_SYSERR; + } + return GNUNET_OK; +} + + +/** + * Run the given MySQL statement. + * + * @param plugin plugin context + * @param statement SQL statement to run + * @return GNUNET_OK on success, GNUNET_SYSERR on error + */ +static int +run_statement (struct Plugin *plugin, const char *statement) +{ + if ((NULL == plugin->dbf) && (GNUNET_OK != iopen (plugin))) + return GNUNET_SYSERR; + mysql_query (plugin->dbf, statement); + if (mysql_error (plugin->dbf)[0]) + { + LOG_MYSQL (GNUNET_ERROR_TYPE_ERROR, "mysql_query", plugin); + iclose (plugin); + return GNUNET_SYSERR; + } + return GNUNET_OK; +} + + +/** + * Create a prepared statement. + * + * @param plugin plugin context + * @param statement SQL statement text to prepare + * @return NULL on error + */ +static struct GNUNET_MysqlStatementHandle * +prepared_statement_create (struct Plugin *plugin, const char *statement) +{ + struct GNUNET_MysqlStatementHandle *ret; + + ret = GNUNET_malloc (sizeof (struct GNUNET_MysqlStatementHandle)); + ret->query = GNUNET_strdup (statement); + GNUNET_CONTAINER_DLL_insert (plugin->shead, plugin->stail, ret); + return ret; +} + + +/** + * Prepare a statement for running. + * + * @param plugin plugin context + * @param ret handle to prepared statement + * @return GNUNET_OK on success + */ +static int +prepare_statement (struct Plugin *plugin, + struct GNUNET_MysqlStatementHandle *ret) +{ + if (GNUNET_YES == ret->valid) + return GNUNET_OK; + if ((NULL == plugin->dbf) && (GNUNET_OK != iopen (plugin))) + return GNUNET_SYSERR; + ret->statement = mysql_stmt_init (plugin->dbf); + if (ret->statement == NULL) + { + iclose (plugin); + return GNUNET_SYSERR; + } + if (mysql_stmt_prepare (ret->statement, ret->query, strlen (ret->query))) + { + GNUNET_log_from (GNUNET_ERROR_TYPE_ERROR, "mysql", + _("Failed to prepare statement `%s'\n"), ret->query); + LOG_MYSQL (GNUNET_ERROR_TYPE_ERROR, "mysql_stmt_prepare", plugin); + mysql_stmt_close (ret->statement); + ret->statement = NULL; + iclose (plugin); + return GNUNET_SYSERR; + } + ret->valid = GNUNET_YES; + return GNUNET_OK; + +} + + +/** + * Bind the parameters for the given MySQL statement + * and run it. + * + * @param plugin plugin context + * @param s statement to bind and run + * @param ap arguments for the binding + * @return GNUNET_SYSERR on error, GNUNET_OK on success + */ +static int +init_params (struct Plugin *plugin, struct GNUNET_MysqlStatementHandle *s, + va_list ap) +{ + MYSQL_BIND qbind[MAX_PARAM]; + unsigned int pc; + unsigned int off; + enum enum_field_types ft; + + pc = mysql_stmt_param_count (s->statement); + if (pc > MAX_PARAM) + { + /* increase internal constant! */ + GNUNET_break (0); + return GNUNET_SYSERR; + } + memset (qbind, 0, sizeof (qbind)); + off = 0; + ft = 0; + while ((pc > 0) && (-1 != (int) (ft = va_arg (ap, enum enum_field_types)))) + { + qbind[off].buffer_type = ft; + switch (ft) + { + case MYSQL_TYPE_FLOAT: + qbind[off].buffer = va_arg (ap, float *); + + break; + case MYSQL_TYPE_LONGLONG: + qbind[off].buffer = va_arg (ap, unsigned long long *); + qbind[off].is_unsigned = va_arg (ap, int); + + break; + case MYSQL_TYPE_LONG: + qbind[off].buffer = va_arg (ap, unsigned int *); + qbind[off].is_unsigned = va_arg (ap, int); + + break; + case MYSQL_TYPE_VAR_STRING: + case MYSQL_TYPE_STRING: + case MYSQL_TYPE_BLOB: + qbind[off].buffer = va_arg (ap, void *); + qbind[off].buffer_length = va_arg (ap, unsigned long); + qbind[off].length = va_arg (ap, unsigned long *); + + break; + default: + /* unsupported type */ + GNUNET_break (0); + return GNUNET_SYSERR; + } + pc--; + off++; + } + if (!((pc == 0) && (-1 != (int) ft) && (va_arg (ap, int) == -1))) + { + GNUNET_assert (0); + return GNUNET_SYSERR; + } + if (mysql_stmt_bind_param (s->statement, qbind)) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + _("`%s' failed at %s:%d with error: %s\n"), + "mysql_stmt_bind_param", __FILE__, __LINE__, + mysql_stmt_error (s->statement)); + iclose (plugin); + return GNUNET_SYSERR; + } + if (mysql_stmt_execute (s->statement)) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + _("`%s' for `%s' failed at %s:%d with error: %s\n"), + "mysql_stmt_execute", s->query, __FILE__, __LINE__, + mysql_stmt_error (s->statement)); + iclose (plugin); + return GNUNET_SYSERR; + } + return GNUNET_OK; +} + + +/** + * Run a prepared SELECT statement. + * + * @param plugin plugin context + * @param s statement to run + * @param result_size number of elements in results array + * @param results pointer to already initialized MYSQL_BIND + * array (of sufficient size) for passing results + * @param ap pairs and triplets of "MYSQL_TYPE_XXX" keys and their respective + * values (size + buffer-reference for pointers); terminated + * with "-1" + * @return GNUNET_SYSERR on error, otherwise GNUNET_OK or GNUNET_NO (no result) + */ +static int +prepared_statement_run_select_va (struct Plugin *plugin, + struct GNUNET_MysqlStatementHandle *s, + unsigned int result_size, + MYSQL_BIND * results, va_list ap) +{ + int ret; + unsigned int rsize; + + if (GNUNET_OK != prepare_statement (plugin, s)) + { + GNUNET_break (0); + return GNUNET_SYSERR; + } + if (GNUNET_OK != init_params (plugin, s, ap)) + { + GNUNET_break (0); + return GNUNET_SYSERR; + } + rsize = mysql_stmt_field_count (s->statement); + if (rsize > result_size) + { + GNUNET_break (0); + return GNUNET_SYSERR; + } + if (mysql_stmt_bind_result (s->statement, results)) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + _("`%s' failed at %s:%d with error: %s\n"), + "mysql_stmt_bind_result", __FILE__, __LINE__, + mysql_stmt_error (s->statement)); + iclose (plugin); + return GNUNET_SYSERR; + } + ret = mysql_stmt_fetch (s->statement); + if (ret == MYSQL_NO_DATA) + return GNUNET_NO; + if (ret != 0) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + _("`%s' failed at %s:%d with error: %s\n"), "mysql_stmt_fetch", + __FILE__, __LINE__, mysql_stmt_error (s->statement)); + iclose (plugin); + return GNUNET_SYSERR; + } + mysql_stmt_reset (s->statement); + return GNUNET_OK; +} + + +/** + * Run a prepared SELECT statement. + * + * @param plugin plugin context + * @param s statement to run + * @param result_size number of elements in results array + * @param results pointer to already initialized MYSQL_BIND + * array (of sufficient size) for passing results + * @param ... pairs and triplets of "MYSQL_TYPE_XXX" keys and their respective + * values (size + buffer-reference for pointers); terminated + * with "-1" + * @return GNUNET_SYSERR on error, otherwise + * the number of successfully affected (or queried) rows + */ +static int +prepared_statement_run_select (struct Plugin *plugin, + struct GNUNET_MysqlStatementHandle *s, + unsigned int result_size, MYSQL_BIND * results, + ...) +{ + va_list ap; + int ret; + + va_start (ap, results); + ret = prepared_statement_run_select_va (plugin, s, result_size, results, ap); + va_end (ap); + return ret; +} + + +/** + * Run a prepared statement that does NOT produce results. + * + * @param plugin plugin context + * @param s statement to run + * @param insert_id NULL or address where to store the row ID of whatever + * was inserted (only for INSERT statements!) + * @param ... pairs and triplets of "MYSQL_TYPE_XXX" keys and their respective + * values (size + buffer-reference for pointers); terminated + * with "-1" + * @return GNUNET_SYSERR on error, otherwise + * the number of successfully affected rows + */ +static int +prepared_statement_run (struct Plugin *plugin, + struct GNUNET_MysqlStatementHandle *s, + unsigned long long *insert_id, ...) +{ + va_list ap; + int affected; + + if (GNUNET_OK != prepare_statement (plugin, s)) + return GNUNET_SYSERR; + va_start (ap, insert_id); + if (GNUNET_OK != init_params (plugin, s, ap)) + { + va_end (ap); + return GNUNET_SYSERR; + } + va_end (ap); + affected = mysql_stmt_affected_rows (s->statement); + if (NULL != insert_id) + *insert_id = (unsigned long long) mysql_stmt_insert_id (s->statement); + mysql_stmt_reset (s->statement); + return affected; +} + + +/** + * Delete an entry from the gn090 table. + * + * @param plugin plugin context + * @param uid unique ID of the entry to delete + * @return GNUNET_OK on success, GNUNET_NO if no such value exists, GNUNET_SYSERR on error + */ +static int +do_delete_entry (struct Plugin *plugin, unsigned long long uid) +{ + int ret; + +#if DEBUG_MYSQL + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Deleting value %llu from gn090 table\n", + uid); +#endif + ret = + prepared_statement_run (plugin, plugin->delete_entry_by_uid, NULL, + MYSQL_TYPE_LONGLONG, &uid, GNUNET_YES, -1); + if (ret >= 0) + return GNUNET_OK; + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + "Deleting value %llu from gn090 table failed\n", uid); + return ret; +} + + +/** + * Get an estimate of how much space the database is + * currently using. + * + * @param cls our "struct Plugin *" + * @return number of bytes used on disk + */ +static unsigned long long +mysql_plugin_estimate_size (void *cls) +{ + struct Plugin *plugin = cls; + MYSQL_BIND cbind[1]; + long long total; + + memset (cbind, 0, sizeof (cbind)); + total = 0; + cbind[0].buffer_type = MYSQL_TYPE_LONGLONG; + cbind[0].buffer = &total; + cbind[0].is_unsigned = GNUNET_NO; + if (GNUNET_OK != + prepared_statement_run_select (plugin, plugin->get_size, 1, cbind, -1)) + return 0; + return total; +} + + +/** + * Store an item in the datastore. + * + * @param cls closure + * @param key key for the item + * @param size number of bytes in data + * @param data content stored + * @param type type of the content + * @param priority priority of the content + * @param anonymity anonymity-level for the content + * @param replication replication-level for the content + * @param expiration expiration time for the content + * @param msg set to error message + * @return GNUNET_OK on success + */ +static int +mysql_plugin_put (void *cls, const GNUNET_HashCode * key, uint32_t size, + const void *data, enum GNUNET_BLOCK_Type type, + uint32_t priority, uint32_t anonymity, uint32_t replication, + struct GNUNET_TIME_Absolute expiration, char **msg) +{ + struct Plugin *plugin = cls; + unsigned int irepl = replication; + unsigned int ipriority = priority; + unsigned int ianonymity = anonymity; + unsigned long long lexpiration = expiration.abs_value; + unsigned long long lrvalue = + (unsigned long long) GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK, + UINT64_MAX); + unsigned long hashSize; + unsigned long hashSize2; + unsigned long lsize; + GNUNET_HashCode vhash; + + if (size > MAX_DATUM_SIZE) + { + GNUNET_break (0); + return GNUNET_SYSERR; + } + hashSize = sizeof (GNUNET_HashCode); + hashSize2 = sizeof (GNUNET_HashCode); + lsize = size; + GNUNET_CRYPTO_hash (data, size, &vhash); + if (GNUNET_OK != + prepared_statement_run (plugin, plugin->insert_entry, NULL, + MYSQL_TYPE_LONG, &irepl, GNUNET_YES, + MYSQL_TYPE_LONG, &type, GNUNET_YES, + MYSQL_TYPE_LONG, &ipriority, GNUNET_YES, + MYSQL_TYPE_LONG, &ianonymity, GNUNET_YES, + MYSQL_TYPE_LONGLONG, &lexpiration, GNUNET_YES, + MYSQL_TYPE_LONGLONG, &lrvalue, GNUNET_YES, + MYSQL_TYPE_BLOB, key, hashSize, &hashSize, + MYSQL_TYPE_BLOB, &vhash, hashSize2, &hashSize2, + MYSQL_TYPE_BLOB, data, lsize, &lsize, -1)) + return GNUNET_SYSERR; +#if DEBUG_MYSQL + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Inserted value `%s' with size %u into gn090 table\n", + GNUNET_h2s (key), (unsigned int) size); +#endif + if (size > 0) + plugin->env->duc (plugin->env->cls, size); + return GNUNET_OK; +} + + +/** + * Update the priority for a particular key in the datastore. If + * the expiration time in value is different than the time found in + * the datastore, the higher value should be kept. For the + * anonymity level, the lower value is to be used. The specified + * priority should be added to the existing priority, ignoring the + * priority in value. + * + * Note that it is possible for multiple values to match this put. + * In that case, all of the respective values are updated. + * + * @param cls our "struct Plugin*" + * @param uid unique identifier of the datum + * @param delta by how much should the priority + * change? If priority + delta < 0 the + * priority should be set to 0 (never go + * negative). + * @param expire new expiration time should be the + * MAX of any existing expiration time and + * this value + * @param msg set to error message + * @return GNUNET_OK on success + */ +static int +mysql_plugin_update (void *cls, uint64_t uid, int delta, + struct GNUNET_TIME_Absolute expire, char **msg) +{ + struct Plugin *plugin = cls; + unsigned long long vkey = uid; + unsigned long long lexpire = expire.abs_value; + int ret; + +#if DEBUG_MYSQL + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Updating value %llu adding %d to priority and maxing exp at %llu\n", + vkey, delta, lexpire); +#endif + ret = + prepared_statement_run (plugin, plugin->update_entry, NULL, + MYSQL_TYPE_LONG, &delta, GNUNET_NO, + MYSQL_TYPE_LONGLONG, &lexpire, GNUNET_YES, + MYSQL_TYPE_LONGLONG, &lexpire, GNUNET_YES, + MYSQL_TYPE_LONGLONG, &vkey, GNUNET_YES, -1); + if (ret != GNUNET_OK) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Failed to update value %llu\n", + vkey); + } + return ret; +} + + +/** + * Run the given select statement and call 'proc' on the resulting + * values (which must be in particular positions). + * + * @param plugin the plugin handle + * @param stmt select statement to run + * @param proc function to call on result + * @param proc_cls closure for proc + * @param ... arguments to initialize stmt + */ +static void +execute_select (struct Plugin *plugin, struct GNUNET_MysqlStatementHandle *stmt, + PluginDatumProcessor proc, void *proc_cls, ...) +{ + va_list ap; + int ret; + unsigned int type; + unsigned int priority; + unsigned int anonymity; + unsigned long long exp; + unsigned long hashSize; + unsigned long size; + unsigned long long uid; + char value[GNUNET_DATASTORE_MAX_VALUE_SIZE]; + GNUNET_HashCode key; + struct GNUNET_TIME_Absolute expiration; + MYSQL_BIND rbind[7]; + + hashSize = sizeof (GNUNET_HashCode); + memset (rbind, 0, sizeof (rbind)); + rbind[0].buffer_type = MYSQL_TYPE_LONG; + rbind[0].buffer = &type; + rbind[0].is_unsigned = 1; + rbind[1].buffer_type = MYSQL_TYPE_LONG; + rbind[1].buffer = &priority; + rbind[1].is_unsigned = 1; + rbind[2].buffer_type = MYSQL_TYPE_LONG; + rbind[2].buffer = &anonymity; + rbind[2].is_unsigned = 1; + rbind[3].buffer_type = MYSQL_TYPE_LONGLONG; + rbind[3].buffer = &exp; + rbind[3].is_unsigned = 1; + rbind[4].buffer_type = MYSQL_TYPE_BLOB; + rbind[4].buffer = &key; + rbind[4].buffer_length = hashSize; + rbind[4].length = &hashSize; + rbind[5].buffer_type = MYSQL_TYPE_BLOB; + rbind[5].buffer = value; + rbind[5].buffer_length = size = sizeof (value); + rbind[5].length = &size; + rbind[6].buffer_type = MYSQL_TYPE_LONGLONG; + rbind[6].buffer = &uid; + rbind[6].is_unsigned = 1; + + va_start (ap, proc_cls); + ret = prepared_statement_run_select_va (plugin, stmt, 7, rbind, ap); + va_end (ap); + if (ret <= 0) + { + proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0); + return; + } + GNUNET_assert (size <= sizeof (value)); + if ((rbind[4].buffer_length != sizeof (GNUNET_HashCode)) || + (hashSize != sizeof (GNUNET_HashCode))) + { + GNUNET_break (0); + proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0); + return; + } +#if DEBUG_MYSQL + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Found %u-byte value under key `%s' with prio %u, anon %u, expire %llu selecting from gn090 table\n", + (unsigned int) size, GNUNET_h2s (&key), priority, anonymity, exp); +#endif + GNUNET_assert (size < MAX_DATUM_SIZE); + expiration.abs_value = exp; + ret = + proc (proc_cls, &key, size, value, type, priority, anonymity, expiration, + uid); + if (ret == GNUNET_NO) + { + do_delete_entry (plugin, uid); + if (size != 0) + plugin->env->duc (plugin->env->cls, -size); + } +} + + + +/** + * Get one of the results for a particular key in the datastore. + * + * @param cls closure + * @param offset offset of the result (modulo num-results); + * specific ordering does not matter for the offset + * @param key key to match, never NULL + * @param vhash hash of the value, maybe NULL (to + * match all values that have the right key). + * Note that for DBlocks there is no difference + * betwen key and vhash, but for other blocks + * there may be! + * @param type entries of which type are relevant? + * Use 0 for any type. + * @param proc function to call on the matching value, + * with NULL for if no value matches + * @param proc_cls closure for proc + */ +static void +mysql_plugin_get_key (void *cls, uint64_t offset, const GNUNET_HashCode * key, + const GNUNET_HashCode * vhash, + enum GNUNET_BLOCK_Type type, PluginDatumProcessor proc, + void *proc_cls) +{ + struct Plugin *plugin = cls; + int ret; + MYSQL_BIND cbind[1]; + long long total; + unsigned long hashSize; + unsigned long hashSize2; + unsigned long long off; + + GNUNET_assert (key != NULL); + GNUNET_assert (NULL != proc); + hashSize = sizeof (GNUNET_HashCode); + hashSize2 = sizeof (GNUNET_HashCode); + memset (cbind, 0, sizeof (cbind)); + total = -1; + cbind[0].buffer_type = MYSQL_TYPE_LONGLONG; + cbind[0].buffer = &total; + cbind[0].is_unsigned = GNUNET_NO; + if (type != 0) + { + if (vhash != NULL) + { + ret = + prepared_statement_run_select (plugin, + plugin-> + count_entry_by_hash_vhash_and_type, 1, + cbind, MYSQL_TYPE_BLOB, key, hashSize, + &hashSize, MYSQL_TYPE_BLOB, vhash, + hashSize2, &hashSize2, MYSQL_TYPE_LONG, + &type, GNUNET_YES, -1); + } + else + { + ret = + prepared_statement_run_select (plugin, + plugin->count_entry_by_hash_and_type, + 1, cbind, MYSQL_TYPE_BLOB, key, + hashSize, &hashSize, MYSQL_TYPE_LONG, + &type, GNUNET_YES, -1); + } + } + else + { + if (vhash != NULL) + { + ret = + prepared_statement_run_select (plugin, + plugin->count_entry_by_hash_and_vhash, + 1, cbind, MYSQL_TYPE_BLOB, key, + hashSize, &hashSize, MYSQL_TYPE_BLOB, + vhash, hashSize2, &hashSize2, -1); + + } + else + { + ret = + prepared_statement_run_select (plugin, plugin->count_entry_by_hash, 1, + cbind, MYSQL_TYPE_BLOB, key, hashSize, + &hashSize, -1); + } + } + if ((ret != GNUNET_OK) || (0 >= total)) + { + proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0); + return; + } + offset = offset % total; + off = (unsigned long long) offset; +#if DEBUG_MYSQL + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Obtaining %llu/%lld result for GET `%s'\n", off, total, + GNUNET_h2s (key)); +#endif + + if (type != GNUNET_BLOCK_TYPE_ANY) + { + if (NULL != vhash) + { + execute_select (plugin, plugin->select_entry_by_hash_vhash_and_type, proc, + proc_cls, MYSQL_TYPE_BLOB, key, hashSize, &hashSize, + MYSQL_TYPE_BLOB, vhash, hashSize, &hashSize, + MYSQL_TYPE_LONG, &type, GNUNET_YES, MYSQL_TYPE_LONGLONG, + &off, GNUNET_YES, -1); + } + else + { + execute_select (plugin, plugin->select_entry_by_hash_and_type, proc, + proc_cls, MYSQL_TYPE_BLOB, key, hashSize, &hashSize, + MYSQL_TYPE_LONG, &type, GNUNET_YES, MYSQL_TYPE_LONGLONG, + &off, GNUNET_YES, -1); + } + } + else + { + if (NULL != vhash) + { + execute_select (plugin, plugin->select_entry_by_hash_and_vhash, proc, + proc_cls, MYSQL_TYPE_BLOB, key, hashSize, &hashSize, + MYSQL_TYPE_BLOB, vhash, hashSize, &hashSize, + MYSQL_TYPE_LONGLONG, &off, GNUNET_YES, -1); + } + else + { + execute_select (plugin, plugin->select_entry_by_hash, proc, proc_cls, + MYSQL_TYPE_BLOB, key, hashSize, &hashSize, + MYSQL_TYPE_LONGLONG, &off, GNUNET_YES, -1); + } + } +} + + +/** + * Get a zero-anonymity datum from the datastore. + * + * @param cls our "struct Plugin*" + * @param offset offset of the result + * @param type entries of which type should be considered? + * Use 0 for any type. + * @param proc function to call on a matching value or NULL + * @param proc_cls closure for iter + */ +static void +mysql_plugin_get_zero_anonymity (void *cls, uint64_t offset, + enum GNUNET_BLOCK_Type type, + PluginDatumProcessor proc, void *proc_cls) +{ + struct Plugin *plugin = cls; + unsigned long long rvalue = + (unsigned long long) GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK, + UINT64_MAX); + + execute_select (plugin, plugin->zero_iter, proc, proc_cls, MYSQL_TYPE_LONG, + &type, GNUNET_YES, MYSQL_TYPE_LONGLONG, &rvalue, GNUNET_YES, + MYSQL_TYPE_LONG, &type, GNUNET_YES, MYSQL_TYPE_LONGLONG, + &rvalue, GNUNET_YES, -1); +} + + +/** + * Context for 'repl_proc' function. + */ +struct ReplCtx +{ + + /** + * Plugin handle. + */ + struct Plugin *plugin; + + /** + * Function to call for the result (or the NULL). + */ + PluginDatumProcessor proc; + + /** + * Closure for proc. + */ + void *proc_cls; +}; + + +/** + * Wrapper for the processor for 'mysql_plugin_get_replication'. + * Decrements the replication counter and calls the original + * iterator. + * + * @param cls closure + * @param key key for the content + * @param size number of bytes in data + * @param data content stored + * @param type type of the content + * @param priority priority of the content + * @param anonymity anonymity-level for the content + * @param expiration expiration time for the content + * @param uid unique identifier for the datum; + * maybe 0 if no unique identifier is available + * + * @return GNUNET_SYSERR to abort the iteration, GNUNET_OK to continue + * (continue on call to "next", of course), + * GNUNET_NO to delete the item and continue (if supported) + */ +static int +repl_proc (void *cls, const GNUNET_HashCode * key, uint32_t size, + const void *data, enum GNUNET_BLOCK_Type type, uint32_t priority, + uint32_t anonymity, struct GNUNET_TIME_Absolute expiration, + uint64_t uid) +{ + struct ReplCtx *rc = cls; + struct Plugin *plugin = rc->plugin; + unsigned long long oid; + int ret; + int iret; + + ret = + rc->proc (rc->proc_cls, key, size, data, type, priority, anonymity, + expiration, uid); + if (NULL != key) + { + oid = (unsigned long long) uid; + iret = + prepared_statement_run (plugin, plugin->dec_repl, NULL, + MYSQL_TYPE_LONGLONG, &oid, GNUNET_YES, -1); + if (iret == GNUNET_SYSERR) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + "Failed to reduce replication counter\n"); + return GNUNET_SYSERR; + } + } + return ret; +} + + +/** + * Get a random item for replication. Returns a single, not expired, + * random item from those with the highest replication counters. The + * item's replication counter is decremented by one IF it was positive + * before. Call 'proc' with all values ZERO or NULL if the datastore + * is empty. + * + * @param cls closure + * @param proc function to call the value (once only). + * @param proc_cls closure for proc + */ +static void +mysql_plugin_get_replication (void *cls, PluginDatumProcessor proc, + void *proc_cls) +{ + struct Plugin *plugin = cls; + struct ReplCtx rc; + unsigned long long rvalue; + unsigned long repl; + MYSQL_BIND results; + + rc.plugin = plugin; + rc.proc = proc; + rc.proc_cls = proc_cls; + memset (&results, 0, sizeof (results)); + results.buffer_type = MYSQL_TYPE_LONG; + results.buffer = &repl; + results.is_unsigned = GNUNET_YES; + + if (1 != + prepared_statement_run_select (plugin, plugin->max_repl, 1, &results, -1)) + { + proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0); + return; + } + + rvalue = + (unsigned long long) GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK, + UINT64_MAX); + execute_select (plugin, plugin->select_replication, &repl_proc, &rc, + MYSQL_TYPE_LONG, &repl, GNUNET_YES, MYSQL_TYPE_LONGLONG, + &rvalue, GNUNET_YES, MYSQL_TYPE_LONG, &repl, GNUNET_YES, + MYSQL_TYPE_LONGLONG, &rvalue, GNUNET_YES, -1); + +} + + +/** + * Get all of the keys in the datastore. + * + * @param cls closure + * @param proc function to call on each key + * @param proc_cls closure for proc + */ +static void +mysql_plugin_get_keys (void *cls, + PluginKeyProcessor proc, + void *proc_cls) +{ + struct Plugin *plugin = cls; + const char *query = "SELECT hash FROM gn090"; + int ret; + MYSQL_STMT *statement; + GNUNET_HashCode key; + MYSQL_BIND cbind[1]; + unsigned long length; + + statement = mysql_stmt_init (plugin->dbf); + if (statement == NULL) + { + iclose (plugin); + return; + } + if (mysql_stmt_prepare (statement, query, strlen (query))) + { + GNUNET_log_from (GNUNET_ERROR_TYPE_ERROR, "mysql", + _("Failed to prepare statement `%s'\n"), query); + LOG_MYSQL (GNUNET_ERROR_TYPE_ERROR, "mysql_stmt_prepare", plugin); + mysql_stmt_close (statement); + iclose (plugin); + return; + } + GNUNET_assert (proc != NULL); + if (mysql_stmt_execute (statement)) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + _("`%s' for `%s' failed at %s:%d with error: %s\n"), + "mysql_stmt_execute", query, __FILE__, __LINE__, + mysql_stmt_error (statement)); + mysql_stmt_close (statement); + iclose (plugin); + return; + } + memset (cbind, 0, sizeof (cbind)); + cbind[0].buffer_type = MYSQL_TYPE_BLOB; + cbind[0].buffer = &key; + cbind[0].buffer_length = sizeof (key); + cbind[0].length = &length; + cbind[0].is_unsigned = GNUNET_NO; + if (mysql_stmt_bind_result (statement, cbind)) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + _("`%s' failed at %s:%d with error: %s\n"), + "mysql_stmt_bind_result", __FILE__, __LINE__, + mysql_stmt_error (statement)); + iclose (plugin); + return; + } + while (0 == (ret = mysql_stmt_fetch (statement))) + { + if (sizeof (GNUNET_HashCode) == length) + proc (proc_cls, &key, 1); + } + if (ret != MYSQL_NO_DATA) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + _("`%s' failed at %s:%d with error: %s\n"), + "mysql_stmt_fetch", __FILE__, __LINE__, + mysql_stmt_error (statement)); + mysql_stmt_close (statement); + iclose (plugin); + return; + } + mysql_stmt_close (statement); +} + + +/** + * Context for 'expi_proc' function. + */ +struct ExpiCtx +{ + + /** + * Plugin handle. + */ + struct Plugin *plugin; + + /** + * Function to call for the result (or the NULL). + */ + PluginDatumProcessor proc; + + /** + * Closure for proc. + */ + void *proc_cls; +}; + + + +/** + * Wrapper for the processor for 'mysql_plugin_get_expiration'. + * If no expired value was found, we do a second query for + * low-priority content. + * + * @param cls closure + * @param key key for the content + * @param size number of bytes in data + * @param data content stored + * @param type type of the content + * @param priority priority of the content + * @param anonymity anonymity-level for the content + * @param expiration expiration time for the content + * @param uid unique identifier for the datum; + * maybe 0 if no unique identifier is available + * + * @return GNUNET_SYSERR to abort the iteration, GNUNET_OK to continue + * (continue on call to "next", of course), + * GNUNET_NO to delete the item and continue (if supported) + */ +static int +expi_proc (void *cls, const GNUNET_HashCode * key, uint32_t size, + const void *data, enum GNUNET_BLOCK_Type type, uint32_t priority, + uint32_t anonymity, struct GNUNET_TIME_Absolute expiration, + uint64_t uid) +{ + struct ExpiCtx *rc = cls; + struct Plugin *plugin = rc->plugin; + + if (NULL == key) + { + execute_select (plugin, plugin->select_priority, rc->proc, rc->proc_cls, + -1); + return GNUNET_SYSERR; + } + return rc->proc (rc->proc_cls, key, size, data, type, priority, anonymity, + expiration, uid); +} + + +/** + * Get a random item for expiration. + * Call 'proc' with all values ZERO or NULL if the datastore is empty. + * + * @param cls closure + * @param proc function to call the value (once only). + * @param proc_cls closure for proc + */ +static void +mysql_plugin_get_expiration (void *cls, PluginDatumProcessor proc, + void *proc_cls) +{ + struct Plugin *plugin = cls; + long long nt; + struct ExpiCtx rc; + + rc.plugin = plugin; + rc.proc = proc; + rc.proc_cls = proc_cls; + nt = (long long) GNUNET_TIME_absolute_get ().abs_value; + execute_select (plugin, plugin->select_expiration, expi_proc, &rc, + MYSQL_TYPE_LONGLONG, &nt, GNUNET_YES, -1); + +} + + +/** + * Drop database. + * + * @param cls the "struct Plugin*" + */ +static void +mysql_plugin_drop (void *cls) +{ + struct Plugin *plugin = cls; + + if (GNUNET_OK != run_statement (plugin, "DROP TABLE gn090")) + return; /* error */ + plugin->env->duc (plugin->env->cls, 0); +} + + +/** + * Entry point for the plugin. + * + * @param cls the "struct GNUNET_DATASTORE_PluginEnvironment*" + * @return our "struct Plugin*" + */ +void * +libgnunet_plugin_datastore_mysql_init (void *cls) +{ + struct GNUNET_DATASTORE_PluginEnvironment *env = cls; + struct GNUNET_DATASTORE_PluginFunctions *api; + struct Plugin *plugin; + + plugin = GNUNET_malloc (sizeof (struct Plugin)); + plugin->env = env; + plugin->cnffile = get_my_cnf_path (env->cfg); + if (GNUNET_OK != iopen (plugin)) + { + iclose (plugin); + GNUNET_free_non_null (plugin->cnffile); + GNUNET_free (plugin); + return NULL; + } +#define MRUNS(a) (GNUNET_OK != run_statement (plugin, a) ) +#define PINIT(a,b) (NULL == (a = prepared_statement_create(plugin, b))) + if (MRUNS + ("CREATE TABLE IF NOT EXISTS gn090 (" + " repl INT(11) UNSIGNED NOT NULL DEFAULT 0," + " type INT(11) UNSIGNED NOT NULL DEFAULT 0," + " prio INT(11) UNSIGNED NOT NULL DEFAULT 0," + " anonLevel INT(11) UNSIGNED NOT NULL DEFAULT 0," + " expire BIGINT UNSIGNED NOT NULL DEFAULT 0," + " rvalue BIGINT UNSIGNED NOT NULL," + " hash BINARY(64) NOT NULL DEFAULT ''," + " vhash BINARY(64) NOT NULL DEFAULT ''," + " value BLOB NOT NULL DEFAULT ''," " uid BIGINT NOT NULL AUTO_INCREMENT," + " PRIMARY KEY (uid)," " INDEX idx_hash (hash(64))," + " INDEX idx_hash_uid (hash(64),uid)," + " INDEX idx_hash_vhash (hash(64),vhash(64))," + " INDEX idx_hash_type_uid (hash(64),type,rvalue)," + " INDEX idx_prio (prio)," " INDEX idx_repl_rvalue (repl,rvalue)," + " INDEX idx_expire (expire)," + " INDEX idx_anonLevel_type_rvalue (anonLevel,type,rvalue)" + ") ENGINE=InnoDB") || MRUNS ("SET AUTOCOMMIT = 1") || + PINIT (plugin->insert_entry, INSERT_ENTRY) || + PINIT (plugin->delete_entry_by_uid, DELETE_ENTRY_BY_UID) || + PINIT (plugin->select_entry_by_hash, SELECT_ENTRY_BY_HASH) || + PINIT (plugin->select_entry_by_hash_and_vhash, + SELECT_ENTRY_BY_HASH_AND_VHASH) || + PINIT (plugin->select_entry_by_hash_and_type, + SELECT_ENTRY_BY_HASH_AND_TYPE) || + PINIT (plugin->select_entry_by_hash_vhash_and_type, + SELECT_ENTRY_BY_HASH_VHASH_AND_TYPE) || + PINIT (plugin->count_entry_by_hash, COUNT_ENTRY_BY_HASH) || + PINIT (plugin->get_size, SELECT_SIZE) || + PINIT (plugin->count_entry_by_hash_and_vhash, + COUNT_ENTRY_BY_HASH_AND_VHASH) || + PINIT (plugin->count_entry_by_hash_and_type, COUNT_ENTRY_BY_HASH_AND_TYPE) + || PINIT (plugin->count_entry_by_hash_vhash_and_type, + COUNT_ENTRY_BY_HASH_VHASH_AND_TYPE) || + PINIT (plugin->update_entry, UPDATE_ENTRY) || + PINIT (plugin->dec_repl, DEC_REPL) || + PINIT (plugin->zero_iter, SELECT_IT_NON_ANONYMOUS) || + PINIT (plugin->select_expiration, SELECT_IT_EXPIRATION) || + PINIT (plugin->select_priority, SELECT_IT_PRIORITY) || + PINIT (plugin->max_repl, SELECT_MAX_REPL) || + PINIT (plugin->select_replication, SELECT_IT_REPLICATION)) + { + iclose (plugin); + GNUNET_free_non_null (plugin->cnffile); + GNUNET_free (plugin); + return NULL; + } +#undef PINIT +#undef MRUNS + + api = GNUNET_malloc (sizeof (struct GNUNET_DATASTORE_PluginFunctions)); + api->cls = plugin; + api->estimate_size = &mysql_plugin_estimate_size; + api->put = &mysql_plugin_put; + api->update = &mysql_plugin_update; + api->get_key = &mysql_plugin_get_key; + api->get_replication = &mysql_plugin_get_replication; + api->get_expiration = &mysql_plugin_get_expiration; + api->get_zero_anonymity = &mysql_plugin_get_zero_anonymity; + api->get_keys = &mysql_plugin_get_keys; + api->drop = &mysql_plugin_drop; + GNUNET_log_from (GNUNET_ERROR_TYPE_INFO, "mysql", + _("Mysql database running\n")); + return api; +} + + +/** + * Exit point from the plugin. + * @param cls our "struct Plugin*" + * @return always NULL + */ +void * +libgnunet_plugin_datastore_mysql_done (void *cls) +{ + struct GNUNET_DATASTORE_PluginFunctions *api = cls; + struct Plugin *plugin = api->cls; + struct GNUNET_MysqlStatementHandle *s; + + iclose (plugin); + while (NULL != (s = plugin->shead)) + { + GNUNET_CONTAINER_DLL_remove (plugin->shead, plugin->stail, s); + GNUNET_free (s->query); + GNUNET_free (s); + } + GNUNET_free_non_null (plugin->cnffile); + GNUNET_free (plugin); + GNUNET_free (api); + mysql_library_end (); + return NULL; +} + +/* end of plugin_datastore_mysql.c */ diff --git a/src/datastore/plugin_datastore_postgres.c b/src/datastore/plugin_datastore_postgres.c new file mode 100644 index 0000000..16393c2 --- /dev/null +++ b/src/datastore/plugin_datastore_postgres.c @@ -0,0 +1,1039 @@ +/* + This file is part of GNUnet + (C) 2009, 2010, 2011 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file datastore/plugin_datastore_postgres.c + * @brief postgres-based datastore backend + * @author Christian Grothoff + */ + +#include "platform.h" +#include "gnunet_datastore_plugin.h" +#include <postgresql/libpq-fe.h> + +#define DEBUG_POSTGRES GNUNET_EXTRA_LOGGING + +/** + * After how many ms "busy" should a DB operation fail for good? + * A low value makes sure that we are more responsive to requests + * (especially PUTs). A high value guarantees a higher success + * rate (SELECTs in iterate can take several seconds despite LIMIT=1). + * + * The default value of 1s should ensure that users do not experience + * huge latencies while at the same time allowing operations to succeed + * with reasonable probability. + */ +#define BUSY_TIMEOUT GNUNET_TIME_UNIT_SECONDS + + +/** + * Context for all functions in this plugin. + */ +struct Plugin +{ + /** + * Our execution environment. + */ + struct GNUNET_DATASTORE_PluginEnvironment *env; + + /** + * Native Postgres database handle. + */ + PGconn *dbh; + +}; + + +/** + * Check if the result obtained from Postgres has + * the desired status code. If not, log an error, clear the + * result and return GNUNET_SYSERR. + * + * @param plugin global context + * @param ret result to check + * @param expected_status expected return value + * @param command name of SQL command that was run + * @param args arguments to SQL command + * @param line line number for error reporting + * @return GNUNET_OK if the result is acceptable + */ +static int +check_result (struct Plugin *plugin, PGresult * ret, int expected_status, + const char *command, const char *args, int line) +{ + if (ret == NULL) + { + GNUNET_log_from (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + "datastore-postgres", + "Postgres failed to allocate result for `%s:%s' at %d\n", + command, args, line); + return GNUNET_SYSERR; + } + if (PQresultStatus (ret) != expected_status) + { + GNUNET_log_from (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + "datastore-postgres", + _("`%s:%s' failed at %s:%d with error: %s"), command, args, + __FILE__, line, PQerrorMessage (plugin->dbh)); + PQclear (ret); + return GNUNET_SYSERR; + } + return GNUNET_OK; +} + +/** + * Run simple SQL statement (without results). + * + * @param plugin global context + * @param sql statement to run + * @param line code line for error reporting + */ +static int +pq_exec (struct Plugin *plugin, const char *sql, int line) +{ + PGresult *ret; + + ret = PQexec (plugin->dbh, sql); + if (GNUNET_OK != + check_result (plugin, ret, PGRES_COMMAND_OK, "PQexec", sql, line)) + return GNUNET_SYSERR; + PQclear (ret); + return GNUNET_OK; +} + +/** + * Prepare SQL statement. + * + * @param plugin global context + * @param name name for the prepared SQL statement + * @param sql SQL code to prepare + * @param nparams number of parameters in sql + * @param line code line for error reporting + * @return GNUNET_OK on success + */ +static int +pq_prepare (struct Plugin *plugin, const char *name, const char *sql, + int nparams, int line) +{ + PGresult *ret; + + ret = PQprepare (plugin->dbh, name, sql, nparams, NULL); + if (GNUNET_OK != + check_result (plugin, ret, PGRES_COMMAND_OK, "PQprepare", sql, line)) + return GNUNET_SYSERR; + PQclear (ret); + return GNUNET_OK; +} + +/** + * @brief Get a database handle + * + * @param plugin global context + * @return GNUNET_OK on success, GNUNET_SYSERR on error + */ +static int +init_connection (struct Plugin *plugin) +{ + char *conninfo; + PGresult *ret; + + /* Open database and precompile statements */ + conninfo = NULL; + (void) GNUNET_CONFIGURATION_get_value_string (plugin->env->cfg, + "datastore-postgres", "CONFIG", + &conninfo); + plugin->dbh = PQconnectdb (conninfo == NULL ? "" : conninfo); + if (NULL == plugin->dbh) + { + /* FIXME: warn about out-of-memory? */ + GNUNET_free_non_null (conninfo); + return GNUNET_SYSERR; + } + if (PQstatus (plugin->dbh) != CONNECTION_OK) + { + GNUNET_log_from (GNUNET_ERROR_TYPE_ERROR, "datastore-postgres", + _ + ("Unable to initialize Postgres with configuration `%s': %s"), + conninfo, PQerrorMessage (plugin->dbh)); + PQfinish (plugin->dbh); + plugin->dbh = NULL; + GNUNET_free_non_null (conninfo); + return GNUNET_SYSERR; + } + GNUNET_free_non_null (conninfo); + ret = + PQexec (plugin->dbh, + "CREATE TABLE gn090 (" " repl INTEGER NOT NULL DEFAULT 0," + " type INTEGER NOT NULL DEFAULT 0," + " prio INTEGER NOT NULL DEFAULT 0," + " anonLevel INTEGER NOT NULL DEFAULT 0," + " expire BIGINT NOT NULL DEFAULT 0," + " rvalue BIGINT NOT NULL DEFAULT 0," + " hash BYTEA NOT NULL DEFAULT ''," + " vhash BYTEA NOT NULL DEFAULT ''," + " value BYTEA NOT NULL DEFAULT '')" "WITH OIDS"); + if ((ret == NULL) || ((PQresultStatus (ret) != PGRES_COMMAND_OK) && (0 != strcmp ("42P07", /* duplicate table */ + PQresultErrorField + (ret, + PG_DIAG_SQLSTATE))))) + { + (void) check_result (plugin, ret, PGRES_COMMAND_OK, "CREATE TABLE", "gn090", + __LINE__); + PQfinish (plugin->dbh); + plugin->dbh = NULL; + return GNUNET_SYSERR; + } + if (PQresultStatus (ret) == PGRES_COMMAND_OK) + { + if ((GNUNET_OK != + pq_exec (plugin, "CREATE INDEX idx_hash ON gn090 (hash)", __LINE__)) || + (GNUNET_OK != + pq_exec (plugin, "CREATE INDEX idx_hash_vhash ON gn090 (hash,vhash)", + __LINE__)) || + (GNUNET_OK != + pq_exec (plugin, "CREATE INDEX idx_prio ON gn090 (prio)", __LINE__)) || + (GNUNET_OK != + pq_exec (plugin, "CREATE INDEX idx_expire ON gn090 (expire)", + __LINE__)) || + (GNUNET_OK != + pq_exec (plugin, + "CREATE INDEX idx_prio_anon ON gn090 (prio,anonLevel)", + __LINE__)) || + (GNUNET_OK != + pq_exec (plugin, + "CREATE INDEX idx_prio_hash_anon ON gn090 (prio,hash,anonLevel)", + __LINE__)) || + (GNUNET_OK != + pq_exec (plugin, "CREATE INDEX idx_repl_rvalue ON gn090 (repl,rvalue)", + __LINE__)) || + (GNUNET_OK != + pq_exec (plugin, "CREATE INDEX idx_expire_hash ON gn090 (expire,hash)", + __LINE__))) + { + PQclear (ret); + PQfinish (plugin->dbh); + plugin->dbh = NULL; + return GNUNET_SYSERR; + } + } + PQclear (ret); + ret = + PQexec (plugin->dbh, + "ALTER TABLE gn090 ALTER value SET STORAGE EXTERNAL"); + if (GNUNET_OK != + check_result (plugin, ret, PGRES_COMMAND_OK, "ALTER TABLE", "gn090", + __LINE__)) + { + PQfinish (plugin->dbh); + plugin->dbh = NULL; + return GNUNET_SYSERR; + } + PQclear (ret); + ret = PQexec (plugin->dbh, "ALTER TABLE gn090 ALTER hash SET STORAGE PLAIN"); + if (GNUNET_OK != + check_result (plugin, ret, PGRES_COMMAND_OK, "ALTER TABLE", "gn090", + __LINE__)) + { + PQfinish (plugin->dbh); + plugin->dbh = NULL; + return GNUNET_SYSERR; + } + PQclear (ret); + ret = PQexec (plugin->dbh, "ALTER TABLE gn090 ALTER vhash SET STORAGE PLAIN"); + if (GNUNET_OK != + check_result (plugin, ret, PGRES_COMMAND_OK, "ALTER TABLE", "gn090", + __LINE__)) + { + PQfinish (plugin->dbh); + plugin->dbh = NULL; + return GNUNET_SYSERR; + } + PQclear (ret); + if ((GNUNET_OK != + pq_prepare (plugin, "getvt", + "SELECT type, prio, anonLevel, expire, hash, value, oid FROM gn090 " + "WHERE hash=$1 AND vhash=$2 AND type=$3 " + "ORDER BY oid ASC LIMIT 1 OFFSET $4", 4, __LINE__)) || + (GNUNET_OK != + pq_prepare (plugin, "gett", + "SELECT type, prio, anonLevel, expire, hash, value, oid FROM gn090 " + "WHERE hash=$1 AND type=$2 " + "ORDER BY oid ASC LIMIT 1 OFFSET $3", 3, __LINE__)) || + (GNUNET_OK != + pq_prepare (plugin, "getv", + "SELECT type, prio, anonLevel, expire, hash, value, oid FROM gn090 " + "WHERE hash=$1 AND vhash=$2 " + "ORDER BY oid ASC LIMIT 1 OFFSET $3", 3, __LINE__)) || + (GNUNET_OK != + pq_prepare (plugin, "get", + "SELECT type, prio, anonLevel, expire, hash, value, oid FROM gn090 " + "WHERE hash=$1 " "ORDER BY oid ASC LIMIT 1 OFFSET $2", 2, + __LINE__)) || + (GNUNET_OK != + pq_prepare (plugin, "put", + "INSERT INTO gn090 (repl, type, prio, anonLevel, expire, rvalue, hash, vhash, value) " + "VALUES ($1, $2, $3, $4, $5, RANDOM(), $6, $7, $8)", 9, + __LINE__)) || + (GNUNET_OK != + pq_prepare (plugin, "update", + "UPDATE gn090 SET prio = prio + $1, expire = CASE WHEN expire < $2 THEN $2 ELSE expire END " + "WHERE oid = $3", 3, __LINE__)) || + (GNUNET_OK != + pq_prepare (plugin, "decrepl", + "UPDATE gn090 SET repl = GREATEST (repl - 1, 0) " + "WHERE oid = $1", 1, __LINE__)) || + (GNUNET_OK != + pq_prepare (plugin, "select_non_anonymous", + "SELECT type, prio, anonLevel, expire, hash, value, oid FROM gn090 " + "WHERE anonLevel = 0 AND type = $1 ORDER BY oid DESC LIMIT 1 OFFSET $2", + 1, __LINE__)) || + (GNUNET_OK != + pq_prepare (plugin, "select_expiration_order", + "(SELECT type, prio, anonLevel, expire, hash, value, oid FROM gn090 " + "WHERE expire < $1 ORDER BY prio ASC LIMIT 1) " "UNION " + "(SELECT type, prio, anonLevel, expire, hash, value, oid FROM gn090 " + "ORDER BY prio ASC LIMIT 1) " "ORDER BY expire ASC LIMIT 1", + 1, __LINE__)) || + (GNUNET_OK != + pq_prepare (plugin, "select_replication_order", + "SELECT type, prio, anonLevel, expire, hash, value, oid FROM gn090 " + "ORDER BY repl DESC,RANDOM() LIMIT 1", 0, __LINE__)) || + (GNUNET_OK != + pq_prepare (plugin, "delrow", "DELETE FROM gn090 " "WHERE oid=$1", 1, + __LINE__)) || + (GNUNET_OK != + pq_prepare (plugin, "get_keys", "SELECT hash FROM gn090", 0, + __LINE__))) + { + PQfinish (plugin->dbh); + plugin->dbh = NULL; + return GNUNET_SYSERR; + } + return GNUNET_OK; +} + + +/** + * Delete the row identified by the given rowid (qid + * in postgres). + * + * @param plugin global context + * @param rowid which row to delete + * @return GNUNET_OK on success + */ +static int +delete_by_rowid (struct Plugin *plugin, unsigned int rowid) +{ + uint32_t browid; + const char *paramValues[] = { (const char *) &browid }; + int paramLengths[] = { sizeof (browid) }; + const int paramFormats[] = { 1 }; + PGresult *ret; + + browid = htonl (rowid); + ret = + PQexecPrepared (plugin->dbh, "delrow", 1, paramValues, paramLengths, + paramFormats, 1); + if (GNUNET_OK != + check_result (plugin, ret, PGRES_COMMAND_OK, "PQexecPrepared", "delrow", + __LINE__)) + { + return GNUNET_SYSERR; + } + PQclear (ret); + return GNUNET_OK; +} + + +/** + * Get an estimate of how much space the database is + * currently using. + * + * @param cls our "struct Plugin*" + * @return number of bytes used on disk + */ +static unsigned long long +postgres_plugin_estimate_size (void *cls) +{ + struct Plugin *plugin = cls; + unsigned long long total; + PGresult *ret; + + ret = + PQexecParams (plugin->dbh, + "SELECT SUM(LENGTH(value))+256*COUNT(*) FROM gn090", 0, + NULL, NULL, NULL, NULL, 1); + if (GNUNET_OK != + check_result (plugin, ret, PGRES_TUPLES_OK, "PQexecParams", "get_size", + __LINE__)) + { + return 0; + } + if ((PQntuples (ret) != 1) || (PQnfields (ret) != 1) || + (PQgetlength (ret, 0, 0) != sizeof (unsigned long long))) + { + GNUNET_break (0); + PQclear (ret); + return 0; + } + total = GNUNET_ntohll (*(const unsigned long long *) PQgetvalue (ret, 0, 0)); + PQclear (ret); + return total; +} + + +/** + * Store an item in the datastore. + * + * @param cls closure + * @param key key for the item + * @param size number of bytes in data + * @param data content stored + * @param type type of the content + * @param priority priority of the content + * @param anonymity anonymity-level for the content + * @param replication replication-level for the content + * @param expiration expiration time for the content + * @param msg set to error message + * @return GNUNET_OK on success + */ +static int +postgres_plugin_put (void *cls, const GNUNET_HashCode * key, uint32_t size, + const void *data, enum GNUNET_BLOCK_Type type, + uint32_t priority, uint32_t anonymity, + uint32_t replication, + struct GNUNET_TIME_Absolute expiration, char **msg) +{ + struct Plugin *plugin = cls; + GNUNET_HashCode vhash; + PGresult *ret; + uint32_t btype = htonl (type); + uint32_t bprio = htonl (priority); + uint32_t banon = htonl (anonymity); + uint32_t brepl = htonl (replication); + uint64_t bexpi = GNUNET_TIME_absolute_hton (expiration).abs_value__; + + const char *paramValues[] = { + (const char *) &brepl, + (const char *) &btype, + (const char *) &bprio, + (const char *) &banon, + (const char *) &bexpi, + (const char *) key, + (const char *) &vhash, + (const char *) data + }; + int paramLengths[] = { + sizeof (brepl), + sizeof (btype), + sizeof (bprio), + sizeof (banon), + sizeof (bexpi), + sizeof (GNUNET_HashCode), + sizeof (GNUNET_HashCode), + size + }; + const int paramFormats[] = { 1, 1, 1, 1, 1, 1, 1, 1 }; + + GNUNET_CRYPTO_hash (data, size, &vhash); + ret = + PQexecPrepared (plugin->dbh, "put", 8, paramValues, paramLengths, + paramFormats, 1); + if (GNUNET_OK != + check_result (plugin, ret, PGRES_COMMAND_OK, "PQexecPrepared", "put", + __LINE__)) + return GNUNET_SYSERR; + PQclear (ret); + plugin->env->duc (plugin->env->cls, size + GNUNET_DATASTORE_ENTRY_OVERHEAD); +#if DEBUG_POSTGRES + GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "datastore-postgres", + "Stored %u bytes in database\n", (unsigned int) size); +#endif + return GNUNET_OK; +} + + +/** + * Function invoked to process the result and call + * the processor. + * + * @param plugin global plugin data + * @param proc function to call the value (once only). + * @param proc_cls closure for proc + * @param res result from exec + * @param line line number for error messages + */ +static void +process_result (struct Plugin *plugin, PluginDatumProcessor proc, + void *proc_cls, PGresult * res, int line) +{ + int iret; + enum GNUNET_BLOCK_Type type; + uint32_t anonymity; + uint32_t priority; + uint32_t size; + unsigned int rowid; + struct GNUNET_TIME_Absolute expiration_time; + GNUNET_HashCode key; + + if (GNUNET_OK != + check_result (plugin, res, PGRES_TUPLES_OK, "PQexecPrepared", "select", + line)) + { +#if DEBUG_POSTGRES + GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "datastore-postgres", + "Ending iteration (postgres error)\n"); +#endif + proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0); + return; + } + + if (0 == PQntuples (res)) + { + /* no result */ +#if DEBUG_POSTGRES + GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "datastore-postgres", + "Ending iteration (no more results)\n"); +#endif + proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0); + PQclear (res); + return; + } + if ((1 != PQntuples (res)) || (7 != PQnfields (res)) || + (sizeof (uint32_t) != PQfsize (res, 0)) || + (sizeof (uint32_t) != PQfsize (res, 6))) + { + GNUNET_break (0); + proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0); + PQclear (res); + return; + } + rowid = ntohl (*(uint32_t *) PQgetvalue (res, 0, 6)); + if ((sizeof (uint32_t) != PQfsize (res, 0)) || + (sizeof (uint32_t) != PQfsize (res, 1)) || + (sizeof (uint32_t) != PQfsize (res, 2)) || + (sizeof (uint64_t) != PQfsize (res, 3)) || + (sizeof (GNUNET_HashCode) != PQgetlength (res, 0, 4))) + { + GNUNET_break (0); + PQclear (res); + delete_by_rowid (plugin, rowid); + proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0); + return; + } + + type = ntohl (*(uint32_t *) PQgetvalue (res, 0, 0)); + priority = ntohl (*(uint32_t *) PQgetvalue (res, 0, 1)); + anonymity = ntohl (*(uint32_t *) PQgetvalue (res, 0, 2)); + expiration_time.abs_value = + GNUNET_ntohll (*(uint64_t *) PQgetvalue (res, 0, 3)); + memcpy (&key, PQgetvalue (res, 0, 4), sizeof (GNUNET_HashCode)); + size = PQgetlength (res, 0, 5); +#if DEBUG_POSTGRES + GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "datastore-postgres", + "Found result of size %u bytes and type %u in database\n", + (unsigned int) size, (unsigned int) type); +#endif + iret = + proc (proc_cls, &key, size, PQgetvalue (res, 0, 5), + (enum GNUNET_BLOCK_Type) type, priority, anonymity, expiration_time, + rowid); + PQclear (res); + if (iret == GNUNET_NO) + { +#if DEBUG_POSTGRES + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Processor asked for item %u to be removed.\n", rowid); +#endif + if (GNUNET_OK == delete_by_rowid (plugin, rowid)) + { +#if DEBUG_POSTGRES + GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "datastore-postgres", + "Deleting %u bytes from database\n", + (unsigned int) size); +#endif + plugin->env->duc (plugin->env->cls, + -(size + GNUNET_DATASTORE_ENTRY_OVERHEAD)); +#if DEBUG_POSTGRES + GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "datastore-postgres", + "Deleted %u bytes from database\n", (unsigned int) size); +#endif + } + } +} + + +/** + * Iterate over the results for a particular key + * in the datastore. + * + * @param cls closure + * @param offset offset of the result (modulo num-results); + * specific ordering does not matter for the offset + * @param key maybe NULL (to match all entries) + * @param vhash hash of the value, maybe NULL (to + * match all values that have the right key). + * Note that for DBlocks there is no difference + * betwen key and vhash, but for other blocks + * there may be! + * @param type entries of which type are relevant? + * Use 0 for any type. + * @param proc function to call on the matching value; + * will be called once with a NULL if no value matches + * @param proc_cls closure for iter + */ +static void +postgres_plugin_get_key (void *cls, uint64_t offset, + const GNUNET_HashCode * key, + const GNUNET_HashCode * vhash, + enum GNUNET_BLOCK_Type type, PluginDatumProcessor proc, + void *proc_cls) +{ + struct Plugin *plugin = cls; + const int paramFormats[] = { 1, 1, 1, 1, 1 }; + int paramLengths[4]; + const char *paramValues[4]; + int nparams; + const char *pname; + PGresult *ret; + uint64_t total; + uint64_t blimit_off; + uint32_t btype; + + GNUNET_assert (key != NULL); + paramValues[0] = (const char *) key; + paramLengths[0] = sizeof (GNUNET_HashCode); + btype = htonl (type); + if (type != 0) + { + if (vhash != NULL) + { + paramValues[1] = (const char *) vhash; + paramLengths[1] = sizeof (GNUNET_HashCode); + paramValues[2] = (const char *) &btype; + paramLengths[2] = sizeof (btype); + paramValues[3] = (const char *) &blimit_off; + paramLengths[3] = sizeof (blimit_off); + nparams = 4; + pname = "getvt"; + ret = + PQexecParams (plugin->dbh, + "SELECT count(*) FROM gn090 WHERE hash=$1 AND vhash=$2 AND type=$3", + 3, NULL, paramValues, paramLengths, paramFormats, 1); + } + else + { + paramValues[1] = (const char *) &btype; + paramLengths[1] = sizeof (btype); + paramValues[2] = (const char *) &blimit_off; + paramLengths[2] = sizeof (blimit_off); + nparams = 3; + pname = "gett"; + ret = + PQexecParams (plugin->dbh, + "SELECT count(*) FROM gn090 WHERE hash=$1 AND type=$2", + 2, NULL, paramValues, paramLengths, paramFormats, 1); + } + } + else + { + if (vhash != NULL) + { + paramValues[1] = (const char *) vhash; + paramLengths[1] = sizeof (GNUNET_HashCode); + paramValues[2] = (const char *) &blimit_off; + paramLengths[2] = sizeof (blimit_off); + nparams = 3; + pname = "getv"; + ret = + PQexecParams (plugin->dbh, + "SELECT count(*) FROM gn090 WHERE hash=$1 AND vhash=$2", + 2, NULL, paramValues, paramLengths, paramFormats, 1); + } + else + { + paramValues[1] = (const char *) &blimit_off; + paramLengths[1] = sizeof (blimit_off); + nparams = 2; + pname = "get"; + ret = + PQexecParams (plugin->dbh, "SELECT count(*) FROM gn090 WHERE hash=$1", + 1, NULL, paramValues, paramLengths, paramFormats, 1); + } + } + if (GNUNET_OK != + check_result (plugin, ret, PGRES_TUPLES_OK, "PQexecParams", pname, + __LINE__)) + { + proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0); + return; + } + if ((PQntuples (ret) != 1) || (PQnfields (ret) != 1) || + (PQgetlength (ret, 0, 0) != sizeof (unsigned long long))) + { + GNUNET_break (0); + PQclear (ret); + proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0); + return; + } + total = GNUNET_ntohll (*(const unsigned long long *) PQgetvalue (ret, 0, 0)); + PQclear (ret); + if (total == 0) + { + proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0); + return; + } + blimit_off = GNUNET_htonll (offset % total); + ret = + PQexecPrepared (plugin->dbh, pname, nparams, paramValues, paramLengths, + paramFormats, 1); + process_result (plugin, proc, proc_cls, ret, __LINE__); +} + + +/** + * Select a subset of the items in the datastore and call + * the given iterator for each of them. + * + * @param cls our "struct Plugin*" + * @param offset offset of the result (modulo num-results); + * specific ordering does not matter for the offset + * @param type entries of which type should be considered? + * Use 0 for any type. + * @param proc function to call on the matching value; + * will be called with a NULL if no value matches + * @param proc_cls closure for proc + */ +static void +postgres_plugin_get_zero_anonymity (void *cls, uint64_t offset, + enum GNUNET_BLOCK_Type type, + PluginDatumProcessor proc, void *proc_cls) +{ + struct Plugin *plugin = cls; + uint32_t btype; + uint64_t boff; + const int paramFormats[] = { 1, 1 }; + int paramLengths[] = { sizeof (btype), sizeof (boff) }; + const char *paramValues[] = { (const char *) &btype, (const char *) &boff }; + PGresult *ret; + + btype = htonl ((uint32_t) type); + boff = GNUNET_htonll (offset); + ret = + PQexecPrepared (plugin->dbh, "select_non_anonymous", 2, paramValues, + paramLengths, paramFormats, 1); + process_result (plugin, proc, proc_cls, ret, __LINE__); +} + + +/** + * Context for 'repl_iter' function. + */ +struct ReplCtx +{ + + /** + * Plugin handle. + */ + struct Plugin *plugin; + + /** + * Function to call for the result (or the NULL). + */ + PluginDatumProcessor proc; + + /** + * Closure for proc. + */ + void *proc_cls; +}; + + +/** + * Wrapper for the iterator for 'sqlite_plugin_replication_get'. + * Decrements the replication counter and calls the original + * iterator. + * + * @param cls closure + * @param key key for the content + * @param size number of bytes in data + * @param data content stored + * @param type type of the content + * @param priority priority of the content + * @param anonymity anonymity-level for the content + * @param expiration expiration time for the content + * @param uid unique identifier for the datum; + * maybe 0 if no unique identifier is available + * + * @return GNUNET_SYSERR to abort the iteration, GNUNET_OK to continue + * (continue on call to "next", of course), + * GNUNET_NO to delete the item and continue (if supported) + */ +static int +repl_proc (void *cls, const GNUNET_HashCode * key, uint32_t size, + const void *data, enum GNUNET_BLOCK_Type type, uint32_t priority, + uint32_t anonymity, struct GNUNET_TIME_Absolute expiration, + uint64_t uid) +{ + struct ReplCtx *rc = cls; + struct Plugin *plugin = rc->plugin; + int ret; + PGresult *qret; + uint32_t boid; + + ret = + rc->proc (rc->proc_cls, key, size, data, type, priority, anonymity, + expiration, uid); + if (NULL != key) + { + boid = htonl ((uint32_t) uid); + const char *paramValues[] = { + (const char *) &boid, + }; + int paramLengths[] = { + sizeof (boid), + }; + const int paramFormats[] = { 1 }; + qret = + PQexecPrepared (plugin->dbh, "decrepl", 1, paramValues, paramLengths, + paramFormats, 1); + if (GNUNET_OK != + check_result (plugin, qret, PGRES_COMMAND_OK, "PQexecPrepared", + "decrepl", __LINE__)) + return GNUNET_SYSERR; + PQclear (qret); + } + return ret; +} + + +/** + * Get a random item for replication. Returns a single, not expired, random item + * from those with the highest replication counters. The item's + * replication counter is decremented by one IF it was positive before. + * Call 'proc' with all values ZERO or NULL if the datastore is empty. + * + * @param cls closure + * @param proc function to call the value (once only). + * @param proc_cls closure for proc + */ +static void +postgres_plugin_get_replication (void *cls, PluginDatumProcessor proc, + void *proc_cls) +{ + struct Plugin *plugin = cls; + struct ReplCtx rc; + PGresult *ret; + + rc.plugin = plugin; + rc.proc = proc; + rc.proc_cls = proc_cls; + ret = + PQexecPrepared (plugin->dbh, "select_replication_order", 0, NULL, NULL, + NULL, 1); + process_result (plugin, &repl_proc, &rc, ret, __LINE__); +} + + +/** + * Get a random item for expiration. + * Call 'proc' with all values ZERO or NULL if the datastore is empty. + * + * @param cls closure + * @param proc function to call the value (once only). + * @param proc_cls closure for proc + */ +static void +postgres_plugin_get_expiration (void *cls, PluginDatumProcessor proc, + void *proc_cls) +{ + struct Plugin *plugin = cls; + uint64_t btime; + const int paramFormats[] = { 1 }; + int paramLengths[] = { sizeof (btime) }; + const char *paramValues[] = { (const char *) &btime }; + PGresult *ret; + + btime = GNUNET_htonll (GNUNET_TIME_absolute_get ().abs_value); + ret = + PQexecPrepared (plugin->dbh, "select_expiration_order", 1, paramValues, + paramLengths, paramFormats, 1); + process_result (plugin, proc, proc_cls, ret, __LINE__); +} + + +/** + * Update the priority for a particular key in the datastore. If + * the expiration time in value is different than the time found in + * the datastore, the higher value should be kept. For the + * anonymity level, the lower value is to be used. The specified + * priority should be added to the existing priority, ignoring the + * priority in value. + * + * Note that it is possible for multiple values to match this put. + * In that case, all of the respective values are updated. + * + * @param cls our "struct Plugin*" + * @param uid unique identifier of the datum + * @param delta by how much should the priority + * change? If priority + delta < 0 the + * priority should be set to 0 (never go + * negative). + * @param expire new expiration time should be the + * MAX of any existing expiration time and + * this value + * @param msg set to error message + * @return GNUNET_OK on success + */ +static int +postgres_plugin_update (void *cls, uint64_t uid, int delta, + struct GNUNET_TIME_Absolute expire, char **msg) +{ + struct Plugin *plugin = cls; + PGresult *ret; + int32_t bdelta = (int32_t) htonl ((uint32_t) delta); + uint32_t boid = htonl ((uint32_t) uid); + uint64_t bexpire = GNUNET_TIME_absolute_hton (expire).abs_value__; + + const char *paramValues[] = { + (const char *) &bdelta, + (const char *) &bexpire, + (const char *) &boid, + }; + int paramLengths[] = { + sizeof (bdelta), + sizeof (bexpire), + sizeof (boid), + }; + const int paramFormats[] = { 1, 1, 1 }; + + ret = + PQexecPrepared (plugin->dbh, "update", 3, paramValues, paramLengths, + paramFormats, 1); + if (GNUNET_OK != + check_result (plugin, ret, PGRES_COMMAND_OK, "PQexecPrepared", "update", + __LINE__)) + return GNUNET_SYSERR; + PQclear (ret); + return GNUNET_OK; +} + + + +/** + * Get all of the keys in the datastore. + * + * @param cls closure + * @param proc function to call on each key + * @param proc_cls closure for proc + */ +static void +postgres_plugin_get_keys (void *cls, + PluginKeyProcessor proc, + void *proc_cls) +{ + struct Plugin *plugin = cls; + int ret; + int i; + GNUNET_HashCode key; + PGresult * res; + + res = PQexecPrepared (plugin->dbh, "get_keys", 0, NULL, NULL, NULL, 1); + ret = PQntuples (res); + for (i=0;i<ret;i++) + { + if (sizeof (GNUNET_HashCode) != PQgetlength (res, i, 0)) + { + memcpy (&key, PQgetvalue (res, i, 0), sizeof (GNUNET_HashCode)); + proc (proc_cls, &key, 1); + } + } + PQclear (res); +} + + + +/** + * Drop database. + */ +static void +postgres_plugin_drop (void *cls) +{ + struct Plugin *plugin = cls; + + pq_exec (plugin, "DROP TABLE gn090", __LINE__); +} + + +/** + * Entry point for the plugin. + * + * @param cls the "struct GNUNET_DATASTORE_PluginEnvironment*" + * @return our "struct Plugin*" + */ +void * +libgnunet_plugin_datastore_postgres_init (void *cls) +{ + struct GNUNET_DATASTORE_PluginEnvironment *env = cls; + struct GNUNET_DATASTORE_PluginFunctions *api; + struct Plugin *plugin; + + plugin = GNUNET_malloc (sizeof (struct Plugin)); + plugin->env = env; + if (GNUNET_OK != init_connection (plugin)) + { + GNUNET_free (plugin); + return NULL; + } + api = GNUNET_malloc (sizeof (struct GNUNET_DATASTORE_PluginFunctions)); + api->cls = plugin; + api->estimate_size = &postgres_plugin_estimate_size; + api->put = &postgres_plugin_put; + api->update = &postgres_plugin_update; + api->get_key = &postgres_plugin_get_key; + api->get_replication = &postgres_plugin_get_replication; + api->get_expiration = &postgres_plugin_get_expiration; + api->get_zero_anonymity = &postgres_plugin_get_zero_anonymity; + api->get_keys = &postgres_plugin_get_keys; + api->drop = &postgres_plugin_drop; + GNUNET_log_from (GNUNET_ERROR_TYPE_INFO, "datastore-postgres", + _("Postgres database running\n")); + return api; +} + + +/** + * Exit point from the plugin. + * @param cls our "struct Plugin*" + * @return always NULL + */ +void * +libgnunet_plugin_datastore_postgres_done (void *cls) +{ + struct GNUNET_DATASTORE_PluginFunctions *api = cls; + struct Plugin *plugin = api->cls; + + PQfinish (plugin->dbh); + GNUNET_free (plugin); + GNUNET_free (api); + return NULL; +} + +/* end of plugin_datastore_postgres.c */ diff --git a/src/datastore/plugin_datastore_sqlite.c b/src/datastore/plugin_datastore_sqlite.c new file mode 100644 index 0000000..cd5ae39 --- /dev/null +++ b/src/datastore/plugin_datastore_sqlite.c @@ -0,0 +1,1264 @@ + /* + * This file is part of GNUnet + * (C) 2009, 2011 Christian Grothoff (and other contributing authors) + * + * GNUnet is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published + * by the Free Software Foundation; either version 3, or (at your + * option) any later version. + * + * GNUnet is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with GNUnet; see the file COPYING. If not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + */ + +/** + * @file datastore/plugin_datastore_sqlite.c + * @brief sqlite-based datastore backend + * @author Christian Grothoff + */ + +#include "platform.h" +#include "gnunet_datastore_plugin.h" +#include <sqlite3.h> + +/** + * Enable or disable logging debug messages. + */ +#define DEBUG_SQLITE GNUNET_EXTRA_LOGGING + +/** + * We allocate items on the stack at times. To prevent a stack + * overflow, we impose a limit on the maximum size for the data per + * item. 64k should be enough. + */ +#define MAX_ITEM_SIZE 65536 + +/** + * After how many ms "busy" should a DB operation fail for good? + * A low value makes sure that we are more responsive to requests + * (especially PUTs). A high value guarantees a higher success + * rate (SELECTs in iterate can take several seconds despite LIMIT=1). + * + * The default value of 250ms should ensure that users do not experience + * huge latencies while at the same time allowing operations to succeed + * with reasonable probability. + */ +#define BUSY_TIMEOUT_MS 250 + + +/** + * Log an error message at log-level 'level' that indicates + * a failure of the command 'cmd' on file 'filename' + * with the message given by strerror(errno). + */ +#define LOG_SQLITE(db, msg, level, cmd) do { GNUNET_log_from (level, "sqlite", _("`%s' failed at %s:%d with error: %s\n"), cmd, __FILE__, __LINE__, sqlite3_errmsg(db->dbh)); if (msg != NULL) GNUNET_asprintf(msg, _("`%s' failed at %s:%u with error: %s"), cmd, __FILE__, __LINE__, sqlite3_errmsg(db->dbh)); } while(0) + + + +/** + * Context for all functions in this plugin. + */ +struct Plugin +{ + /** + * Our execution environment. + */ + struct GNUNET_DATASTORE_PluginEnvironment *env; + + /** + * Database filename. + */ + char *fn; + + /** + * Native SQLite database handle. + */ + sqlite3 *dbh; + + /** + * Precompiled SQL for deletion. + */ + sqlite3_stmt *delRow; + + /** + * Precompiled SQL for update. + */ + sqlite3_stmt *updPrio; + + /** + * Get maximum repl value in database. + */ + sqlite3_stmt *maxRepl; + + /** + * Precompiled SQL for replication decrement. + */ + sqlite3_stmt *updRepl; + + /** + * Precompiled SQL for replication selection. + */ + sqlite3_stmt *selRepl; + + /** + * Precompiled SQL for expiration selection. + */ + sqlite3_stmt *selExpi; + + /** + * Precompiled SQL for expiration selection. + */ + sqlite3_stmt *selZeroAnon; + + /** + * Precompiled SQL for insertion. + */ + sqlite3_stmt *insertContent; + + /** + * Should the database be dropped on shutdown? + */ + int drop_on_shutdown; + +}; + + +/** + * @brief Prepare a SQL statement + * + * @param dbh handle to the database + * @param zSql SQL statement, UTF-8 encoded + * @param ppStmt set to the prepared statement + * @return 0 on success + */ +static int +sq_prepare (sqlite3 * dbh, const char *zSql, sqlite3_stmt ** ppStmt) +{ + char *dummy; + int result; + + result = + sqlite3_prepare_v2 (dbh, zSql, strlen (zSql), ppStmt, + (const char **) &dummy); +#if DEBUG_SQLITE && 0 + GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite", + "Prepared `%s' / %p: %d\n", zSql, *ppStmt, result); +#endif + return result; +} + + +/** + * Create our database indices. + * + * @param dbh handle to the database + */ +static void +create_indices (sqlite3 * dbh) +{ + /* create indices */ + if ((SQLITE_OK != + sqlite3_exec (dbh, "CREATE INDEX IF NOT EXISTS idx_hash ON gn090 (hash)", + NULL, NULL, NULL)) || + (SQLITE_OK != + sqlite3_exec (dbh, + "CREATE INDEX IF NOT EXISTS idx_hash_vhash ON gn090 (hash,vhash)", + NULL, NULL, NULL)) || + (SQLITE_OK != + sqlite3_exec (dbh, + "CREATE INDEX IF NOT EXISTS idx_expire_repl ON gn090 (expire ASC,repl DESC)", + NULL, NULL, NULL)) || + (SQLITE_OK != + sqlite3_exec (dbh, + "CREATE INDEX IF NOT EXISTS idx_comb ON gn090 (anonLevel ASC,expire ASC,prio,type,hash)", + NULL, NULL, NULL)) || + (SQLITE_OK != + sqlite3_exec (dbh, + "CREATE INDEX IF NOT EXISTS idx_anon_type_hash ON gn090 (anonLevel ASC,type,hash)", + NULL, NULL, NULL)) || + (SQLITE_OK != + sqlite3_exec (dbh, + "CREATE INDEX IF NOT EXISTS idx_expire ON gn090 (expire ASC)", + NULL, NULL, NULL)) || + (SQLITE_OK != + sqlite3_exec (dbh, + "CREATE INDEX IF NOT EXISTS idx_repl_rvalue ON gn090 (repl,rvalue)", + NULL, NULL, NULL)) || + (SQLITE_OK != + sqlite3_exec (dbh, + "CREATE INDEX IF NOT EXISTS idx_repl ON gn090 (repl DESC)", + NULL, NULL, NULL))) + GNUNET_log_from (GNUNET_ERROR_TYPE_ERROR, "sqlite", + "Failed to create indices: %s\n", sqlite3_errmsg (dbh)); +} + + +#if 0 +#define CHECK(a) GNUNET_break(a) +#define ENULL NULL +#else +#define ENULL &e +#define ENULL_DEFINED 1 +#define CHECK(a) if (! a) { GNUNET_log(GNUNET_ERROR_TYPE_ERROR, "%s\n", e); sqlite3_free(e); } +#endif + + +/** + * Initialize the database connections and associated + * data structures (create tables and indices + * as needed as well). + * + * @param cfg our configuration + * @param plugin the plugin context (state for this module) + * @return GNUNET_OK on success + */ +static int +database_setup (const struct GNUNET_CONFIGURATION_Handle *cfg, + struct Plugin *plugin) +{ + sqlite3_stmt *stmt; + char *afsdir; + +#if ENULL_DEFINED + char *e; +#endif + + if (GNUNET_OK != + GNUNET_CONFIGURATION_get_value_filename (cfg, "datastore-sqlite", + "FILENAME", &afsdir)) + { + GNUNET_log_from (GNUNET_ERROR_TYPE_ERROR, "sqlite", + _ + ("Option `%s' in section `%s' missing in configuration!\n"), + "FILENAME", "datastore-sqlite"); + return GNUNET_SYSERR; + } + if (GNUNET_OK != GNUNET_DISK_file_test (afsdir)) + { + if (GNUNET_OK != GNUNET_DISK_directory_create_for_file (afsdir)) + { + GNUNET_break (0); + GNUNET_free (afsdir); + return GNUNET_SYSERR; + } + /* database is new or got deleted, reset payload to zero! */ + plugin->env->duc (plugin->env->cls, 0); + } +#ifdef ENABLE_NLS + plugin->fn = + GNUNET_STRINGS_to_utf8 (afsdir, strlen (afsdir), nl_langinfo (CODESET)); +#else + plugin->fn = GNUNET_STRINGS_to_utf8 (afsdir, strlen (afsdir), "UTF-8"); /* good luck */ +#endif + GNUNET_free (afsdir); + + /* Open database and precompile statements */ + if (sqlite3_open (plugin->fn, &plugin->dbh) != SQLITE_OK) + { + GNUNET_log_from (GNUNET_ERROR_TYPE_ERROR, "sqlite", + _("Unable to initialize SQLite: %s.\n"), + sqlite3_errmsg (plugin->dbh)); + return GNUNET_SYSERR; + } + CHECK (SQLITE_OK == + sqlite3_exec (plugin->dbh, "PRAGMA temp_store=MEMORY", NULL, NULL, + ENULL)); + CHECK (SQLITE_OK == + sqlite3_exec (plugin->dbh, "PRAGMA synchronous=OFF", NULL, NULL, + ENULL)); + CHECK (SQLITE_OK == + sqlite3_exec (plugin->dbh, "PRAGMA legacy_file_format=OFF", NULL, NULL, + ENULL)); + CHECK (SQLITE_OK == + sqlite3_exec (plugin->dbh, "PRAGMA auto_vacuum=INCREMENTAL", NULL, + NULL, ENULL)); + CHECK (SQLITE_OK == + sqlite3_exec (plugin->dbh, "PRAGMA locking_mode=EXCLUSIVE", NULL, NULL, + ENULL)); + CHECK (SQLITE_OK == + sqlite3_exec (plugin->dbh, "PRAGMA count_changes=OFF", NULL, NULL, + ENULL)); + CHECK (SQLITE_OK == + sqlite3_exec (plugin->dbh, "PRAGMA page_size=4092", NULL, NULL, + ENULL)); + + CHECK (SQLITE_OK == sqlite3_busy_timeout (plugin->dbh, BUSY_TIMEOUT_MS)); + + + /* We have to do it here, because otherwise precompiling SQL might fail */ + CHECK (SQLITE_OK == + sq_prepare (plugin->dbh, + "SELECT 1 FROM sqlite_master WHERE tbl_name = 'gn090'", + &stmt)); + if ((sqlite3_step (stmt) == SQLITE_DONE) && + (sqlite3_exec + (plugin->dbh, + "CREATE TABLE gn090 (" " repl INT4 NOT NULL DEFAULT 0," + " type INT4 NOT NULL DEFAULT 0," " prio INT4 NOT NULL DEFAULT 0," + " anonLevel INT4 NOT NULL DEFAULT 0," + " expire INT8 NOT NULL DEFAULT 0," " rvalue INT8 NOT NULL," + " hash TEXT NOT NULL DEFAULT ''," " vhash TEXT NOT NULL DEFAULT ''," + " value BLOB NOT NULL DEFAULT '')", NULL, NULL, NULL) != SQLITE_OK)) + { + LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR, "sqlite3_exec"); + sqlite3_finalize (stmt); + return GNUNET_SYSERR; + } + sqlite3_finalize (stmt); + create_indices (plugin->dbh); + + if ((sq_prepare + (plugin->dbh, + "UPDATE gn090 " + "SET prio = prio + ?, expire = MAX(expire,?) WHERE _ROWID_ = ?", + &plugin->updPrio) != SQLITE_OK) || + (sq_prepare + (plugin->dbh, + "UPDATE gn090 " "SET repl = MAX (0, repl - 1) WHERE _ROWID_ = ?", + &plugin->updRepl) != SQLITE_OK) || + (sq_prepare + (plugin->dbh, + "SELECT type,prio,anonLevel,expire,hash,value,_ROWID_ " "FROM gn090 " +#if SQLITE_VERSION_NUMBER >= 3007000 + "INDEXED BY idx_repl_rvalue " +#endif + "WHERE repl=?2 AND " " (rvalue>=?1 OR " + " NOT EXISTS (SELECT 1 FROM gn090 " +#if SQLITE_VERSION_NUMBER >= 3007000 + "INDEXED BY idx_repl_rvalue " +#endif + "WHERE repl=?2 AND rvalue>=?1 LIMIT 1) ) " + "ORDER BY rvalue ASC LIMIT 1", &plugin->selRepl) != SQLITE_OK) || + (sq_prepare (plugin->dbh, "SELECT MAX(repl) FROM gn090" +#if SQLITE_VERSION_NUMBER >= 3007000 + " INDEXED BY idx_repl_rvalue" +#endif + "", &plugin->maxRepl) != SQLITE_OK) || + (sq_prepare + (plugin->dbh, + "SELECT type,prio,anonLevel,expire,hash,value,_ROWID_ " "FROM gn090 " +#if SQLITE_VERSION_NUMBER >= 3007000 + "INDEXED BY idx_expire " +#endif + "WHERE NOT EXISTS (SELECT 1 FROM gn090 WHERE expire < ?1 LIMIT 1) OR (expire < ?1) " + "ORDER BY expire ASC LIMIT 1", &plugin->selExpi) != SQLITE_OK) || + (sq_prepare + (plugin->dbh, + "SELECT type,prio,anonLevel,expire,hash,value,_ROWID_ " "FROM gn090 " +#if SQLITE_VERSION_NUMBER >= 3007000 + "INDEXED BY idx_anon_type_hash " +#endif + "WHERE (anonLevel = 0 AND type=?1) " + "ORDER BY hash DESC LIMIT 1 OFFSET ?2", + &plugin->selZeroAnon) != SQLITE_OK) || + (sq_prepare + (plugin->dbh, + "INSERT INTO gn090 (repl, type, prio, anonLevel, expire, rvalue, hash, vhash, value) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + &plugin->insertContent) != SQLITE_OK) || + (sq_prepare + (plugin->dbh, "DELETE FROM gn090 WHERE _ROWID_ = ?", + &plugin->delRow) != SQLITE_OK)) + { + LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR, "precompiling"); + return GNUNET_SYSERR; + } + + return GNUNET_OK; +} + + +/** + * Shutdown database connection and associate data + * structures. + * @param plugin the plugin context (state for this module) + */ +static void +database_shutdown (struct Plugin *plugin) +{ + int result; + +#if SQLITE_VERSION_NUMBER >= 3007000 + sqlite3_stmt *stmt; +#endif + + if (plugin->delRow != NULL) + sqlite3_finalize (plugin->delRow); + if (plugin->updPrio != NULL) + sqlite3_finalize (plugin->updPrio); + if (plugin->updRepl != NULL) + sqlite3_finalize (plugin->updRepl); + if (plugin->selRepl != NULL) + sqlite3_finalize (plugin->selRepl); + if (plugin->maxRepl != NULL) + sqlite3_finalize (plugin->maxRepl); + if (plugin->selExpi != NULL) + sqlite3_finalize (plugin->selExpi); + if (plugin->selZeroAnon != NULL) + sqlite3_finalize (plugin->selZeroAnon); + if (plugin->insertContent != NULL) + sqlite3_finalize (plugin->insertContent); + result = sqlite3_close (plugin->dbh); +#if SQLITE_VERSION_NUMBER >= 3007000 + if (result == SQLITE_BUSY) + { + GNUNET_log_from (GNUNET_ERROR_TYPE_WARNING, "sqlite", + _ + ("Tried to close sqlite without finalizing all prepared statements.\n")); + stmt = sqlite3_next_stmt (plugin->dbh, NULL); + while (stmt != NULL) + { +#if DEBUG_SQLITE + GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite", + "Closing statement %p\n", stmt); +#endif + result = sqlite3_finalize (stmt); + if (result != SQLITE_OK) + GNUNET_log_from (GNUNET_ERROR_TYPE_WARNING, "sqlite", + "Failed to close statement %p: %d\n", stmt, result); + stmt = sqlite3_next_stmt (plugin->dbh, NULL); + } + result = sqlite3_close (plugin->dbh); + } +#endif + if (SQLITE_OK != result) + LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR, "sqlite3_close"); + + GNUNET_free_non_null (plugin->fn); +} + + +/** + * Delete the database entry with the given + * row identifier. + * + * @param plugin the plugin context (state for this module) + * @param rid the ID of the row to delete + */ +static int +delete_by_rowid (struct Plugin *plugin, unsigned long long rid) +{ + if (SQLITE_OK != sqlite3_bind_int64 (plugin->delRow, 1, rid)) + { + LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + "sqlite3_bind_XXXX"); + if (SQLITE_OK != sqlite3_reset (plugin->delRow)) + LOG_SQLITE (plugin, NULL, + GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + "sqlite3_reset"); + return GNUNET_SYSERR; + } + if (SQLITE_DONE != sqlite3_step (plugin->delRow)) + { + LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + "sqlite3_step"); + if (SQLITE_OK != sqlite3_reset (plugin->delRow)) + LOG_SQLITE (plugin, NULL, + GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + "sqlite3_reset"); + return GNUNET_SYSERR; + } + if (SQLITE_OK != sqlite3_reset (plugin->delRow)) + LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + "sqlite3_reset"); + return GNUNET_OK; +} + + +/** + * Store an item in the datastore. + * + * @param cls closure + * @param key key for the item + * @param size number of bytes in data + * @param data content stored + * @param type type of the content + * @param priority priority of the content + * @param anonymity anonymity-level for the content + * @param replication replication-level for the content + * @param expiration expiration time for the content + * @param msg set to an error message + * @return GNUNET_OK on success + */ +static int +sqlite_plugin_put (void *cls, const GNUNET_HashCode * key, uint32_t size, + const void *data, enum GNUNET_BLOCK_Type type, + uint32_t priority, uint32_t anonymity, uint32_t replication, + struct GNUNET_TIME_Absolute expiration, char **msg) +{ + struct Plugin *plugin = cls; + int n; + int ret; + sqlite3_stmt *stmt; + GNUNET_HashCode vhash; + uint64_t rvalue; + + if (size > MAX_ITEM_SIZE) + return GNUNET_SYSERR; +#if DEBUG_SQLITE + GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite", + "Storing in database block with type %u/key `%s'/priority %u/expiration in %llu ms (%lld).\n", + type, GNUNET_h2s (key), priority, + (unsigned long long) + GNUNET_TIME_absolute_get_remaining (expiration).rel_value, + (long long) expiration.abs_value); +#endif + GNUNET_CRYPTO_hash (data, size, &vhash); + stmt = plugin->insertContent; + rvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK, UINT64_MAX); + if ((SQLITE_OK != sqlite3_bind_int (stmt, 1, replication)) || + (SQLITE_OK != sqlite3_bind_int (stmt, 2, type)) || + (SQLITE_OK != sqlite3_bind_int (stmt, 3, priority)) || + (SQLITE_OK != sqlite3_bind_int (stmt, 4, anonymity)) || + (SQLITE_OK != sqlite3_bind_int64 (stmt, 5, expiration.abs_value)) || + (SQLITE_OK != sqlite3_bind_int64 (stmt, 6, rvalue)) || + (SQLITE_OK != + sqlite3_bind_blob (stmt, 7, key, sizeof (GNUNET_HashCode), + SQLITE_TRANSIENT)) || + (SQLITE_OK != + sqlite3_bind_blob (stmt, 8, &vhash, sizeof (GNUNET_HashCode), + SQLITE_TRANSIENT)) || + (SQLITE_OK != sqlite3_bind_blob (stmt, 9, data, size, SQLITE_TRANSIENT))) + { + LOG_SQLITE (plugin, msg, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + "sqlite3_bind_XXXX"); + if (SQLITE_OK != sqlite3_reset (stmt)) + LOG_SQLITE (plugin, NULL, + GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + "sqlite3_reset"); + return GNUNET_SYSERR; + } + n = sqlite3_step (stmt); + switch (n) + { + case SQLITE_DONE: + plugin->env->duc (plugin->env->cls, size + GNUNET_DATASTORE_ENTRY_OVERHEAD); +#if DEBUG_SQLITE + GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite", + "Stored new entry (%u bytes)\n", + size + GNUNET_DATASTORE_ENTRY_OVERHEAD); +#endif + ret = GNUNET_OK; + break; + case SQLITE_BUSY: + GNUNET_break (0); + LOG_SQLITE (plugin, msg, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + "sqlite3_step"); + ret = GNUNET_SYSERR; + break; + default: + LOG_SQLITE (plugin, msg, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + "sqlite3_step"); + if (SQLITE_OK != sqlite3_reset (stmt)) + LOG_SQLITE (plugin, NULL, + GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + "sqlite3_reset"); + database_shutdown (plugin); + database_setup (plugin->env->cfg, plugin); + return GNUNET_SYSERR; + } + if (SQLITE_OK != sqlite3_reset (stmt)) + LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + "sqlite3_reset"); + return ret; +} + + +/** + * Update the priority for a particular key in the datastore. If + * the expiration time in value is different than the time found in + * the datastore, the higher value should be kept. For the + * anonymity level, the lower value is to be used. The specified + * priority should be added to the existing priority, ignoring the + * priority in value. + * + * Note that it is possible for multiple values to match this put. + * In that case, all of the respective values are updated. + * + * @param cls the plugin context (state for this module) + * @param uid unique identifier of the datum + * @param delta by how much should the priority + * change? If priority + delta < 0 the + * priority should be set to 0 (never go + * negative). + * @param expire new expiration time should be the + * MAX of any existing expiration time and + * this value + * @param msg set to an error message + * @return GNUNET_OK on success + */ +static int +sqlite_plugin_update (void *cls, uint64_t uid, int delta, + struct GNUNET_TIME_Absolute expire, char **msg) +{ + struct Plugin *plugin = cls; + int n; + + if ((SQLITE_OK != sqlite3_bind_int (plugin->updPrio, 1, delta)) || + (SQLITE_OK != sqlite3_bind_int64 (plugin->updPrio, 2, expire.abs_value)) + || (SQLITE_OK != sqlite3_bind_int64 (plugin->updPrio, 3, uid))) + { + LOG_SQLITE (plugin, msg, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + "sqlite3_bind_XXXX"); + if (SQLITE_OK != sqlite3_reset (plugin->updPrio)) + LOG_SQLITE (plugin, NULL, + GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + "sqlite3_reset"); + return GNUNET_SYSERR; + + } + n = sqlite3_step (plugin->updPrio); + if (SQLITE_OK != sqlite3_reset (plugin->updPrio)) + LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + "sqlite3_reset"); + switch (n) + { + case SQLITE_DONE: +#if DEBUG_SQLITE + GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite", "Block updated\n"); +#endif + return GNUNET_OK; + case SQLITE_BUSY: + LOG_SQLITE (plugin, msg, GNUNET_ERROR_TYPE_WARNING | GNUNET_ERROR_TYPE_BULK, + "sqlite3_step"); + return GNUNET_NO; + default: + LOG_SQLITE (plugin, msg, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + "sqlite3_step"); + return GNUNET_SYSERR; + } +} + + +/** + * Execute statement that gets a row and call the callback + * with the result. Resets the statement afterwards. + * + * @param plugin the plugin + * @param stmt the statement + * @param proc processor to call + * @param proc_cls closure for 'proc' + */ +static void +execute_get (struct Plugin *plugin, sqlite3_stmt * stmt, + PluginDatumProcessor proc, void *proc_cls) +{ + int n; + struct GNUNET_TIME_Absolute expiration; + unsigned long long rowid; + unsigned int size; + int ret; + + n = sqlite3_step (stmt); + switch (n) + { + case SQLITE_ROW: + size = sqlite3_column_bytes (stmt, 5); + rowid = sqlite3_column_int64 (stmt, 6); + if (sqlite3_column_bytes (stmt, 4) != sizeof (GNUNET_HashCode)) + { + GNUNET_log_from (GNUNET_ERROR_TYPE_WARNING, "sqlite", + _ + ("Invalid data in database. Trying to fix (by deletion).\n")); + if (SQLITE_OK != sqlite3_reset (stmt)) + LOG_SQLITE (plugin, NULL, + GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + "sqlite3_reset"); + if (GNUNET_OK == delete_by_rowid (plugin, rowid)) + plugin->env->duc (plugin->env->cls, + -(size + GNUNET_DATASTORE_ENTRY_OVERHEAD)); + break; + } + expiration.abs_value = sqlite3_column_int64 (stmt, 3); +#if DEBUG_SQLITE + GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite", + "Found reply in database with expiration %llu\n", + (unsigned long long) expiration.abs_value); +#endif + ret = proc (proc_cls, sqlite3_column_blob (stmt, 4) /* key */ , + size, sqlite3_column_blob (stmt, 5) /* data */ , + sqlite3_column_int (stmt, 0) /* type */ , + sqlite3_column_int (stmt, 1) /* priority */ , + sqlite3_column_int (stmt, 2) /* anonymity */ , + expiration, rowid); + if (SQLITE_OK != sqlite3_reset (stmt)) + LOG_SQLITE (plugin, NULL, + GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + "sqlite3_reset"); + if ((GNUNET_NO == ret) && (GNUNET_OK == delete_by_rowid (plugin, rowid))) + plugin->env->duc (plugin->env->cls, + -(size + GNUNET_DATASTORE_ENTRY_OVERHEAD)); + return; + case SQLITE_DONE: + /* database must be empty */ + if (SQLITE_OK != sqlite3_reset (stmt)) + LOG_SQLITE (plugin, NULL, + GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + "sqlite3_reset"); + break; + case SQLITE_BUSY: + case SQLITE_ERROR: + case SQLITE_MISUSE: + default: + LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + "sqlite3_step"); + if (SQLITE_OK != sqlite3_reset (stmt)) + LOG_SQLITE (plugin, NULL, + GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + "sqlite3_reset"); + GNUNET_break (0); + database_shutdown (plugin); + database_setup (plugin->env->cfg, plugin); + break; + } + if (SQLITE_OK != sqlite3_reset (stmt)) + LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + "sqlite3_reset"); + proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0); +} + + + +/** + * Select a subset of the items in the datastore and call + * the given processor for the item. + * + * @param cls our plugin context + * @param offset offset of the result (modulo num-results); + * specific ordering does not matter for the offset + * @param type entries of which type should be considered? + * Use 0 for any type. + * @param proc function to call on each matching value; + * will be called once with a NULL value at the end + * @param proc_cls closure for proc + */ +static void +sqlite_plugin_get_zero_anonymity (void *cls, uint64_t offset, + enum GNUNET_BLOCK_Type type, + PluginDatumProcessor proc, void *proc_cls) +{ + struct Plugin *plugin = cls; + sqlite3_stmt *stmt; + + GNUNET_assert (type != GNUNET_BLOCK_TYPE_ANY); + stmt = plugin->selZeroAnon; + if ((SQLITE_OK != sqlite3_bind_int (stmt, 1, type)) || + (SQLITE_OK != sqlite3_bind_int64 (stmt, 2, offset))) + { + LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + "sqlite3_bind_XXXX"); + if (SQLITE_OK != sqlite3_reset (stmt)) + LOG_SQLITE (plugin, NULL, + GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + "sqlite3_reset"); + proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0); + return; + } + execute_get (plugin, stmt, proc, proc_cls); +} + + + +/** + * Get results for a particular key in the datastore. + * + * @param cls closure + * @param offset offset (mod count). + * @param key key to match, never NULL + * @param vhash hash of the value, maybe NULL (to + * match all values that have the right key). + * Note that for DBlocks there is no difference + * betwen key and vhash, but for other blocks + * there may be! + * @param type entries of which type are relevant? + * Use 0 for any type. + * @param proc function to call on each matching value; + * will be called once with a NULL value at the end + * @param proc_cls closure for proc + */ +static void +sqlite_plugin_get_key (void *cls, uint64_t offset, const GNUNET_HashCode * key, + const GNUNET_HashCode * vhash, + enum GNUNET_BLOCK_Type type, PluginDatumProcessor proc, + void *proc_cls) +{ + struct Plugin *plugin = cls; + int ret; + int total; + int limit_off; + unsigned int sqoff; + sqlite3_stmt *stmt; + char scratch[256]; + + GNUNET_assert (proc != NULL); + GNUNET_assert (key != NULL); + GNUNET_snprintf (scratch, sizeof (scratch), + "SELECT count(*) FROM gn090 WHERE hash=?%s%s", + vhash == NULL ? "" : " AND vhash=?", + type == 0 ? "" : " AND type=?"); + if (sq_prepare (plugin->dbh, scratch, &stmt) != SQLITE_OK) + { + LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + "sqlite_prepare"); + proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0); + return; + } + sqoff = 1; + ret = + sqlite3_bind_blob (stmt, sqoff++, key, sizeof (GNUNET_HashCode), + SQLITE_TRANSIENT); + if ((vhash != NULL) && (ret == SQLITE_OK)) + ret = + sqlite3_bind_blob (stmt, sqoff++, vhash, sizeof (GNUNET_HashCode), + SQLITE_TRANSIENT); + if ((type != 0) && (ret == SQLITE_OK)) + ret = sqlite3_bind_int (stmt, sqoff++, type); + if (SQLITE_OK != ret) + { + LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR, "sqlite_bind"); + sqlite3_finalize (stmt); + proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0); + return; + } + ret = sqlite3_step (stmt); + if (ret != SQLITE_ROW) + { + LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + "sqlite_step"); + sqlite3_finalize (stmt); + proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0); + return; + } + total = sqlite3_column_int (stmt, 0); + sqlite3_finalize (stmt); + if (0 == total) + { + proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0); + return; + } + limit_off = (int) (offset % total); + if (limit_off < 0) + limit_off += total; + GNUNET_snprintf (scratch, sizeof (scratch), + "SELECT type, prio, anonLevel, expire, hash, value, _ROWID_ " + "FROM gn090 WHERE hash=?%s%s " + "ORDER BY _ROWID_ ASC LIMIT 1 OFFSET ?", + vhash == NULL ? "" : " AND vhash=?", + type == 0 ? "" : " AND type=?"); + if (sq_prepare (plugin->dbh, scratch, &stmt) != SQLITE_OK) + { + LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + "sqlite_prepare"); + proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0); + return; + } + sqoff = 1; + ret = + sqlite3_bind_blob (stmt, sqoff++, key, sizeof (GNUNET_HashCode), + SQLITE_TRANSIENT); + if ((vhash != NULL) && (ret == SQLITE_OK)) + ret = + sqlite3_bind_blob (stmt, sqoff++, vhash, sizeof (GNUNET_HashCode), + SQLITE_TRANSIENT); + if ((type != 0) && (ret == SQLITE_OK)) + ret = sqlite3_bind_int (stmt, sqoff++, type); + if (ret == SQLITE_OK) + ret = sqlite3_bind_int64 (stmt, sqoff++, limit_off); + if (ret != SQLITE_OK) + { + LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + "sqlite_bind"); + proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0); + return; + } + execute_get (plugin, stmt, proc, proc_cls); + sqlite3_finalize (stmt); +} + + + +/** + * Context for 'repl_proc' function. + */ +struct ReplCtx +{ + + /** + * Function to call for the result (or the NULL). + */ + PluginDatumProcessor proc; + + /** + * Closure for proc. + */ + void *proc_cls; + + /** + * UID to use. + */ + uint64_t uid; + + /** + * Yes if UID was set. + */ + int have_uid; +}; + + +/** + * Wrapper for the processor for 'sqlite_plugin_replication_get'. + * Decrements the replication counter and calls the original + * processor. + * + * @param cls closure + * @param key key for the content + * @param size number of bytes in data + * @param data content stored + * @param type type of the content + * @param priority priority of the content + * @param anonymity anonymity-level for the content + * @param expiration expiration time for the content + * @param uid unique identifier for the datum; + * maybe 0 if no unique identifier is available + * + * @return GNUNET_OK for normal return, + * GNUNET_NO to delete the item + */ +static int +repl_proc (void *cls, const GNUNET_HashCode * key, uint32_t size, + const void *data, enum GNUNET_BLOCK_Type type, uint32_t priority, + uint32_t anonymity, struct GNUNET_TIME_Absolute expiration, + uint64_t uid) +{ + struct ReplCtx *rc = cls; + int ret; + + ret = + rc->proc (rc->proc_cls, key, size, data, type, priority, anonymity, + expiration, uid); + if (key != NULL) + { + rc->uid = uid; + rc->have_uid = GNUNET_YES; + } + return ret; +} + + +/** + * Get a random item for replication. Returns a single random item + * from those with the highest replication counters. The item's + * replication counter is decremented by one IF it was positive before. + * Call 'proc' with all values ZERO or NULL if the datastore is empty. + * + * @param cls closure + * @param proc function to call the value (once only). + * @param proc_cls closure for proc + */ +static void +sqlite_plugin_get_replication (void *cls, PluginDatumProcessor proc, + void *proc_cls) +{ + struct Plugin *plugin = cls; + struct ReplCtx rc; + uint64_t rvalue; + uint32_t repl; + sqlite3_stmt *stmt; + +#if DEBUG_SQLITE + GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite", + "Getting random block based on replication order.\n"); +#endif + rc.have_uid = GNUNET_NO; + rc.proc = proc; + rc.proc_cls = proc_cls; + stmt = plugin->maxRepl; + if (SQLITE_ROW != sqlite3_step (stmt)) + { + if (SQLITE_OK != sqlite3_reset (stmt)) + LOG_SQLITE (plugin, NULL, + GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + "sqlite3_reset"); + /* DB empty */ + proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0); + return; + } + repl = sqlite3_column_int (stmt, 0); + if (SQLITE_OK != sqlite3_reset (stmt)) + LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + "sqlite3_reset"); + stmt = plugin->selRepl; + rvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK, UINT64_MAX); + if (SQLITE_OK != sqlite3_bind_int64 (stmt, 1, rvalue)) + { + LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + "sqlite3_bind_XXXX"); + if (SQLITE_OK != sqlite3_reset (stmt)) + LOG_SQLITE (plugin, NULL, + GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + "sqlite3_reset"); + proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0); + return; + } + if (SQLITE_OK != sqlite3_bind_int (stmt, 2, repl)) + { + LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + "sqlite3_bind_XXXX"); + if (SQLITE_OK != sqlite3_reset (stmt)) + LOG_SQLITE (plugin, NULL, + GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + "sqlite3_reset"); + proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0); + return; + } + execute_get (plugin, stmt, &repl_proc, &rc); + if (GNUNET_YES == rc.have_uid) + { + if (SQLITE_OK != sqlite3_bind_int64 (plugin->updRepl, 1, rc.uid)) + { + LOG_SQLITE (plugin, NULL, + GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + "sqlite3_bind_XXXX"); + if (SQLITE_OK != sqlite3_reset (plugin->updRepl)) + LOG_SQLITE (plugin, NULL, + GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + "sqlite3_reset"); + return; + } + if (SQLITE_DONE != sqlite3_step (plugin->updRepl)) + LOG_SQLITE (plugin, NULL, + GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + "sqlite3_step"); + if (SQLITE_OK != sqlite3_reset (plugin->updRepl)) + LOG_SQLITE (plugin, NULL, + GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + "sqlite3_reset"); + } +} + + + +/** + * Get a random item that has expired or has low priority. + * Call 'proc' with all values ZERO or NULL if the datastore is empty. + * + * @param cls closure + * @param proc function to call the value (once only). + * @param proc_cls closure for proc + */ +static void +sqlite_plugin_get_expiration (void *cls, PluginDatumProcessor proc, + void *proc_cls) +{ + struct Plugin *plugin = cls; + sqlite3_stmt *stmt; + struct GNUNET_TIME_Absolute now; + +#if DEBUG_SQLITE + GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite", + "Getting random block based on expiration and priority order.\n"); +#endif + now = GNUNET_TIME_absolute_get (); + stmt = plugin->selExpi; + if (SQLITE_OK != sqlite3_bind_int64 (stmt, 1, now.abs_value)) + { + LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + "sqlite3_bind_XXXX"); + if (SQLITE_OK != sqlite3_reset (stmt)) + LOG_SQLITE (plugin, NULL, + GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + "sqlite3_reset"); + proc (proc_cls, NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0); + return; + } + execute_get (plugin, stmt, proc, proc_cls); +} + + + +/** + * Get all of the keys in the datastore. + * + * @param cls closure + * @param proc function to call on each key + * @param proc_cls closure for proc + */ +static void +sqlite_plugin_get_keys (void *cls, + PluginKeyProcessor proc, + void *proc_cls) +{ + struct Plugin *plugin = cls; + const GNUNET_HashCode *key; + sqlite3_stmt *stmt; + int ret; + + GNUNET_assert (proc != NULL); + if (sq_prepare (plugin->dbh, "SELECT hash FROM gn090", &stmt) != SQLITE_OK) + { + LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + "sqlite_prepare"); + return; + } + while (SQLITE_ROW == (ret = sqlite3_step (stmt))) + { + key = sqlite3_column_blob (stmt, 1); + if (sizeof (GNUNET_HashCode) == sqlite3_column_bytes (stmt, 1)) + proc (proc_cls, key, 1); + } + if (SQLITE_DONE != ret) + LOG_SQLITE (plugin, NULL, GNUNET_ERROR_TYPE_ERROR, "sqlite_step"); + sqlite3_finalize (stmt); +} + + +/** + * Drop database. + * + * @param cls our plugin context + */ +static void +sqlite_plugin_drop (void *cls) +{ + struct Plugin *plugin = cls; + + plugin->drop_on_shutdown = GNUNET_YES; +} + + +/** + * Get an estimate of how much space the database is + * currently using. + * + * @param cls the 'struct Plugin' + * @return the size of the database on disk (estimate) + */ +static unsigned long long +sqlite_plugin_estimate_size (void *cls) +{ + struct Plugin *plugin = cls; + sqlite3_stmt *stmt; + uint64_t pages; + uint64_t page_size; + +#if ENULL_DEFINED + char *e; +#endif + + if (SQLITE_VERSION_NUMBER < 3006000) + { + GNUNET_log_from (GNUNET_ERROR_TYPE_WARNING, "datastore-sqlite", + _ + ("sqlite version to old to determine size, assuming zero\n")); + return 0; + } + CHECK (SQLITE_OK == sqlite3_exec (plugin->dbh, "VACUUM", NULL, NULL, ENULL)); + CHECK (SQLITE_OK == + sqlite3_exec (plugin->dbh, "PRAGMA auto_vacuum=INCREMENTAL", NULL, + NULL, ENULL)); + CHECK (SQLITE_OK == sq_prepare (plugin->dbh, "PRAGMA page_count", &stmt)); + if (SQLITE_ROW == sqlite3_step (stmt)) + pages = sqlite3_column_int64 (stmt, 0); + else + pages = 0; + sqlite3_finalize (stmt); + CHECK (SQLITE_OK == sq_prepare (plugin->dbh, "PRAGMA page_size", &stmt)); + CHECK (SQLITE_ROW == sqlite3_step (stmt)); + page_size = sqlite3_column_int64 (stmt, 0); + sqlite3_finalize (stmt); + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + _ + ("Using sqlite page utilization to estimate payload (%llu pages of size %llu bytes)\n"), + (unsigned long long) pages, (unsigned long long) page_size); + return pages * page_size; +} + + +/** + * Entry point for the plugin. + * + * @param cls the "struct GNUNET_DATASTORE_PluginEnvironment*" + * @return NULL on error, othrewise the plugin context + */ +void * +libgnunet_plugin_datastore_sqlite_init (void *cls) +{ + static struct Plugin plugin; + struct GNUNET_DATASTORE_PluginEnvironment *env = cls; + struct GNUNET_DATASTORE_PluginFunctions *api; + + if (plugin.env != NULL) + return NULL; /* can only initialize once! */ + memset (&plugin, 0, sizeof (struct Plugin)); + plugin.env = env; + if (GNUNET_OK != database_setup (env->cfg, &plugin)) + { + database_shutdown (&plugin); + return NULL; + } + api = GNUNET_malloc (sizeof (struct GNUNET_DATASTORE_PluginFunctions)); + api->cls = &plugin; + api->estimate_size = &sqlite_plugin_estimate_size; + api->put = &sqlite_plugin_put; + api->update = &sqlite_plugin_update; + api->get_key = &sqlite_plugin_get_key; + api->get_replication = &sqlite_plugin_get_replication; + api->get_expiration = &sqlite_plugin_get_expiration; + api->get_zero_anonymity = &sqlite_plugin_get_zero_anonymity; + api->get_keys = &sqlite_plugin_get_keys; + api->drop = &sqlite_plugin_drop; + GNUNET_log_from (GNUNET_ERROR_TYPE_INFO, "sqlite", + _("Sqlite database running\n")); + return api; +} + + +/** + * Exit point from the plugin. + * + * @param cls the plugin context (as returned by "init") + * @return always NULL + */ +void * +libgnunet_plugin_datastore_sqlite_done (void *cls) +{ + char *fn; + struct GNUNET_DATASTORE_PluginFunctions *api = cls; + struct Plugin *plugin = api->cls; + +#if DEBUG_SQLITE + GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite", + "sqlite plugin is done\n"); +#endif + + fn = NULL; + if (plugin->drop_on_shutdown) + fn = GNUNET_strdup (plugin->fn); +#if DEBUG_SQLITE + GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite", + "Shutting down database\n"); +#endif + database_shutdown (plugin); + plugin->env = NULL; + GNUNET_free (api); + if (fn != NULL) + { + if (0 != UNLINK (fn)) + GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", fn); + GNUNET_free (fn); + } +#if DEBUG_SQLITE + GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite", + "sqlite plugin is finished\n"); +#endif + return NULL; +} + +/* end of plugin_datastore_sqlite.c */ diff --git a/src/datastore/plugin_datastore_template.c b/src/datastore/plugin_datastore_template.c new file mode 100644 index 0000000..174d3d2 --- /dev/null +++ b/src/datastore/plugin_datastore_template.c @@ -0,0 +1,262 @@ +/* + This file is part of GNUnet + (C) 2009, 2011 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file datastore/plugin_datastore_template.c + * @brief template-based datastore backend + * @author Christian Grothoff + */ + +#include "platform.h" +#include "gnunet_datastore_plugin.h" + + +/** + * Context for all functions in this plugin. + */ +struct Plugin +{ + /** + * Our execution environment. + */ + struct GNUNET_DATASTORE_PluginEnvironment *env; +}; + + +/** + * Get an estimate of how much space the database is + * currently using. + * + * @param cls our "struct Plugin*" + * @return number of bytes used on disk + */ +static unsigned long long +template_plugin_estimate_size (void *cls) +{ + GNUNET_break (0); + return 0; +} + + +/** + * Store an item in the datastore. + * + * @param cls closure + * @param key key for the item + * @param size number of bytes in data + * @param data content stored + * @param type type of the content + * @param priority priority of the content + * @param anonymity anonymity-level for the content + * @param replication replication-level for the content + * @param expiration expiration time for the content + * @param msg set to error message + * @return GNUNET_OK on success + */ +static int +template_plugin_put (void *cls, const GNUNET_HashCode * key, uint32_t size, + const void *data, enum GNUNET_BLOCK_Type type, + uint32_t priority, uint32_t anonymity, + uint32_t replication, + struct GNUNET_TIME_Absolute expiration, char **msg) +{ + GNUNET_break (0); + *msg = GNUNET_strdup ("not implemented"); + return GNUNET_SYSERR; +} + + +/** + * Get one of the results for a particular key in the datastore. + * + * @param cls closure + * @param offset offset of the result (modulo num-results); + * specific ordering does not matter for the offset + * @param key maybe NULL (to match all entries) + * @param vhash hash of the value, maybe NULL (to + * match all values that have the right key). + * Note that for DBlocks there is no difference + * betwen key and vhash, but for other blocks + * there may be! + * @param type entries of which type are relevant? + * Use 0 for any type. + * @param proc function to call on each matching value; + * will be called with NULL if nothing matches + * @param proc_cls closure for proc + */ +static void +template_plugin_get_key (void *cls, uint64_t offset, + const GNUNET_HashCode * key, + const GNUNET_HashCode * vhash, + enum GNUNET_BLOCK_Type type, PluginDatumProcessor proc, + void *proc_cls) +{ + GNUNET_break (0); +} + + + +/** + * Get a random item for replication. Returns a single, not expired, + * random item from those with the highest replication counters. The + * item's replication counter is decremented by one IF it was positive + * before. Call 'proc' with all values ZERO or NULL if the datastore + * is empty. + * + * @param cls closure + * @param proc function to call the value (once only). + * @param proc_cls closure for proc + */ +static void +template_plugin_get_replication (void *cls, PluginDatumProcessor proc, + void *proc_cls) +{ + GNUNET_break (0); +} + + +/** + * Get a random item for expiration. Call 'proc' with all values ZERO + * or NULL if the datastore is empty. + * + * @param cls closure + * @param proc function to call the value (once only). + * @param proc_cls closure for proc + */ +static void +template_plugin_get_expiration (void *cls, PluginDatumProcessor proc, + void *proc_cls) +{ + GNUNET_break (0); +} + + +/** + * Update the priority for a particular key in the datastore. If + * the expiration time in value is different than the time found in + * the datastore, the higher value should be kept. For the + * anonymity level, the lower value is to be used. The specified + * priority should be added to the existing priority, ignoring the + * priority in value. + * + * Note that it is possible for multiple values to match this put. + * In that case, all of the respective values are updated. + * + * @param cls our "struct Plugin*" + * @param uid unique identifier of the datum + * @param delta by how much should the priority + * change? If priority + delta < 0 the + * priority should be set to 0 (never go + * negative). + * @param expire new expiration time should be the + * MAX of any existing expiration time and + * this value + * @param msg set to error message + * @return GNUNET_OK on success + */ +static int +template_plugin_update (void *cls, uint64_t uid, int delta, + struct GNUNET_TIME_Absolute expire, char **msg) +{ + GNUNET_break (0); + *msg = GNUNET_strdup ("not implemented"); + return GNUNET_SYSERR; +} + + +/** + * Call the given processor on an item with zero anonymity. + * + * @param cls our "struct Plugin*" + * @param offset offset of the result (modulo num-results); + * specific ordering does not matter for the offset + * @param type entries of which type should be considered? + * Use 0 for any type. + * @param proc function to call on each matching value; + * will be called with NULL if no value matches + * @param proc_cls closure for proc + */ +static void +template_plugin_get_zero_anonymity (void *cls, uint64_t offset, + enum GNUNET_BLOCK_Type type, + PluginDatumProcessor proc, void *proc_cls) +{ + GNUNET_break (0); +} + + +/** + * Drop database. + */ +static void +template_plugin_drop (void *cls) +{ + GNUNET_break (0); +} + + +/** + * Entry point for the plugin. + * + * @param cls the "struct GNUNET_DATASTORE_PluginEnvironment*" + * @return our "struct Plugin*" + */ +void * +libgnunet_plugin_datastore_template_init (void *cls) +{ + struct GNUNET_DATASTORE_PluginEnvironment *env = cls; + struct GNUNET_DATASTORE_PluginFunctions *api; + struct Plugin *plugin; + + plugin = GNUNET_malloc (sizeof (struct Plugin)); + plugin->env = env; + api = GNUNET_malloc (sizeof (struct GNUNET_DATASTORE_PluginFunctions)); + api->cls = plugin; + api->estimate_size = &template_plugin_estimate_size; + api->put = &template_plugin_put; + api->update = &template_plugin_update; + api->get_key = &template_plugin_get_key; + api->get_replication = &template_plugin_get_replication; + api->get_expiration = &template_plugin_get_expiration; + api->get_zero_anonymity = &template_plugin_get_zero_anonymity; + api->drop = &template_plugin_drop; + GNUNET_log_from (GNUNET_ERROR_TYPE_INFO, "template", + _("Template database running\n")); + return api; +} + + +/** + * Exit point from the plugin. + * @param cls our "struct Plugin*" + * @return always NULL + */ +void * +libgnunet_plugin_datastore_template_done (void *cls) +{ + struct GNUNET_DATASTORE_PluginFunctions *api = cls; + struct Plugin *plugin = api->cls; + + GNUNET_free (plugin); + GNUNET_free (api); + return NULL; +} + +/* end of plugin_datastore_template.c */ diff --git a/src/datastore/test_datastore_api.c b/src/datastore/test_datastore_api.c new file mode 100644 index 0000000..25836ca --- /dev/null +++ b/src/datastore/test_datastore_api.c @@ -0,0 +1,589 @@ +/* + This file is part of GNUnet. + (C) 2004, 2005, 2006, 2007, 2009 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ +/* + * @file datastore/test_datastore_api.c + * @brief Test for the basic datastore API. + * @author Christian Grothoff + * + * TODO: + * - test reservation failure + */ + +#include "platform.h" +#include "gnunet_util_lib.h" +#include "gnunet_protocols.h" +#include "gnunet_datastore_service.h" + +#define VERBOSE GNUNET_NO + +#define START_DATASTORE GNUNET_YES + +/** + * How long until we give up on transmitting the message? + */ +#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 15) + +#define ITERATIONS 256 + +static struct GNUNET_DATASTORE_Handle *datastore; + +static struct GNUNET_TIME_Absolute now; + +static int ok; + +/** + * Name of plugin under test. + */ +static const char *plugin_name; + +static size_t +get_size (int i) +{ + return 8 * i; +} + + +static const void * +get_data (int i) +{ + static char buf[60000]; + + memset (buf, i, 8 * i); + return buf; +} + + +static int +get_type (int i) +{ + return i + 1; +} + + +static int +get_priority (int i) +{ + return i + 1; +} + + +static int +get_anonymity (int i) +{ + return i; +} + + +static struct GNUNET_TIME_Absolute +get_expiration (int i) +{ + struct GNUNET_TIME_Absolute av; + + av.abs_value = now.abs_value + 20000000 - i * 1000; + return av; +} + +enum RunPhase +{ + RP_DONE = 0, + RP_PUT = 1, + RP_GET = 2, + RP_DEL = 3, + RP_DO_DEL = 4, + RP_DELVALIDATE = 5, + RP_RESERVE = 6, + RP_PUT_MULTIPLE = 7, + RP_PUT_MULTIPLE_NEXT = 8, + RP_GET_MULTIPLE = 9, + RP_GET_MULTIPLE_NEXT = 10, + RP_UPDATE = 11, + RP_UPDATE_VALIDATE = 12, + RP_ERROR +}; + + +struct CpsRunContext +{ + GNUNET_HashCode key; + int i; + int rid; + const struct GNUNET_CONFIGURATION_Handle *cfg; + void *data; + size_t size; + enum RunPhase phase; + uint64_t uid; + uint64_t offset; + uint64_t first_uid; +}; + + +static void +run_continuation (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc); + + +static void +check_success (void *cls, int success, struct GNUNET_TIME_Absolute min_expiration, const char *msg) +{ + struct CpsRunContext *crc = cls; + + if (GNUNET_OK != success) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Operation %d/%d not successfull: `%s'\n", crc->phase, crc->i, + msg); + crc->phase = RP_ERROR; + } + GNUNET_free_non_null (crc->data); + crc->data = NULL; + GNUNET_SCHEDULER_add_continuation (&run_continuation, crc, + GNUNET_SCHEDULER_REASON_PREREQ_DONE); +} + + +static void +get_reserved (void *cls, int success, struct GNUNET_TIME_Absolute min_expiration, const char *msg) +{ + struct CpsRunContext *crc = cls; + + if (0 >= success) + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Error obtaining reservation: `%s'\n", + msg); + GNUNET_assert (0 < success); + crc->rid = success; + GNUNET_SCHEDULER_add_continuation (&run_continuation, crc, + GNUNET_SCHEDULER_REASON_PREREQ_DONE); +} + + +static void +check_value (void *cls, const GNUNET_HashCode * key, size_t size, + const void *data, enum GNUNET_BLOCK_Type type, uint32_t priority, + uint32_t anonymity, struct GNUNET_TIME_Absolute expiration, + uint64_t uid) +{ + struct CpsRunContext *crc = cls; + int i; + + i = crc->i; + if (NULL == key) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Value check failed (got NULL key) in %d/%d\n", crc->phase, + crc->i); + crc->phase = RP_ERROR; + GNUNET_SCHEDULER_add_continuation (&run_continuation, crc, + GNUNET_SCHEDULER_REASON_PREREQ_DONE); + return; + } +#if 0 + FPRINTF (stderr, "Check value got `%s' of size %u, type %d, expire %llu\n", + GNUNET_h2s (key), (unsigned int) size, type, + (unsigned long long) expiration.abs_value); + FPRINTF (stderr, + "Check value iteration %d wants size %u, type %d, expire %llu\n", i, + (unsigned int) get_size (i), get_type (i), + (unsigned long long) get_expiration (i).abs_value); +#endif + GNUNET_assert (size == get_size (i)); + GNUNET_assert (0 == memcmp (data, get_data (i), size)); + GNUNET_assert (type == get_type (i)); + GNUNET_assert (priority == get_priority (i)); + GNUNET_assert (anonymity == get_anonymity (i)); + GNUNET_assert (expiration.abs_value == get_expiration (i).abs_value); + crc->offset++; + if (crc->i == 0) + { + crc->phase = RP_DEL; + crc->i = ITERATIONS; + } + GNUNET_SCHEDULER_add_continuation (&run_continuation, crc, + GNUNET_SCHEDULER_REASON_PREREQ_DONE); +} + + +static void +delete_value (void *cls, const GNUNET_HashCode * key, size_t size, + const void *data, enum GNUNET_BLOCK_Type type, uint32_t priority, + uint32_t anonymity, struct GNUNET_TIME_Absolute expiration, + uint64_t uid) +{ + struct CpsRunContext *crc = cls; + + GNUNET_assert (crc->data == NULL); + GNUNET_assert (NULL != key); + crc->size = size; + crc->key = *key; + crc->data = GNUNET_malloc (size); + memcpy (crc->data, data, size); + crc->phase = RP_DO_DEL; + GNUNET_SCHEDULER_add_continuation (&run_continuation, crc, + GNUNET_SCHEDULER_REASON_PREREQ_DONE); +} + + +static void +check_nothing (void *cls, const GNUNET_HashCode * key, size_t size, + const void *data, enum GNUNET_BLOCK_Type type, uint32_t priority, + uint32_t anonymity, struct GNUNET_TIME_Absolute expiration, + uint64_t uid) +{ + struct CpsRunContext *crc = cls; + + GNUNET_assert (key == NULL); + if (crc->i == 0) + crc->phase = RP_RESERVE; + GNUNET_SCHEDULER_add_continuation (&run_continuation, crc, + GNUNET_SCHEDULER_REASON_PREREQ_DONE); +} + + +static void +check_multiple (void *cls, const GNUNET_HashCode * key, size_t size, + const void *data, enum GNUNET_BLOCK_Type type, + uint32_t priority, uint32_t anonymity, + struct GNUNET_TIME_Absolute expiration, uint64_t uid) +{ + struct CpsRunContext *crc = cls; + + GNUNET_assert (key != NULL); + switch (crc->phase) + { + case RP_GET_MULTIPLE: + crc->phase = RP_GET_MULTIPLE_NEXT; + crc->first_uid = uid; + crc->offset++; + break; + case RP_GET_MULTIPLE_NEXT: + GNUNET_assert (uid != crc->first_uid); + crc->phase = RP_UPDATE; + break; + default: + GNUNET_break (0); + crc->phase = RP_ERROR; + break; + } + if (priority == get_priority (42)) + crc->uid = uid; + GNUNET_SCHEDULER_add_continuation (&run_continuation, crc, + GNUNET_SCHEDULER_REASON_PREREQ_DONE); +} + + +static void +check_update (void *cls, const GNUNET_HashCode * key, size_t size, + const void *data, enum GNUNET_BLOCK_Type type, uint32_t priority, + uint32_t anonymity, struct GNUNET_TIME_Absolute expiration, + uint64_t uid) +{ + struct CpsRunContext *crc = cls; + + GNUNET_assert (key != NULL); + if ((anonymity == get_anonymity (42)) && (size == get_size (42)) && + (priority == get_priority (42) + 100)) + crc->phase = RP_DONE; + else + { + GNUNET_assert (size == get_size (43)); + crc->offset++; + } + GNUNET_SCHEDULER_add_continuation (&run_continuation, crc, + GNUNET_SCHEDULER_REASON_PREREQ_DONE); +} + + +static void +run_continuation (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct CpsRunContext *crc = cls; + + ok = (int) crc->phase; +#if VERBOSE + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Test in phase %u\n", crc->phase); +#endif + switch (crc->phase) + { + case RP_PUT: +#if VERBOSE + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Executing `%s' number %u\n", "PUT", + crc->i); +#endif + GNUNET_CRYPTO_hash (&crc->i, sizeof (int), &crc->key); + GNUNET_DATASTORE_put (datastore, 0, &crc->key, get_size (crc->i), + get_data (crc->i), get_type (crc->i), + get_priority (crc->i), get_anonymity (crc->i), 0, + get_expiration (crc->i), 1, 1, TIMEOUT, + &check_success, crc); + crc->i++; + if (crc->i == ITERATIONS) + crc->phase = RP_GET; + break; + case RP_GET: + crc->i--; +#if VERBOSE + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Executing `%s' number %u\n", "GET", + crc->i); +#endif + GNUNET_CRYPTO_hash (&crc->i, sizeof (int), &crc->key); + GNUNET_DATASTORE_get_key (datastore, crc->offset, &crc->key, + get_type (crc->i), 1, 1, TIMEOUT, &check_value, + crc); + break; + case RP_DEL: + crc->i--; +#if VERBOSE + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Executing `%s' number %u\n", "DEL", + crc->i); +#endif + crc->data = NULL; + GNUNET_CRYPTO_hash (&crc->i, sizeof (int), &crc->key); + GNUNET_assert (NULL != + GNUNET_DATASTORE_get_key (datastore, crc->offset, &crc->key, + get_type (crc->i), 1, 1, TIMEOUT, + &delete_value, crc)); + break; + case RP_DO_DEL: +#if VERBOSE + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Executing `%s' number %u\n", "DO_DEL", + crc->i); +#endif + if (crc->i == 0) + { + crc->i = ITERATIONS; + crc->phase = RP_DELVALIDATE; + } + else + { + crc->phase = RP_DEL; + } + GNUNET_assert (NULL != + GNUNET_DATASTORE_remove (datastore, &crc->key, crc->size, + crc->data, 1, 1, TIMEOUT, + &check_success, crc)); + break; + case RP_DELVALIDATE: + crc->i--; +#if VERBOSE + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Executing `%s' number %u\n", + "DEL-VALIDATE", crc->i); +#endif + GNUNET_CRYPTO_hash (&crc->i, sizeof (int), &crc->key); + GNUNET_assert (NULL != + GNUNET_DATASTORE_get_key (datastore, crc->offset, &crc->key, + get_type (crc->i), 1, 1, TIMEOUT, + &check_nothing, crc)); + break; + case RP_RESERVE: + crc->phase = RP_PUT_MULTIPLE; + GNUNET_DATASTORE_reserve (datastore, 128 * 1024, 2, 1, 1, TIMEOUT, + &get_reserved, crc); + break; + case RP_PUT_MULTIPLE: + crc->phase = RP_PUT_MULTIPLE_NEXT; + GNUNET_DATASTORE_put (datastore, crc->rid, &crc->key, get_size (42), + get_data (42), get_type (42), get_priority (42), + get_anonymity (42), 0, get_expiration (42), 1, 1, + TIMEOUT, &check_success, crc); + break; + case RP_PUT_MULTIPLE_NEXT: + crc->phase = RP_GET_MULTIPLE; + GNUNET_DATASTORE_put (datastore, crc->rid, &crc->key, get_size (43), + get_data (43), get_type (42), get_priority (43), + get_anonymity (43), 0, get_expiration (43), 1, 1, + TIMEOUT, &check_success, crc); + break; + case RP_GET_MULTIPLE: + GNUNET_assert (NULL != + GNUNET_DATASTORE_get_key (datastore, crc->offset, &crc->key, + get_type (42), 1, 1, TIMEOUT, + &check_multiple, crc)); + break; + case RP_GET_MULTIPLE_NEXT: + GNUNET_assert (NULL != + GNUNET_DATASTORE_get_key (datastore, crc->offset, &crc->key, + get_type (42), 1, 1, TIMEOUT, + &check_multiple, crc)); + break; + case RP_UPDATE: + GNUNET_assert (crc->uid > 0); + crc->phase = RP_UPDATE_VALIDATE; + GNUNET_DATASTORE_update (datastore, crc->uid, 100, get_expiration (42), 1, + 1, TIMEOUT, &check_success, crc); + break; + case RP_UPDATE_VALIDATE: + GNUNET_assert (NULL != + GNUNET_DATASTORE_get_key (datastore, crc->offset, &crc->key, + get_type (42), 1, 1, TIMEOUT, + &check_update, crc)); + break; + case RP_DONE: +#if VERBOSE + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Finished, disconnecting\n"); +#endif + GNUNET_DATASTORE_disconnect (datastore, GNUNET_YES); + GNUNET_free (crc); + ok = 0; + break; + case RP_ERROR: + GNUNET_DATASTORE_disconnect (datastore, GNUNET_YES); + GNUNET_free (crc); + ok = 43; + break; + } +} + + +static void +run_tests (void *cls, int32_t success, struct GNUNET_TIME_Absolute min_expiration, const char *msg) +{ + struct CpsRunContext *crc = cls; + + switch (success) + { + case GNUNET_YES: + GNUNET_SCHEDULER_add_continuation (&run_continuation, crc, + GNUNET_SCHEDULER_REASON_PREREQ_DONE); + return; + case GNUNET_NO: + FPRINTF (stderr, "%s", "Test 'put' operation failed, key already exists (!?)\n"); + GNUNET_free (crc); + return; + case GNUNET_SYSERR: + FPRINTF (stderr, + "Test 'put' operation failed with error `%s' database likely not setup, skipping test.\n", + msg); + GNUNET_free (crc); + return; + default: + GNUNET_assert (0); + } +} + + +static void +run (void *cls, char *const *args, const char *cfgfile, + const struct GNUNET_CONFIGURATION_Handle *cfg) +{ + struct CpsRunContext *crc; + static GNUNET_HashCode zkey; + + crc = GNUNET_malloc (sizeof (struct CpsRunContext)); + crc->cfg = cfg; + crc->phase = RP_PUT; + now = GNUNET_TIME_absolute_get (); + datastore = GNUNET_DATASTORE_connect (cfg); + if (NULL == + GNUNET_DATASTORE_put (datastore, 0, &zkey, 4, "TEST", + GNUNET_BLOCK_TYPE_TEST, 0, 0, 0, + GNUNET_TIME_relative_to_absolute + (GNUNET_TIME_UNIT_SECONDS), 0, 1, + GNUNET_TIME_UNIT_MINUTES, &run_tests, crc)) + { + FPRINTF (stderr, "%s", "Test 'put' operation failed.\n"); + ok = 1; + GNUNET_free (crc); + } +} + + +static int +check () +{ + char cfg_name[128]; + +#if START_DATASTORE + struct GNUNET_OS_Process *proc; +#endif + char *const argv[] = { + "test-datastore-api", + "-c", + cfg_name, +#if VERBOSE + "-L", "DEBUG", +#endif + NULL + }; + struct GNUNET_GETOPT_CommandLineOption options[] = { + GNUNET_GETOPT_OPTION_END + }; + GNUNET_snprintf (cfg_name, sizeof (cfg_name), + "test_datastore_api_data_%s.conf", plugin_name); +#if START_DATASTORE + proc = + GNUNET_OS_start_process (GNUNET_YES, NULL, NULL, "gnunet-service-arm", + "gnunet-service-arm", +#if VERBOSE + "-L", "DEBUG", +#endif + "-c", cfg_name, NULL); +#endif + GNUNET_assert (NULL != proc); + GNUNET_PROGRAM_run ((sizeof (argv) / sizeof (char *)) - 1, argv, + "test-datastore-api", "nohelp", options, &run, NULL); +#if START_DATASTORE + sleep (1); /* give datastore chance to receive 'DROP' request */ + if (0 != GNUNET_OS_process_kill (proc, SIGTERM)) + { + GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING, "kill"); + ok = 1; + } + GNUNET_OS_process_wait (proc); + GNUNET_OS_process_close (proc); + proc = NULL; +#endif + if (ok != 0) + FPRINTF (stderr, "Missed some testcases: %u\n", ok); + return ok; +} + +int +main (int argc, char *argv[]) +{ + int ret; + char *pos; + char dir_name[128]; + + sleep (1); + /* determine name of plugin to use */ + plugin_name = argv[0]; + while (NULL != (pos = strstr (plugin_name, "_"))) + plugin_name = pos + 1; + if (NULL != (pos = strstr (plugin_name, "."))) + pos[0] = 0; + else + pos = (char *) plugin_name; + + GNUNET_snprintf (dir_name, sizeof (dir_name), "/tmp/test-gnunet-datastore-%s", + plugin_name); + GNUNET_DISK_directory_remove (dir_name); + GNUNET_log_setup ("test-datastore-api", +#if VERBOSE + "DEBUG", +#else + "WARNING", +#endif + NULL); + ret = check (); + if (pos != plugin_name) + pos[0] = '.'; + GNUNET_DISK_directory_remove (dir_name); + return ret; +} + +/* end of test_datastore_api.c */ diff --git a/src/datastore/test_datastore_api_data_mysql.conf b/src/datastore/test_datastore_api_data_mysql.conf new file mode 100644 index 0000000..8e5a3dc --- /dev/null +++ b/src/datastore/test_datastore_api_data_mysql.conf @@ -0,0 +1,28 @@ +@INLINE@ test_defaults.conf +[PATHS] +SERVICEHOME = /tmp/test-gnunet-datastore-mysql/ +DEFAULTCONFIG = test_datastore_api_data_mysql.conf + +[TESTING] +WEAKRANDOM = YES + +[arm] +PORT = 42466 +DEFAULTSERVICES = + +[statistics] +PORT = 22667 + +[resolver] +PORT = 42464 + +[datastore] +QUOTA = 10 MB +DATABASE = mysql + +[datastore-mysql] +DATABASE = gnunetcheck + +[fs] +AUTOSTART = NO + diff --git a/src/datastore/test_datastore_api_data_postgres.conf b/src/datastore/test_datastore_api_data_postgres.conf new file mode 100644 index 0000000..046c561 --- /dev/null +++ b/src/datastore/test_datastore_api_data_postgres.conf @@ -0,0 +1,28 @@ +@INLINE@ test_defaults.conf +[PATHS] +SERVICEHOME = /tmp/test-gnunet-datastore-postgres/ +DEFAULTCONFIG = test_datastore_api_data_postgres.conf + +[TESTING] +WEAKRANDOM = YES + +[arm] +PORT = 42466 +DEFAULTSERVICES = + +[statistics] +PORT = 22667 + +[resolver] +PORT = 42464 + +[datastore] +QUOTA = 10 MB +DATABASE = postgres + +[datastore-postgres] +CONFIG = dbname=gnunetcheck + +[fs] +AUTOSTART = NO + diff --git a/src/datastore/test_datastore_api_data_sqlite.conf b/src/datastore/test_datastore_api_data_sqlite.conf new file mode 100644 index 0000000..098f9d2 --- /dev/null +++ b/src/datastore/test_datastore_api_data_sqlite.conf @@ -0,0 +1,24 @@ +@INLINE@ test_defaults.conf +[PATHS] +SERVICEHOME = /tmp/test-gnunet-datastore-sqlite/ +DEFAULTCONFIG = test_datastore_api_data_sqlite.conf + +[TESTING] +WEAKRANDOM = YES + +[arm] +PORT = 42466 +DEFAULTSERVICES = + +[statistics] +PORT = 22667 + +[resolver] +PORT = 42464 + +[datastore] +QUOTA = 10 MB + +[fs] +AUTOSTART = NO + diff --git a/src/datastore/test_datastore_api_management.c b/src/datastore/test_datastore_api_management.c new file mode 100644 index 0000000..4015c2c --- /dev/null +++ b/src/datastore/test_datastore_api_management.c @@ -0,0 +1,373 @@ +/* + This file is part of GNUnet. + (C) 2004, 2005, 2006, 2007, 2009, 2011 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ +/* + * @file datastore/test_datastore_api_management.c + * @brief Test for the space management functions of the datastore implementation. + * @author Christian Grothoff + */ + +#include "platform.h" +#include "gnunet_util_lib.h" +#include "gnunet_protocols.h" +#include "gnunet_datastore_service.h" + +#define VERBOSE GNUNET_NO + +/** + * How long until we give up on transmitting the message? + */ +#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 60) + +/** + * Number of iterations to run; must be large enough + * so that the quota will be exceeded! + */ +#define ITERATIONS 5000 + +static struct GNUNET_DATASTORE_Handle *datastore; + +static struct GNUNET_TIME_Absolute now; + +static int ok; + +static const char *plugin_name; + +static size_t +get_size (int i) +{ + return 8 + 8 * (i % 256); +} + + +static const void * +get_data (int i) +{ + static char buf[60000]; + + memset (buf, i, 8 + 8 * (i % 256)); + return buf; +} + + +static int +get_type (int i) +{ + return 1; +} + + +static int +get_priority (int i) +{ + return i + 1; +} + + +static int +get_anonymity (int i) +{ + return i; +} + + +static struct GNUNET_TIME_Absolute +get_expiration (int i) +{ + struct GNUNET_TIME_Absolute av; + + av.abs_value = now.abs_value + i * 1000; + return av; +} + +enum RunPhase +{ + RP_PUT, + RP_GET, + RP_DONE, + RP_GET_FAIL +}; + + +struct CpsRunContext +{ + GNUNET_HashCode key; + int i; + int found; + const struct GNUNET_CONFIGURATION_Handle *cfg; + void *data; + enum RunPhase phase; + uint64_t offset; +}; + + +static void +run_continuation (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc); + + +static void +check_success (void *cls, int success, struct GNUNET_TIME_Absolute min_expiration, const char *msg) +{ + struct CpsRunContext *crc = cls; + + if (GNUNET_OK != success) + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "%s\n", msg); + GNUNET_assert (GNUNET_OK == success); + GNUNET_free_non_null (crc->data); + crc->data = NULL; + GNUNET_SCHEDULER_add_continuation (&run_continuation, crc, + GNUNET_SCHEDULER_REASON_PREREQ_DONE); +} + + +static void +check_value (void *cls, const GNUNET_HashCode * key, size_t size, + const void *data, enum GNUNET_BLOCK_Type type, uint32_t priority, + uint32_t anonymity, struct GNUNET_TIME_Absolute expiration, + uint64_t uid) +{ + struct CpsRunContext *crc = cls; + int i; + + if (NULL == key) + { + crc->phase = RP_GET_FAIL; + GNUNET_SCHEDULER_add_continuation (&run_continuation, crc, + GNUNET_SCHEDULER_REASON_PREREQ_DONE); + return; + } + i = crc->i; + GNUNET_assert (size == get_size (i)); + GNUNET_assert (0 == memcmp (data, get_data (i), size)); + GNUNET_assert (type == get_type (i)); + GNUNET_assert (priority == get_priority (i)); + GNUNET_assert (anonymity == get_anonymity (i)); + GNUNET_assert (expiration.abs_value == get_expiration (i).abs_value); + crc->offset++; + crc->i--; + if (crc->i == 0) + crc->phase = RP_DONE; + GNUNET_SCHEDULER_add_continuation (&run_continuation, crc, + GNUNET_SCHEDULER_REASON_PREREQ_DONE); +} + + +static void +check_nothing (void *cls, const GNUNET_HashCode * key, size_t size, + const void *data, enum GNUNET_BLOCK_Type type, uint32_t priority, + uint32_t anonymity, struct GNUNET_TIME_Absolute expiration, + uint64_t uid) +{ + struct CpsRunContext *crc = cls; + + GNUNET_assert (key == NULL); + if (0 == --crc->i) + crc->phase = RP_DONE; + GNUNET_SCHEDULER_add_continuation (&run_continuation, crc, + GNUNET_SCHEDULER_REASON_PREREQ_DONE); +} + + +static void +run_continuation (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct CpsRunContext *crc = cls; + + ok = (int) crc->phase; + switch (crc->phase) + { + case RP_PUT: +#if VERBOSE + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Executing `%s' number %u\n", "PUT", + crc->i); +#endif + GNUNET_CRYPTO_hash (&crc->i, sizeof (int), &crc->key); + GNUNET_DATASTORE_put (datastore, 0, &crc->key, get_size (crc->i), + get_data (crc->i), get_type (crc->i), + get_priority (crc->i), get_anonymity (crc->i), 0, + get_expiration (crc->i), 1, 1, TIMEOUT, + &check_success, crc); + crc->i++; + if (crc->i == ITERATIONS) + { + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Sleeping to give datastore time to clean up\n"); + sleep (1); + crc->phase = RP_GET; + crc->i--; + } + break; + case RP_GET: +#if VERBOSE + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Executing `%s' number %u\n", "GET", + crc->i); +#endif + GNUNET_CRYPTO_hash (&crc->i, sizeof (int), &crc->key); + GNUNET_DATASTORE_get_key (datastore, crc->offset++, &crc->key, + get_type (crc->i), 1, 1, TIMEOUT, &check_value, + crc); + break; + case RP_GET_FAIL: +#if VERBOSE + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Executing `%s' number %u\n", "GET(f)", + crc->i); +#endif + GNUNET_CRYPTO_hash (&crc->i, sizeof (int), &crc->key); + GNUNET_DATASTORE_get_key (datastore, crc->offset++, &crc->key, + get_type (crc->i), 1, 1, TIMEOUT, &check_nothing, + crc); + break; + case RP_DONE: + GNUNET_assert (0 == crc->i); +#if VERBOSE + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Finished, disconnecting\n"); +#endif + GNUNET_DATASTORE_disconnect (datastore, GNUNET_YES); + GNUNET_free (crc); + ok = 0; + } +} + + +static void +run_tests (void *cls, int success, struct GNUNET_TIME_Absolute min_expiration, const char *msg) +{ + struct CpsRunContext *crc = cls; + + if (success != GNUNET_YES) + { + FPRINTF (stderr, + "Test 'put' operation failed with error `%s' database likely not setup, skipping test.", + msg); + GNUNET_free (crc); + return; + } + GNUNET_SCHEDULER_add_continuation (&run_continuation, crc, + GNUNET_SCHEDULER_REASON_PREREQ_DONE); +} + + +static void +run (void *cls, char *const *args, const char *cfgfile, + const struct GNUNET_CONFIGURATION_Handle *cfg) +{ + struct CpsRunContext *crc; + static GNUNET_HashCode zkey; + + crc = GNUNET_malloc (sizeof (struct CpsRunContext)); + crc->cfg = cfg; + crc->phase = RP_PUT; + now = GNUNET_TIME_absolute_get (); + datastore = GNUNET_DATASTORE_connect (cfg); + if (NULL == + GNUNET_DATASTORE_put (datastore, 0, &zkey, 4, "TEST", + GNUNET_BLOCK_TYPE_TEST, 0, 0, 0, + GNUNET_TIME_relative_to_absolute + (GNUNET_TIME_UNIT_SECONDS), 0, 1, + GNUNET_TIME_UNIT_MINUTES, &run_tests, crc)) + { + FPRINTF (stderr, "%s", "Test 'put' operation failed.\n"); + GNUNET_free (crc); + ok = 1; + } +} + + + +static int +check () +{ + struct GNUNET_OS_Process *proc; + char cfg_name[128]; + + char *const argv[] = { + "test-datastore-api-management", + "-c", + cfg_name, +#if VERBOSE + "-L", "DEBUG", +#endif + NULL + }; + struct GNUNET_GETOPT_CommandLineOption options[] = { + GNUNET_GETOPT_OPTION_END + }; + GNUNET_snprintf (cfg_name, sizeof (cfg_name), + "test_datastore_api_data_%s.conf", plugin_name); + proc = + GNUNET_OS_start_process (GNUNET_YES, NULL, NULL, "gnunet-service-arm", + "gnunet-service-arm", +#if VERBOSE + "-L", "DEBUG", +#endif + "-c", cfg_name, NULL); + GNUNET_assert (NULL != proc); + GNUNET_PROGRAM_run ((sizeof (argv) / sizeof (char *)) - 1, argv, + "test-datastore-api-management", "nohelp", options, &run, + NULL); + sleep (1); /* give datastore chance to process 'DROP' request */ + if (0 != GNUNET_OS_process_kill (proc, SIGTERM)) + { + GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING, "kill"); + ok = 1; + } + GNUNET_OS_process_wait (proc); + GNUNET_OS_process_close (proc); + proc = NULL; + if (ok != 0) + FPRINTF (stderr, "Missed some testcases: %u\n", ok); + return ok; +} + +int +main (int argc, char *argv[]) +{ + int ret; + + char *pos; + char dir_name[128]; + + sleep (1); + /* determine name of plugin to use */ + plugin_name = argv[0]; + while (NULL != (pos = strstr (plugin_name, "_"))) + plugin_name = pos + 1; + if (NULL != (pos = strstr (plugin_name, "."))) + pos[0] = 0; + else + pos = (char *) plugin_name; + + GNUNET_snprintf (dir_name, sizeof (dir_name), "/tmp/test-gnunet-datastore-%s", + plugin_name); + GNUNET_DISK_directory_remove (dir_name); + GNUNET_log_setup ("test-datastore-api-management", +#if VERBOSE + "DEBUG", +#else + "WARNING", +#endif + NULL); + ret = check (); + if (pos != plugin_name) + pos[0] = '.'; + GNUNET_DISK_directory_remove (dir_name); + return ret; +} + +/* end of test_datastore_api_management.c */ diff --git a/src/datastore/test_defaults.conf b/src/datastore/test_defaults.conf new file mode 100644 index 0000000..113d6bb --- /dev/null +++ b/src/datastore/test_defaults.conf @@ -0,0 +1,30 @@ +[datastore] +PORT = 22654 +QUOTA = 1 MB + +[dht] +AUTOSTART = NO + +[dns] +AUTOSTART = NO + +[mesh] +AUTOSTART = NO + +[nse] +AUTOSTART = NO + +[dv] +AUTOSTART = NO + +[chat] +AUTOSTART = NO + +[vpn] +AUTOSTART = NO + +[gns] +AUTOSTART = NO + +[dv] +AUTOSTART = NO diff --git a/src/datastore/test_plugin_datastore.c b/src/datastore/test_plugin_datastore.c new file mode 100644 index 0000000..16fe40c --- /dev/null +++ b/src/datastore/test_plugin_datastore.c @@ -0,0 +1,421 @@ +/* + This file is part of GNUnet. + (C) 2011 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ +/* + * @file test_plugin_datastore.c + * @brief Test database plugin directly, calling each API function once + * @author Christian Grothoff + */ + +#include "platform.h" +#include "gnunet_util_lib.h" +#include "gnunet_protocols.h" +#include "gnunet_datastore_plugin.h" + +#define VERBOSE GNUNET_NO + +/** + * Number of put operations to perform. + */ +#define PUT_10 10 + +static unsigned long long stored_bytes; + +static unsigned long long stored_entries; + +static unsigned long long stored_ops; + +static const char *plugin_name; + +static int ok; + +enum RunPhase +{ + RP_ERROR = 0, + RP_PUT, + RP_GET, + RP_UPDATE, + RP_ITER_ZERO, + RP_REPL_GET, + RP_EXPI_GET, + RP_DROP +}; + + +struct CpsRunContext +{ + const struct GNUNET_CONFIGURATION_Handle *cfg; + struct GNUNET_DATASTORE_PluginFunctions *api; + enum RunPhase phase; + unsigned int cnt; + unsigned int i; + uint64_t offset; +}; + + +/** + * Function called by plugins to notify us about a + * change in their disk utilization. + * + * @param cls closure (NULL) + * @param delta change in disk utilization, + * 0 for "reset to empty" + */ +static void +disk_utilization_change_cb (void *cls, int delta) +{ + /* do nothing */ +} + + +static void +gen_key (int i, GNUNET_HashCode * key) +{ + memset (key, 0, sizeof (GNUNET_HashCode)); + key->bits[0] = (unsigned int) i; + GNUNET_CRYPTO_hash (key, sizeof (GNUNET_HashCode), key); +} + + +static void +put_value (struct GNUNET_DATASTORE_PluginFunctions *api, int i, int k) +{ + char value[65536]; + size_t size; + GNUNET_HashCode key; + char *msg; + unsigned int prio; + + /* most content is 32k */ + size = 32 * 1024; + + if (GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 16) == 0) /* but some of it is less! */ + size = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 32 * 1024); + size = size - (size & 7); /* always multiple of 8 */ + + /* generate random key */ + gen_key (i, &key); + memset (value, i, size); + if (i > 255) + memset (value, i - 255, size / 2); + value[0] = k; + msg = NULL; + prio = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 100); +#if VERBOSE + FPRINTF (stderr, "putting type %u, anon %u under key %s\n", i + 1, i, + GNUNET_h2s (&key)); +#endif + if (GNUNET_OK != api->put (api->cls, &key, size, value, i + 1 /* type */ , + prio, i /* anonymity */ , + 0 /* replication */ , + GNUNET_TIME_relative_to_absolute + (GNUNET_TIME_relative_multiply + (GNUNET_TIME_UNIT_MILLISECONDS, + 60 * 60 * 60 * 1000 + + GNUNET_CRYPTO_random_u32 + (GNUNET_CRYPTO_QUALITY_WEAK, 1000))), &msg)) + { + FPRINTF (stderr, "ERROR: `%s'\n", msg); + GNUNET_free_non_null (msg); + return; + } + stored_bytes += size; + stored_ops++; + stored_entries++; +} + + +static void +test (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc); + + +static uint64_t guid; + + +static int +iterate_one_shot (void *cls, const GNUNET_HashCode * key, uint32_t size, + const void *data, enum GNUNET_BLOCK_Type type, + uint32_t priority, uint32_t anonymity, + struct GNUNET_TIME_Absolute expiration, uint64_t uid) +{ + struct CpsRunContext *crc = cls; + + GNUNET_assert (key != NULL); + guid = uid; + crc->phase++; +#if VERBOSE + FPRINTF (stderr, + "Found result type=%u, priority=%u, size=%u, expire=%llu, key %s\n", + type, priority, size, (unsigned long long) expiration.abs_value, + GNUNET_h2s (key)); +#endif + GNUNET_SCHEDULER_add_now (&test, crc); + return GNUNET_OK; +} + + +/** + * Function called when the service shuts + * down. Unloads our datastore plugin. + * + * @param api api to unload + * @param cfg configuration to use + */ +static void +unload_plugin (struct GNUNET_DATASTORE_PluginFunctions *api, + const struct GNUNET_CONFIGURATION_Handle *cfg) +{ + char *name; + char *libname; + + if (GNUNET_OK != + GNUNET_CONFIGURATION_get_value_string (cfg, "DATASTORE", "DATABASE", + &name)) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + _("No `%s' specified for `%s' in configuration!\n"), "DATABASE", + "DATASTORE"); + return; + } + GNUNET_asprintf (&libname, "libgnunet_plugin_datastore_%s", name); + GNUNET_break (NULL == GNUNET_PLUGIN_unload (libname, api)); + GNUNET_free (libname); + GNUNET_free (name); +} + + + +/** + * Last task run during shutdown. Disconnects us from + * the transport and core. + */ +static void +cleaning_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct CpsRunContext *crc = cls; + + unload_plugin (crc->api, crc->cfg); + GNUNET_free (crc); +} + + +static void +test (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct CpsRunContext *crc = cls; + int j; + unsigned long long os; + unsigned long long cs; + GNUNET_HashCode key; + + if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN)) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Test aborted.\n"); + crc->phase = RP_ERROR; + } +#if VERBOSE + FPRINTF (stderr, "In phase %d, iteration %u\n", crc->phase, crc->cnt); +#endif + switch (crc->phase) + { + case RP_ERROR: + ok = 1; + GNUNET_break (0); + crc->api->drop (crc->api->cls); + GNUNET_SCHEDULER_add_now (&cleaning_task, crc); + break; + case RP_PUT: + os = 0; + for (j = 0; j < PUT_10; j++) + { + put_value (crc->api, j, crc->i); + cs = crc->api->estimate_size (crc->api->cls); + GNUNET_assert (os <= cs); + os = cs; + } + crc->phase++; + GNUNET_SCHEDULER_add_now (&test, crc); + break; + case RP_GET: + if (crc->cnt == 1) + { + crc->cnt = 0; + crc->phase++; + GNUNET_SCHEDULER_add_now (&test, crc); + break; + } + gen_key (5, &key); + crc->api->get_key (crc->api->cls, crc->offset++, &key, NULL, + GNUNET_BLOCK_TYPE_ANY, &iterate_one_shot, crc); + break; + case RP_UPDATE: + GNUNET_assert (GNUNET_OK == + crc->api->update (crc->api->cls, guid, 1, + GNUNET_TIME_UNIT_ZERO_ABS, NULL)); + crc->phase++; + GNUNET_SCHEDULER_add_now (&test, crc); + break; + + case RP_ITER_ZERO: + if (crc->cnt == 1) + { + crc->cnt = 0; + crc->phase++; + GNUNET_SCHEDULER_add_now (&test, crc); + break; + } + crc->api->get_zero_anonymity (crc->api->cls, 0, 1, &iterate_one_shot, crc); + break; + case RP_REPL_GET: + crc->api->get_replication (crc->api->cls, &iterate_one_shot, crc); + break; + case RP_EXPI_GET: + crc->api->get_expiration (crc->api->cls, &iterate_one_shot, crc); + break; + case RP_DROP: + crc->api->drop (crc->api->cls); + GNUNET_SCHEDULER_add_now (&cleaning_task, crc); + break; + } +} + + +/** + * Load the datastore plugin. + */ +static struct GNUNET_DATASTORE_PluginFunctions * +load_plugin (const struct GNUNET_CONFIGURATION_Handle *cfg) +{ + static struct GNUNET_DATASTORE_PluginEnvironment env; + struct GNUNET_DATASTORE_PluginFunctions *ret; + char *name; + char *libname; + + if (GNUNET_OK != + GNUNET_CONFIGURATION_get_value_string (cfg, "DATASTORE", "DATABASE", + &name)) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + _("No `%s' specified for `%s' in configuration!\n"), "DATABASE", + "DATASTORE"); + return NULL; + } + env.cfg = cfg; + env.duc = &disk_utilization_change_cb; + env.cls = NULL; + GNUNET_log (GNUNET_ERROR_TYPE_INFO, _("Loading `%s' datastore plugin\n"), + name); + GNUNET_asprintf (&libname, "libgnunet_plugin_datastore_%s", name); + if (NULL == (ret = GNUNET_PLUGIN_load (libname, &env))) + { + FPRINTF (stderr, "Failed to load plugin `%s'!\n", name); + return NULL; + } + GNUNET_free (libname); + GNUNET_free (name); + return ret; +} + + +static void +run (void *cls, char *const *args, const char *cfgfile, + const struct GNUNET_CONFIGURATION_Handle *c) +{ + struct GNUNET_DATASTORE_PluginFunctions *api; + struct CpsRunContext *crc; + + api = load_plugin (c); + if (api == NULL) + { + FPRINTF (stderr, + "%s", "Could not initialize plugin, assuming database not configured. Test not run!\n"); + return; + } + crc = GNUNET_malloc (sizeof (struct CpsRunContext)); + crc->api = api; + crc->cfg = c; + crc->phase = RP_PUT; + GNUNET_SCHEDULER_add_now (&test, crc); +} + + +static int +check () +{ + char cfg_name[128]; + + char *const argv[] = { + "test-plugin-datastore", + "-c", + cfg_name, +#if VERBOSE + "-L", "DEBUG", +#endif + NULL + }; + struct GNUNET_GETOPT_CommandLineOption options[] = { + GNUNET_GETOPT_OPTION_END + }; + + GNUNET_snprintf (cfg_name, sizeof (cfg_name), + "test_plugin_datastore_data_%s.conf", plugin_name); + GNUNET_PROGRAM_run ((sizeof (argv) / sizeof (char *)) - 1, argv, + "test-plugin-datastore", "nohelp", options, &run, NULL); + if (ok != 0) + FPRINTF (stderr, "Missed some testcases: %u\n", ok); + return ok; +} + + +int +main (int argc, char *argv[]) +{ + int ret; + char *pos; + char dir_name[128]; + + sleep (1); + /* determine name of plugin to use */ + plugin_name = argv[0]; + while (NULL != (pos = strstr (plugin_name, "_"))) + plugin_name = pos + 1; + if (NULL != (pos = strstr (plugin_name, "."))) + pos[0] = 0; + else + pos = (char *) plugin_name; + + GNUNET_snprintf (dir_name, sizeof (dir_name), + "/tmp/test-gnunet-datastore-plugin-%s", plugin_name); + GNUNET_DISK_directory_remove (dir_name); + GNUNET_log_setup ("test-plugin-datastore", +#if VERBOSE + "DEBUG", +#else + "WARNING", +#endif + NULL); + ret = check (); + if (pos != plugin_name) + pos[0] = '.'; + GNUNET_DISK_directory_remove (dir_name); + + return ret; +} + +/* end of test_plugin_datastore.c */ diff --git a/src/datastore/test_plugin_datastore_data_mysql.conf b/src/datastore/test_plugin_datastore_data_mysql.conf new file mode 100644 index 0000000..957818a --- /dev/null +++ b/src/datastore/test_plugin_datastore_data_mysql.conf @@ -0,0 +1,11 @@ +@INLINE@ test_defaults.conf +[PATHS] +SERVICEHOME = /tmp/test-gnunet-datastore-plugin-mysql/ +DEFAULTCONFIG = test_plugin_datastore_data_mysql.conf + +[datastore] +DATABASE = mysql + +[datastore-mysql] +DATABASE = gnunetcheck + diff --git a/src/datastore/test_plugin_datastore_data_postgres.conf b/src/datastore/test_plugin_datastore_data_postgres.conf new file mode 100644 index 0000000..d796496 --- /dev/null +++ b/src/datastore/test_plugin_datastore_data_postgres.conf @@ -0,0 +1,11 @@ +@INLINE@ test_defaults.conf +[PATHS] +SERVICEHOME = /tmp/test-gnunet-datastore-plugin-postgres/ +DEFAULTCONFIG = test_plugin_datastore_data_postgres.conf + +[datastore] +DATABASE = postgres + +[datastore-postgres] +CONFIG = dbname=gnunetcheck + diff --git a/src/datastore/test_plugin_datastore_data_sqlite.conf b/src/datastore/test_plugin_datastore_data_sqlite.conf new file mode 100644 index 0000000..f1fcf5f --- /dev/null +++ b/src/datastore/test_plugin_datastore_data_sqlite.conf @@ -0,0 +1,5 @@ +@INLINE@ test_defaults.conf +[PATHS] +SERVICEHOME = /tmp/test-gnunet-datastore-plugin-sqlite/ +DEFAULTCONFIG = test_plugin_datastore_data_sqlite.conf + |