diff options
| author | Bertrand Marc <beberking@gmail.com> | 2012-05-02 21:43:37 +0200 |
|---|---|---|
| committer | Bertrand Marc <beberking@gmail.com> | 2012-05-02 21:43:37 +0200 |
| commit | 2b81464a43485fcc8ce079fafdee7b7a171835f4 (patch) | |
| tree | 394774c0f735199b57d51a2d3840356317853fe1 /src/fs | |
Imported Upstream version 0.9.2upstream/0.9.2
Diffstat (limited to 'src/fs')
94 files changed, 39546 insertions, 0 deletions
diff --git a/src/fs/Makefile.am b/src/fs/Makefile.am new file mode 100644 index 0000000..0de739d --- /dev/null +++ b/src/fs/Makefile.am @@ -0,0 +1,466 @@ +INCLUDES = -I$(top_srcdir)/src/include + +if MINGW + WINFLAGS = -Wl,--no-undefined -Wl,--export-all-symbols +endif + +if USE_COVERAGE + AM_CFLAGS = --coverage -O0 + XLIB = -lgcov +endif + +pkgcfgdir= $(pkgdatadir)/config.d/ + +pkgcfg_DATA = \ + fs.conf + +plugindir = $(libdir)/gnunet + + +lib_LTLIBRARIES = libgnunetfs.la + +plugin_LTLIBRARIES = \ + libgnunet_plugin_block_fs.la + +noinst_LIBRARIES = libgnunetfstest.a + +libgnunetfs_la_SOURCES = \ + fs_api.c fs_api.h fs.h \ + fs_directory.c \ + fs_dirmetascan.c \ + fs_download.c \ + fs_file_information.c \ + fs_getopt.c \ + fs_list_indexed.c \ + fs_publish.c \ + fs_publish_ksk.c \ + fs_misc.c \ + fs_namespace.c \ + fs_namespace_advertise.c \ + fs_search.c \ + fs_sharetree.c \ + fs_tree.c fs_tree.h \ + fs_unindex.c \ + fs_uri.c + +libgnunetfs_la_LIBADD = \ + $(top_builddir)/src/datastore/libgnunetdatastore.la \ + $(top_builddir)/src/util/libgnunetutil.la \ + $(GN_LIBINTL) $(XLIB) -lunistring -lextractor + +libgnunetfs_la_LDFLAGS = \ + $(GN_LIB_LDFLAGS) $(WINFLAGS) \ + -version-info 2:0:0 + + +libgnunetfstest_a_SOURCES = \ + fs_test_lib.c fs_test_lib.h + +libgnunetfstest_a_LIBADD = \ + $(top_builddir)/src/testing/libgnunettesting.la + +bin_PROGRAMS = \ + gnunet-directory \ + gnunet-download \ + gnunet-publish \ + gnunet-helper-fs-publish \ + gnunet-pseudonym \ + gnunet-search \ + gnunet-service-fs \ + gnunet-fs \ + gnunet-unindex + +bin_SCRIPTS = \ + gnunet-download-manager.scm + +gnunet_directory_SOURCES = \ + gnunet-directory.c +gnunet_directory_LDADD = \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la \ + -lextractor \ + $(GN_LIBINTL) +gnunet_directory_DEPENDENCIES = \ + libgnunetfs.la + +gnunet_fs_SOURCES = \ + gnunet-fs.c +gnunet_fs_LDADD = \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la \ + -lextractor \ + $(GN_LIBINTL) +gnunet_fs_DEPENDENCIES = \ + libgnunetfs.la + +gnunet_download_SOURCES = \ + gnunet-download.c +gnunet_download_LDADD = \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la \ + $(GN_LIBINTL) +gnunet_download_DEPENDENCIES = \ + libgnunetfs.la + +gnunet_publish_SOURCES = \ + gnunet-publish.c +gnunet_publish_LDADD = \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la \ + -lextractor \ + $(GN_LIBINTL) +gnunet_publish_DEPENDENCIES = \ + libgnunetfs.la + +gnunet_helper_fs_publish_SOURCES = \ + gnunet-helper-fs-publish.c +gnunet_helper_fs_publish_LDADD = \ + $(top_builddir)/src/util/libgnunetutil.la \ + -lextractor \ + $(GN_LIBINTL) +gnunet_helper_fs_publish_DEPENDENCIES = \ + libgnunetfs.la + +gnunet_pseudonym_SOURCES = \ + gnunet-pseudonym.c +gnunet_pseudonym_LDADD = \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la \ + -lextractor \ + $(GN_LIBINTL) +gnunet_pseudonym_DEPENDENCIES = \ + libgnunetfs.la + +gnunet_search_SOURCES = \ + gnunet-search.c +gnunet_search_LDADD = \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la \ + -lextractor \ + $(GN_LIBINTL) +gnunet_search_DEPENDENCIES = \ + libgnunetfs.la + +gnunet_service_fs_SOURCES = \ + gnunet-service-fs.c gnunet-service-fs.h \ + gnunet-service-fs_cp.c gnunet-service-fs_cp.h \ + gnunet-service-fs_indexing.c gnunet-service-fs_indexing.h \ + gnunet-service-fs_lc.c gnunet-service-fs_lc.h \ + gnunet-service-fs_pe.c gnunet-service-fs_pe.h \ + gnunet-service-fs_pr.c gnunet-service-fs_pr.h \ + gnunet-service-fs_push.c gnunet-service-fs_push.h \ + gnunet-service-fs_put.c gnunet-service-fs_put.h +gnunet_service_fs_LDADD = \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/dht/libgnunetdht.la \ + $(top_builddir)/src/block/libgnunetblock.la \ + $(top_builddir)/src/datastore/libgnunetdatastore.la \ + $(top_builddir)/src/statistics/libgnunetstatistics.la \ + $(top_builddir)/src/ats/libgnunetats.la \ + $(top_builddir)/src/core/libgnunetcore.la \ + $(top_builddir)/src/util/libgnunetutil.la \ + $(GN_LIBINTL) -lm +gnunet_service_fs_DEPENDENCIES = \ + libgnunetfs.la + +gnunet_unindex_SOURCES = \ + gnunet-unindex.c +gnunet_unindex_LDADD = \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la \ + $(GN_LIBINTL) +gnunet_unindex_DEPENDENCIES = \ + libgnunetfs.la + + +libgnunet_plugin_block_fs_la_SOURCES = \ + plugin_block_fs.c +libgnunet_plugin_block_fs_la_LIBADD = \ + $(top_builddir)/src/block/libgnunetblock.la \ + $(top_builddir)/src/util/libgnunetutil.la +libgnunet_plugin_block_fs_la_LDFLAGS = \ + $(GN_PLUGIN_LDFLAGS) +libgnunet_plugin_block_fs_la_DEPENDENCIES = \ + $(top_builddir)/src/block/libgnunetblock.la + + + +if HAVE_BENCHMARKS + FS_BENCHMARKS = \ + perf_gnunet_service_fs_p2p \ + perf_gnunet_service_fs_p2p_dht \ + perf_gnunet_service_fs_p2p_index \ + perf_gnunet_service_fs_p2p_trust +endif + +check_PROGRAMS = \ + test_fs_directory \ + test_fs_download \ + test_fs_download_indexed \ + test_fs_download_persistence \ + test_fs_file_information \ + test_fs_getopt \ + test_fs_list_indexed \ + test_fs_namespace \ + test_fs_namespace_list_updateable \ + test_fs_publish \ + test_fs_publish_persistence \ + test_fs_search \ + test_fs_search_persistence \ + test_fs_start_stop \ + test_fs_test_lib \ + test_fs_unindex \ + test_fs_unindex_persistence \ + test_fs_uri \ + test_gnunet_service_fs_migration \ + test_gnunet_service_fs_p2p \ + $(FS_BENCHMARKS) + + +if HAVE_PYTHON_PEXPECT +check_SCRIPTS = \ + test_gnunet_fs_psd.py \ + test_gnunet_fs_rec.py \ + test_gnunet_fs_idx.py \ + test_gnunet_fs_ns.py +endif + + +if ENABLE_TEST_RUN +TESTS = \ + test_fs_directory \ + test_fs_download \ + test_fs_download_indexed \ + test_fs_download_persistence \ + test_fs_file_information \ + test_fs_list_indexed \ + test_fs_namespace \ + test_fs_namespace_list_updateable \ + test_fs_publish \ + test_fs_publish_persistence \ + test_fs_search \ + test_fs_search_persistence \ + test_fs_start_stop \ + test_fs_unindex \ + test_fs_unindex_persistence \ + test_fs_uri \ + test_fs_test_lib \ + test_gnunet_service_fs_migration \ + test_gnunet_service_fs_p2p \ + perf_gnunet_service_fs_p2p \ + perf_gnunet_service_fs_p2p_index \ + perf_gnunet_service_fs_p2p_trust \ + $(check_SCRIPTS) +endif + + + + +test_fs_directory_SOURCES = \ + test_fs_directory.c +test_fs_directory_LDADD = \ + -lextractor \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la + +test_fs_download_SOURCES = \ + test_fs_download.c +test_fs_download_LDADD = \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la + +test_fs_download_indexed_SOURCES = \ + test_fs_download_indexed.c +test_fs_download_indexed_LDADD = \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la + +test_fs_download_persistence_SOURCES = \ + test_fs_download_persistence.c +test_fs_download_persistence_LDADD = \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la + +test_fs_file_information_SOURCES = \ + test_fs_file_information.c +test_fs_file_information_LDADD = \ + -lextractor \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la + +test_fs_getopt_SOURCES = \ + test_fs_getopt.c +test_fs_getopt_LDADD = \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la + +test_fs_list_indexed_SOURCES = \ + test_fs_list_indexed.c +test_fs_list_indexed_LDADD = \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la + +test_fs_namespace_SOURCES = \ + test_fs_namespace.c +test_fs_namespace_LDADD = \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la + +test_fs_namespace_list_updateable_SOURCES = \ + test_fs_namespace_list_updateable.c +test_fs_namespace_list_updateable_LDADD = \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la + +test_fs_publish_SOURCES = \ + test_fs_publish.c +test_fs_publish_LDADD = \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la + +test_fs_publish_persistence_SOURCES = \ + test_fs_publish_persistence.c +test_fs_publish_persistence_LDADD = \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la + +test_fs_search_SOURCES = \ + test_fs_search.c +test_fs_search_LDADD = $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la + +test_fs_search_persistence_SOURCES = \ + test_fs_search_persistence.c +test_fs_search_persistence_LDADD = $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la + +test_fs_start_stop_SOURCES = \ + test_fs_start_stop.c +test_fs_start_stop_LDADD = \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la + +test_fs_unindex_SOURCES = \ + test_fs_unindex.c +test_fs_unindex_LDADD = \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la + +test_fs_unindex_persistence_SOURCES = \ + test_fs_unindex_persistence.c +test_fs_unindex_persistence_LDADD = \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la + +test_fs_uri_SOURCES = \ + test_fs_uri.c +test_fs_uri_LDADD = \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la + +test_fs_test_lib_SOURCES = \ + test_fs_test_lib.c +test_fs_test_lib_LDADD = \ + $(top_builddir)/src/fs/libgnunetfstest.a \ + $(top_builddir)/src/testing/libgnunettesting.la \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la + +test_gnunet_service_fs_p2p_SOURCES = \ + test_gnunet_service_fs_p2p.c +test_gnunet_service_fs_p2p_LDADD = \ + $(top_builddir)/src/fs/libgnunetfstest.a \ + $(top_builddir)/src/testing/libgnunettesting.la \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la + +test_gnunet_service_fs_migration_SOURCES = \ + test_gnunet_service_fs_migration.c +test_gnunet_service_fs_migration_LDADD = \ + $(top_builddir)/src/fs/libgnunetfstest.a \ + $(top_builddir)/src/testing/libgnunettesting.la \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la + +perf_gnunet_service_fs_p2p_SOURCES = \ + perf_gnunet_service_fs_p2p.c +perf_gnunet_service_fs_p2p_LDADD = \ + $(top_builddir)/src/fs/libgnunetfstest.a \ + $(top_builddir)/src/statistics/libgnunetstatistics.la \ + $(top_builddir)/src/testing/libgnunettesting.la \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la + +perf_gnunet_service_fs_p2p_index_SOURCES = \ + perf_gnunet_service_fs_p2p.c +perf_gnunet_service_fs_p2p_index_LDADD = \ + $(top_builddir)/src/fs/libgnunetfstest.a \ + $(top_builddir)/src/statistics/libgnunetstatistics.la \ + $(top_builddir)/src/testing/libgnunettesting.la \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la + +perf_gnunet_service_fs_p2p_dht_SOURCES = \ + perf_gnunet_service_fs_p2p.c +perf_gnunet_service_fs_p2p_dht_LDADD = \ + $(top_builddir)/src/fs/libgnunetfstest.a \ + $(top_builddir)/src/statistics/libgnunetstatistics.la \ + $(top_builddir)/src/testing/libgnunettesting.la \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la + +perf_gnunet_service_fs_p2p_trust_SOURCES = \ + perf_gnunet_service_fs_p2p_trust.c +perf_gnunet_service_fs_p2p_trust_LDADD = \ + $(top_builddir)/src/fs/libgnunetfstest.a \ + $(top_builddir)/src/statistics/libgnunetstatistics.la \ + $(top_builddir)/src/testing/libgnunettesting.la \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la + + +do_subst = $(SED) -e 's,[@]PYTHON[@],$(PYTHON),g' + +test_gnunet_fs_psd.py: test_gnunet_fs_psd.py.in Makefile + $(do_subst) < $(srcdir)/test_gnunet_fs_psd.py.in > test_gnunet_fs_psd.py + chmod +x test_gnunet_fs_psd.py + +test_gnunet_fs_rec.py: test_gnunet_fs_rec.py.in Makefile + $(do_subst) < $(srcdir)/test_gnunet_fs_rec.py.in > test_gnunet_fs_rec.py + chmod +x test_gnunet_fs_rec.py + +test_gnunet_fs_ns.py: test_gnunet_fs_ns.py.in Makefile + $(do_subst) < $(srcdir)/test_gnunet_fs_ns.py.in > test_gnunet_fs_ns.py + chmod +x test_gnunet_fs_ns.py + +test_gnunet_fs_idx.py: test_gnunet_fs_idx.py.in Makefile + $(do_subst) < $(srcdir)/test_gnunet_fs_idx.py.in > test_gnunet_fs_idx.py + chmod +x test_gnunet_fs_idx.py + + +EXTRA_DIST = \ + test_fs_defaults.conf \ + fs_test_lib_data.conf \ + test_fs_data.conf \ + test_fs_download_data.conf \ + test_fs_file_information_data.conf \ + fs_test_lib_data.conf \ + test_fs_list_indexed_data.conf \ + test_fs_namespace_data.conf \ + test_fs_publish_data.conf \ + test_fs_search_data.conf \ + test_fs_unindex_data.conf \ + test_fs_uri_data.conf \ + test_gnunet_service_fs_migration_data.conf \ + test_gnunet_fs_idx_data.conf \ + test_gnunet_fs_ns_data.conf \ + test_gnunet_fs_psd_data.conf \ + test_gnunet_fs_rec_data.conf \ + test_gnunet_fs_rec_data.tgz \ + test_gnunet_fs_psd.py.in \ + test_gnunet_fs_rec.py.in \ + test_gnunet_fs_ns.py.in \ + test_gnunet_fs_idx.py.in \ + $(bin_SCRIPTS) + +CLEANFILES = $(check_SCRIPTS) diff --git a/src/fs/Makefile.in b/src/fs/Makefile.in new file mode 100644 index 0000000..cbc844e --- /dev/null +++ b/src/fs/Makefile.in @@ -0,0 +1,1788 @@ +# Makefile.in generated by automake 1.11.1 from Makefile.am. +# @configure_input@ + +# Copyright (C) 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, +# 2003, 2004, 2005, 2006, 2007, 2008, 2009 Free Software Foundation, +# Inc. +# This Makefile.in is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY, to the extent permitted by law; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. + +@SET_MAKE@ + + + + + +VPATH = @srcdir@ +pkgdatadir = $(datadir)/@PACKAGE@ +pkgincludedir = $(includedir)/@PACKAGE@ +pkglibdir = $(libdir)/@PACKAGE@ +pkglibexecdir = $(libexecdir)/@PACKAGE@ +am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd +install_sh_DATA = $(install_sh) -c -m 644 +install_sh_PROGRAM = $(install_sh) -c +install_sh_SCRIPT = $(install_sh) -c +INSTALL_HEADER = $(INSTALL_DATA) +transform = $(program_transform_name) +NORMAL_INSTALL = : +PRE_INSTALL = : +POST_INSTALL = : +NORMAL_UNINSTALL = : +PRE_UNINSTALL = : +POST_UNINSTALL = : +build_triplet = @build@ +host_triplet = @host@ +target_triplet = @target@ +bin_PROGRAMS = gnunet-directory$(EXEEXT) gnunet-download$(EXEEXT) \ + gnunet-publish$(EXEEXT) gnunet-helper-fs-publish$(EXEEXT) \ + gnunet-pseudonym$(EXEEXT) gnunet-search$(EXEEXT) \ + gnunet-service-fs$(EXEEXT) gnunet-fs$(EXEEXT) \ + gnunet-unindex$(EXEEXT) +check_PROGRAMS = test_fs_directory$(EXEEXT) test_fs_download$(EXEEXT) \ + test_fs_download_indexed$(EXEEXT) \ + test_fs_download_persistence$(EXEEXT) \ + test_fs_file_information$(EXEEXT) test_fs_getopt$(EXEEXT) \ + test_fs_list_indexed$(EXEEXT) test_fs_namespace$(EXEEXT) \ + test_fs_namespace_list_updateable$(EXEEXT) \ + test_fs_publish$(EXEEXT) test_fs_publish_persistence$(EXEEXT) \ + test_fs_search$(EXEEXT) test_fs_search_persistence$(EXEEXT) \ + test_fs_start_stop$(EXEEXT) test_fs_test_lib$(EXEEXT) \ + test_fs_unindex$(EXEEXT) test_fs_unindex_persistence$(EXEEXT) \ + test_fs_uri$(EXEEXT) test_gnunet_service_fs_migration$(EXEEXT) \ + test_gnunet_service_fs_p2p$(EXEEXT) $(am__EXEEXT_1) +@ENABLE_TEST_RUN_TRUE@TESTS = test_fs_directory$(EXEEXT) \ +@ENABLE_TEST_RUN_TRUE@ test_fs_download$(EXEEXT) \ +@ENABLE_TEST_RUN_TRUE@ test_fs_download_indexed$(EXEEXT) \ +@ENABLE_TEST_RUN_TRUE@ test_fs_download_persistence$(EXEEXT) \ +@ENABLE_TEST_RUN_TRUE@ test_fs_file_information$(EXEEXT) \ +@ENABLE_TEST_RUN_TRUE@ test_fs_list_indexed$(EXEEXT) \ +@ENABLE_TEST_RUN_TRUE@ test_fs_namespace$(EXEEXT) \ +@ENABLE_TEST_RUN_TRUE@ test_fs_namespace_list_updateable$(EXEEXT) \ +@ENABLE_TEST_RUN_TRUE@ test_fs_publish$(EXEEXT) \ +@ENABLE_TEST_RUN_TRUE@ test_fs_publish_persistence$(EXEEXT) \ +@ENABLE_TEST_RUN_TRUE@ test_fs_search$(EXEEXT) \ +@ENABLE_TEST_RUN_TRUE@ test_fs_search_persistence$(EXEEXT) \ +@ENABLE_TEST_RUN_TRUE@ test_fs_start_stop$(EXEEXT) \ +@ENABLE_TEST_RUN_TRUE@ test_fs_unindex$(EXEEXT) \ +@ENABLE_TEST_RUN_TRUE@ test_fs_unindex_persistence$(EXEEXT) \ +@ENABLE_TEST_RUN_TRUE@ test_fs_uri$(EXEEXT) \ +@ENABLE_TEST_RUN_TRUE@ test_fs_test_lib$(EXEEXT) \ +@ENABLE_TEST_RUN_TRUE@ test_gnunet_service_fs_migration$(EXEEXT) \ +@ENABLE_TEST_RUN_TRUE@ test_gnunet_service_fs_p2p$(EXEEXT) \ +@ENABLE_TEST_RUN_TRUE@ perf_gnunet_service_fs_p2p$(EXEEXT) \ +@ENABLE_TEST_RUN_TRUE@ perf_gnunet_service_fs_p2p_index$(EXEEXT) \ +@ENABLE_TEST_RUN_TRUE@ perf_gnunet_service_fs_p2p_trust$(EXEEXT) \ +@ENABLE_TEST_RUN_TRUE@ $(check_SCRIPTS) +subdir = src/fs +DIST_COMMON = $(srcdir)/Makefile.am $(srcdir)/Makefile.in \ + $(srcdir)/fs.conf.in +ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 +am__aclocal_m4_deps = $(top_srcdir)/m4/absolute-header.m4 \ + $(top_srcdir)/m4/align.m4 $(top_srcdir)/m4/argz.m4 \ + $(top_srcdir)/m4/gettext.m4 $(top_srcdir)/m4/iconv.m4 \ + $(top_srcdir)/m4/lib-ld.m4 $(top_srcdir)/m4/lib-link.m4 \ + $(top_srcdir)/m4/lib-prefix.m4 $(top_srcdir)/m4/libcurl.m4 \ + $(top_srcdir)/m4/libgcrypt.m4 $(top_srcdir)/m4/libtool.m4 \ + $(top_srcdir)/m4/libunistring.m4 $(top_srcdir)/m4/ltdl.m4 \ + $(top_srcdir)/m4/ltoptions.m4 $(top_srcdir)/m4/ltsugar.m4 \ + $(top_srcdir)/m4/ltversion.m4 $(top_srcdir)/m4/lt~obsolete.m4 \ + $(top_srcdir)/m4/nls.m4 $(top_srcdir)/m4/po.m4 \ + $(top_srcdir)/m4/progtest.m4 $(top_srcdir)/acinclude.m4 \ + $(top_srcdir)/configure.ac +am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \ + $(ACLOCAL_M4) +mkinstalldirs = $(install_sh) -d +CONFIG_HEADER = $(top_builddir)/gnunet_config.h +CONFIG_CLEAN_FILES = fs.conf +CONFIG_CLEAN_VPATH_FILES = +LIBRARIES = $(noinst_LIBRARIES) +ARFLAGS = cru +AM_V_AR = $(am__v_AR_$(V)) +am__v_AR_ = $(am__v_AR_$(AM_DEFAULT_VERBOSITY)) +am__v_AR_0 = @echo " AR " $@; +AM_V_at = $(am__v_at_$(V)) +am__v_at_ = $(am__v_at_$(AM_DEFAULT_VERBOSITY)) +am__v_at_0 = @ +libgnunetfstest_a_AR = $(AR) $(ARFLAGS) +libgnunetfstest_a_DEPENDENCIES = \ + $(top_builddir)/src/testing/libgnunettesting.la +am_libgnunetfstest_a_OBJECTS = fs_test_lib.$(OBJEXT) +libgnunetfstest_a_OBJECTS = $(am_libgnunetfstest_a_OBJECTS) +am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`; +am__vpath_adj = case $$p in \ + $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \ + *) f=$$p;; \ + esac; +am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`; +am__install_max = 40 +am__nobase_strip_setup = \ + srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'` +am__nobase_strip = \ + for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||" +am__nobase_list = $(am__nobase_strip_setup); \ + for p in $$list; do echo "$$p $$p"; done | \ + sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \ + $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \ + if (++n[$$2] == $(am__install_max)) \ + { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \ + END { for (dir in files) print dir, files[dir] }' +am__base_list = \ + sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \ + sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g' +am__installdirs = "$(DESTDIR)$(libdir)" "$(DESTDIR)$(plugindir)" \ + "$(DESTDIR)$(bindir)" "$(DESTDIR)$(bindir)" \ + "$(DESTDIR)$(pkgcfgdir)" +LTLIBRARIES = $(lib_LTLIBRARIES) $(plugin_LTLIBRARIES) +am_libgnunet_plugin_block_fs_la_OBJECTS = plugin_block_fs.lo +libgnunet_plugin_block_fs_la_OBJECTS = \ + $(am_libgnunet_plugin_block_fs_la_OBJECTS) +AM_V_lt = $(am__v_lt_$(V)) +am__v_lt_ = $(am__v_lt_$(AM_DEFAULT_VERBOSITY)) +am__v_lt_0 = --silent +libgnunet_plugin_block_fs_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CCLD) \ + $(AM_CFLAGS) $(CFLAGS) $(libgnunet_plugin_block_fs_la_LDFLAGS) \ + $(LDFLAGS) -o $@ +am__DEPENDENCIES_1 = +libgnunetfs_la_DEPENDENCIES = \ + $(top_builddir)/src/datastore/libgnunetdatastore.la \ + $(top_builddir)/src/util/libgnunetutil.la \ + $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_1) +am_libgnunetfs_la_OBJECTS = fs_api.lo fs_directory.lo \ + fs_dirmetascan.lo fs_download.lo fs_file_information.lo \ + fs_getopt.lo fs_list_indexed.lo fs_publish.lo \ + fs_publish_ksk.lo fs_misc.lo fs_namespace.lo \ + fs_namespace_advertise.lo fs_search.lo fs_sharetree.lo \ + fs_tree.lo fs_unindex.lo fs_uri.lo +libgnunetfs_la_OBJECTS = $(am_libgnunetfs_la_OBJECTS) +libgnunetfs_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CCLD) \ + $(AM_CFLAGS) $(CFLAGS) $(libgnunetfs_la_LDFLAGS) $(LDFLAGS) -o \ + $@ +@HAVE_BENCHMARKS_TRUE@am__EXEEXT_1 = \ +@HAVE_BENCHMARKS_TRUE@ perf_gnunet_service_fs_p2p$(EXEEXT) \ +@HAVE_BENCHMARKS_TRUE@ perf_gnunet_service_fs_p2p_dht$(EXEEXT) \ +@HAVE_BENCHMARKS_TRUE@ perf_gnunet_service_fs_p2p_index$(EXEEXT) \ +@HAVE_BENCHMARKS_TRUE@ perf_gnunet_service_fs_p2p_trust$(EXEEXT) +PROGRAMS = $(bin_PROGRAMS) +am_gnunet_directory_OBJECTS = gnunet-directory.$(OBJEXT) +gnunet_directory_OBJECTS = $(am_gnunet_directory_OBJECTS) +am_gnunet_download_OBJECTS = gnunet-download.$(OBJEXT) +gnunet_download_OBJECTS = $(am_gnunet_download_OBJECTS) +am_gnunet_fs_OBJECTS = gnunet-fs.$(OBJEXT) +gnunet_fs_OBJECTS = $(am_gnunet_fs_OBJECTS) +am_gnunet_helper_fs_publish_OBJECTS = \ + gnunet-helper-fs-publish.$(OBJEXT) +gnunet_helper_fs_publish_OBJECTS = \ + $(am_gnunet_helper_fs_publish_OBJECTS) +am_gnunet_pseudonym_OBJECTS = gnunet-pseudonym.$(OBJEXT) +gnunet_pseudonym_OBJECTS = $(am_gnunet_pseudonym_OBJECTS) +am_gnunet_publish_OBJECTS = gnunet-publish.$(OBJEXT) +gnunet_publish_OBJECTS = $(am_gnunet_publish_OBJECTS) +am_gnunet_search_OBJECTS = gnunet-search.$(OBJEXT) +gnunet_search_OBJECTS = $(am_gnunet_search_OBJECTS) +am_gnunet_service_fs_OBJECTS = gnunet-service-fs.$(OBJEXT) \ + gnunet-service-fs_cp.$(OBJEXT) \ + gnunet-service-fs_indexing.$(OBJEXT) \ + gnunet-service-fs_lc.$(OBJEXT) gnunet-service-fs_pe.$(OBJEXT) \ + gnunet-service-fs_pr.$(OBJEXT) \ + gnunet-service-fs_push.$(OBJEXT) \ + gnunet-service-fs_put.$(OBJEXT) +gnunet_service_fs_OBJECTS = $(am_gnunet_service_fs_OBJECTS) +am_gnunet_unindex_OBJECTS = gnunet-unindex.$(OBJEXT) +gnunet_unindex_OBJECTS = $(am_gnunet_unindex_OBJECTS) +am_perf_gnunet_service_fs_p2p_OBJECTS = \ + perf_gnunet_service_fs_p2p.$(OBJEXT) +perf_gnunet_service_fs_p2p_OBJECTS = \ + $(am_perf_gnunet_service_fs_p2p_OBJECTS) +perf_gnunet_service_fs_p2p_DEPENDENCIES = \ + $(top_builddir)/src/fs/libgnunetfstest.a \ + $(top_builddir)/src/statistics/libgnunetstatistics.la \ + $(top_builddir)/src/testing/libgnunettesting.la \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la +am_perf_gnunet_service_fs_p2p_dht_OBJECTS = \ + perf_gnunet_service_fs_p2p.$(OBJEXT) +perf_gnunet_service_fs_p2p_dht_OBJECTS = \ + $(am_perf_gnunet_service_fs_p2p_dht_OBJECTS) +perf_gnunet_service_fs_p2p_dht_DEPENDENCIES = \ + $(top_builddir)/src/fs/libgnunetfstest.a \ + $(top_builddir)/src/statistics/libgnunetstatistics.la \ + $(top_builddir)/src/testing/libgnunettesting.la \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la +am_perf_gnunet_service_fs_p2p_index_OBJECTS = \ + perf_gnunet_service_fs_p2p.$(OBJEXT) +perf_gnunet_service_fs_p2p_index_OBJECTS = \ + $(am_perf_gnunet_service_fs_p2p_index_OBJECTS) +perf_gnunet_service_fs_p2p_index_DEPENDENCIES = \ + $(top_builddir)/src/fs/libgnunetfstest.a \ + $(top_builddir)/src/statistics/libgnunetstatistics.la \ + $(top_builddir)/src/testing/libgnunettesting.la \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la +am_perf_gnunet_service_fs_p2p_trust_OBJECTS = \ + perf_gnunet_service_fs_p2p_trust.$(OBJEXT) +perf_gnunet_service_fs_p2p_trust_OBJECTS = \ + $(am_perf_gnunet_service_fs_p2p_trust_OBJECTS) +perf_gnunet_service_fs_p2p_trust_DEPENDENCIES = \ + $(top_builddir)/src/fs/libgnunetfstest.a \ + $(top_builddir)/src/statistics/libgnunetstatistics.la \ + $(top_builddir)/src/testing/libgnunettesting.la \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la +am_test_fs_directory_OBJECTS = test_fs_directory.$(OBJEXT) +test_fs_directory_OBJECTS = $(am_test_fs_directory_OBJECTS) +test_fs_directory_DEPENDENCIES = \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la +am_test_fs_download_OBJECTS = test_fs_download.$(OBJEXT) +test_fs_download_OBJECTS = $(am_test_fs_download_OBJECTS) +test_fs_download_DEPENDENCIES = $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la +am_test_fs_download_indexed_OBJECTS = \ + test_fs_download_indexed.$(OBJEXT) +test_fs_download_indexed_OBJECTS = \ + $(am_test_fs_download_indexed_OBJECTS) +test_fs_download_indexed_DEPENDENCIES = \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la +am_test_fs_download_persistence_OBJECTS = \ + test_fs_download_persistence.$(OBJEXT) +test_fs_download_persistence_OBJECTS = \ + $(am_test_fs_download_persistence_OBJECTS) +test_fs_download_persistence_DEPENDENCIES = \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la +am_test_fs_file_information_OBJECTS = \ + test_fs_file_information.$(OBJEXT) +test_fs_file_information_OBJECTS = \ + $(am_test_fs_file_information_OBJECTS) +test_fs_file_information_DEPENDENCIES = \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la +am_test_fs_getopt_OBJECTS = test_fs_getopt.$(OBJEXT) +test_fs_getopt_OBJECTS = $(am_test_fs_getopt_OBJECTS) +test_fs_getopt_DEPENDENCIES = $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la +am_test_fs_list_indexed_OBJECTS = test_fs_list_indexed.$(OBJEXT) +test_fs_list_indexed_OBJECTS = $(am_test_fs_list_indexed_OBJECTS) +test_fs_list_indexed_DEPENDENCIES = \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la +am_test_fs_namespace_OBJECTS = test_fs_namespace.$(OBJEXT) +test_fs_namespace_OBJECTS = $(am_test_fs_namespace_OBJECTS) +test_fs_namespace_DEPENDENCIES = \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la +am_test_fs_namespace_list_updateable_OBJECTS = \ + test_fs_namespace_list_updateable.$(OBJEXT) +test_fs_namespace_list_updateable_OBJECTS = \ + $(am_test_fs_namespace_list_updateable_OBJECTS) +test_fs_namespace_list_updateable_DEPENDENCIES = \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la +am_test_fs_publish_OBJECTS = test_fs_publish.$(OBJEXT) +test_fs_publish_OBJECTS = $(am_test_fs_publish_OBJECTS) +test_fs_publish_DEPENDENCIES = $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la +am_test_fs_publish_persistence_OBJECTS = \ + test_fs_publish_persistence.$(OBJEXT) +test_fs_publish_persistence_OBJECTS = \ + $(am_test_fs_publish_persistence_OBJECTS) +test_fs_publish_persistence_DEPENDENCIES = \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la +am_test_fs_search_OBJECTS = test_fs_search.$(OBJEXT) +test_fs_search_OBJECTS = $(am_test_fs_search_OBJECTS) +test_fs_search_DEPENDENCIES = $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la +am_test_fs_search_persistence_OBJECTS = \ + test_fs_search_persistence.$(OBJEXT) +test_fs_search_persistence_OBJECTS = \ + $(am_test_fs_search_persistence_OBJECTS) +test_fs_search_persistence_DEPENDENCIES = \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la +am_test_fs_start_stop_OBJECTS = test_fs_start_stop.$(OBJEXT) +test_fs_start_stop_OBJECTS = $(am_test_fs_start_stop_OBJECTS) +test_fs_start_stop_DEPENDENCIES = \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la +am_test_fs_test_lib_OBJECTS = test_fs_test_lib.$(OBJEXT) +test_fs_test_lib_OBJECTS = $(am_test_fs_test_lib_OBJECTS) +test_fs_test_lib_DEPENDENCIES = \ + $(top_builddir)/src/fs/libgnunetfstest.a \ + $(top_builddir)/src/testing/libgnunettesting.la \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la +am_test_fs_unindex_OBJECTS = test_fs_unindex.$(OBJEXT) +test_fs_unindex_OBJECTS = $(am_test_fs_unindex_OBJECTS) +test_fs_unindex_DEPENDENCIES = $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la +am_test_fs_unindex_persistence_OBJECTS = \ + test_fs_unindex_persistence.$(OBJEXT) +test_fs_unindex_persistence_OBJECTS = \ + $(am_test_fs_unindex_persistence_OBJECTS) +test_fs_unindex_persistence_DEPENDENCIES = \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la +am_test_fs_uri_OBJECTS = test_fs_uri.$(OBJEXT) +test_fs_uri_OBJECTS = $(am_test_fs_uri_OBJECTS) +test_fs_uri_DEPENDENCIES = $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la +am_test_gnunet_service_fs_migration_OBJECTS = \ + test_gnunet_service_fs_migration.$(OBJEXT) +test_gnunet_service_fs_migration_OBJECTS = \ + $(am_test_gnunet_service_fs_migration_OBJECTS) +test_gnunet_service_fs_migration_DEPENDENCIES = \ + $(top_builddir)/src/fs/libgnunetfstest.a \ + $(top_builddir)/src/testing/libgnunettesting.la \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la +am_test_gnunet_service_fs_p2p_OBJECTS = \ + test_gnunet_service_fs_p2p.$(OBJEXT) +test_gnunet_service_fs_p2p_OBJECTS = \ + $(am_test_gnunet_service_fs_p2p_OBJECTS) +test_gnunet_service_fs_p2p_DEPENDENCIES = \ + $(top_builddir)/src/fs/libgnunetfstest.a \ + $(top_builddir)/src/testing/libgnunettesting.la \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la +SCRIPTS = $(bin_SCRIPTS) +DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir) +depcomp = $(SHELL) $(top_srcdir)/depcomp +am__depfiles_maybe = depfiles +am__mv = mv -f +COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \ + $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) +LTCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) \ + $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \ + $(AM_CFLAGS) $(CFLAGS) +AM_V_CC = $(am__v_CC_$(V)) +am__v_CC_ = $(am__v_CC_$(AM_DEFAULT_VERBOSITY)) +am__v_CC_0 = @echo " CC " $@; +CCLD = $(CC) +LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \ + $(AM_LDFLAGS) $(LDFLAGS) -o $@ +AM_V_CCLD = $(am__v_CCLD_$(V)) +am__v_CCLD_ = $(am__v_CCLD_$(AM_DEFAULT_VERBOSITY)) +am__v_CCLD_0 = @echo " CCLD " $@; +AM_V_GEN = $(am__v_GEN_$(V)) +am__v_GEN_ = $(am__v_GEN_$(AM_DEFAULT_VERBOSITY)) +am__v_GEN_0 = @echo " GEN " $@; +SOURCES = $(libgnunetfstest_a_SOURCES) \ + $(libgnunet_plugin_block_fs_la_SOURCES) \ + $(libgnunetfs_la_SOURCES) $(gnunet_directory_SOURCES) \ + $(gnunet_download_SOURCES) $(gnunet_fs_SOURCES) \ + $(gnunet_helper_fs_publish_SOURCES) \ + $(gnunet_pseudonym_SOURCES) $(gnunet_publish_SOURCES) \ + $(gnunet_search_SOURCES) $(gnunet_service_fs_SOURCES) \ + $(gnunet_unindex_SOURCES) \ + $(perf_gnunet_service_fs_p2p_SOURCES) \ + $(perf_gnunet_service_fs_p2p_dht_SOURCES) \ + $(perf_gnunet_service_fs_p2p_index_SOURCES) \ + $(perf_gnunet_service_fs_p2p_trust_SOURCES) \ + $(test_fs_directory_SOURCES) $(test_fs_download_SOURCES) \ + $(test_fs_download_indexed_SOURCES) \ + $(test_fs_download_persistence_SOURCES) \ + $(test_fs_file_information_SOURCES) $(test_fs_getopt_SOURCES) \ + $(test_fs_list_indexed_SOURCES) $(test_fs_namespace_SOURCES) \ + $(test_fs_namespace_list_updateable_SOURCES) \ + $(test_fs_publish_SOURCES) \ + $(test_fs_publish_persistence_SOURCES) \ + $(test_fs_search_SOURCES) \ + $(test_fs_search_persistence_SOURCES) \ + $(test_fs_start_stop_SOURCES) $(test_fs_test_lib_SOURCES) \ + $(test_fs_unindex_SOURCES) \ + $(test_fs_unindex_persistence_SOURCES) $(test_fs_uri_SOURCES) \ + $(test_gnunet_service_fs_migration_SOURCES) \ + $(test_gnunet_service_fs_p2p_SOURCES) +DIST_SOURCES = $(libgnunetfstest_a_SOURCES) \ + $(libgnunet_plugin_block_fs_la_SOURCES) \ + $(libgnunetfs_la_SOURCES) $(gnunet_directory_SOURCES) \ + $(gnunet_download_SOURCES) $(gnunet_fs_SOURCES) \ + $(gnunet_helper_fs_publish_SOURCES) \ + $(gnunet_pseudonym_SOURCES) $(gnunet_publish_SOURCES) \ + $(gnunet_search_SOURCES) $(gnunet_service_fs_SOURCES) \ + $(gnunet_unindex_SOURCES) \ + $(perf_gnunet_service_fs_p2p_SOURCES) \ + $(perf_gnunet_service_fs_p2p_dht_SOURCES) \ + $(perf_gnunet_service_fs_p2p_index_SOURCES) \ + $(perf_gnunet_service_fs_p2p_trust_SOURCES) \ + $(test_fs_directory_SOURCES) $(test_fs_download_SOURCES) \ + $(test_fs_download_indexed_SOURCES) \ + $(test_fs_download_persistence_SOURCES) \ + $(test_fs_file_information_SOURCES) $(test_fs_getopt_SOURCES) \ + $(test_fs_list_indexed_SOURCES) $(test_fs_namespace_SOURCES) \ + $(test_fs_namespace_list_updateable_SOURCES) \ + $(test_fs_publish_SOURCES) \ + $(test_fs_publish_persistence_SOURCES) \ + $(test_fs_search_SOURCES) \ + $(test_fs_search_persistence_SOURCES) \ + $(test_fs_start_stop_SOURCES) $(test_fs_test_lib_SOURCES) \ + $(test_fs_unindex_SOURCES) \ + $(test_fs_unindex_persistence_SOURCES) $(test_fs_uri_SOURCES) \ + $(test_gnunet_service_fs_migration_SOURCES) \ + $(test_gnunet_service_fs_p2p_SOURCES) +DATA = $(pkgcfg_DATA) +ETAGS = etags +CTAGS = ctags +am__tty_colors = \ +red=; grn=; lgn=; blu=; std= +DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) +ACLOCAL = @ACLOCAL@ +AMTAR = @AMTAR@ +AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@ +AR = @AR@ +ARGZ_H = @ARGZ_H@ +AS = @AS@ +AUTOCONF = @AUTOCONF@ +AUTOHEADER = @AUTOHEADER@ +AUTOMAKE = @AUTOMAKE@ +AWK = @AWK@ +CC = @CC@ +CCDEPMODE = @CCDEPMODE@ +CFLAGS = @CFLAGS@ +CPP = @CPP@ +CPPFLAGS = @CPPFLAGS@ +CXX = @CXX@ +CXXCPP = @CXXCPP@ +CXXDEPMODE = @CXXDEPMODE@ +CXXFLAGS = @CXXFLAGS@ +CYGPATH_W = @CYGPATH_W@ +DEFAULT_INTERFACE = @DEFAULT_INTERFACE@ +DEFS = @DEFS@ +DEPDIR = @DEPDIR@ +DLLDIR = @DLLDIR@ +DLLTOOL = @DLLTOOL@ +DSYMUTIL = @DSYMUTIL@ +DUMPBIN = @DUMPBIN@ +ECHO_C = @ECHO_C@ +ECHO_N = @ECHO_N@ +ECHO_T = @ECHO_T@ +EGREP = @EGREP@ +EXEEXT = @EXEEXT@ +EXT_LIBS = @EXT_LIBS@ +EXT_LIB_PATH = @EXT_LIB_PATH@ +FGREP = @FGREP@ +GMSGFMT = @GMSGFMT@ +GMSGFMT_015 = @GMSGFMT_015@ +GNUNETDNS_GROUP = @GNUNETDNS_GROUP@ +GN_DAEMON_CONFIG_DIR = @GN_DAEMON_CONFIG_DIR@ +GN_DAEMON_HOME_DIR = @GN_DAEMON_HOME_DIR@ +GN_INTLINCL = @GN_INTLINCL@ +GN_LIBINTL = @GN_LIBINTL@ +GN_LIB_LDFLAGS = @GN_LIB_LDFLAGS@ +GN_PLUGIN_LDFLAGS = @GN_PLUGIN_LDFLAGS@ +GN_USER_HOME_DIR = @GN_USER_HOME_DIR@ +GREP = @GREP@ +HAVE_LIBUNISTRING = @HAVE_LIBUNISTRING@ +INCLTDL = @INCLTDL@ +INSTALL = @INSTALL@ +INSTALL_DATA = @INSTALL_DATA@ +INSTALL_PROGRAM = @INSTALL_PROGRAM@ +INSTALL_SCRIPT = @INSTALL_SCRIPT@ +INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@ +INTLLIBS = @INTLLIBS@ +INTL_MACOSX_LIBS = @INTL_MACOSX_LIBS@ +LD = @LD@ +LDFLAGS = @LDFLAGS@ +LIBADD_DL = @LIBADD_DL@ +LIBADD_DLD_LINK = @LIBADD_DLD_LINK@ +LIBADD_DLOPEN = @LIBADD_DLOPEN@ +LIBADD_SHL_LOAD = @LIBADD_SHL_LOAD@ +LIBCURL = @LIBCURL@ +LIBCURL_CPPFLAGS = @LIBCURL_CPPFLAGS@ +LIBGCRYPT_CFLAGS = @LIBGCRYPT_CFLAGS@ +LIBGCRYPT_CONFIG = @LIBGCRYPT_CONFIG@ +LIBGCRYPT_LIBS = @LIBGCRYPT_LIBS@ +LIBICONV = @LIBICONV@ +LIBINTL = @LIBINTL@ +LIBLTDL = @LIBLTDL@ +LIBOBJS = @LIBOBJS@ +LIBPREFIX = @LIBPREFIX@ +LIBS = @LIBS@ +LIBTOOL = @LIBTOOL@ +LIBUNISTRING = @LIBUNISTRING@ +LIPO = @LIPO@ +LN_S = @LN_S@ +LTDLDEPS = @LTDLDEPS@ +LTDLINCL = @LTDLINCL@ +LTDLOPEN = @LTDLOPEN@ +LTLIBICONV = @LTLIBICONV@ +LTLIBINTL = @LTLIBINTL@ +LTLIBOBJS = @LTLIBOBJS@ +LTLIBUNISTRING = @LTLIBUNISTRING@ +LT_CONFIG_H = @LT_CONFIG_H@ +LT_DLLOADERS = @LT_DLLOADERS@ +LT_DLPREOPEN = @LT_DLPREOPEN@ +MAKEINFO = @MAKEINFO@ +MKDIR_P = @MKDIR_P@ +MSGFMT = @MSGFMT@ +MSGFMT_015 = @MSGFMT_015@ +MSGMERGE = @MSGMERGE@ +MYSQL_CPPFLAGS = @MYSQL_CPPFLAGS@ +MYSQL_LDFLAGS = @MYSQL_LDFLAGS@ +NM = @NM@ +NMEDIT = @NMEDIT@ +OBJC = @OBJC@ +OBJCDEPMODE = @OBJCDEPMODE@ +OBJCFLAGS = @OBJCFLAGS@ +OBJDUMP = @OBJDUMP@ +OBJEXT = @OBJEXT@ +OTOOL = @OTOOL@ +OTOOL64 = @OTOOL64@ +PACKAGE = @PACKAGE@ +PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@ +PACKAGE_NAME = @PACKAGE_NAME@ +PACKAGE_STRING = @PACKAGE_STRING@ +PACKAGE_TARNAME = @PACKAGE_TARNAME@ +PACKAGE_URL = @PACKAGE_URL@ +PACKAGE_VERSION = @PACKAGE_VERSION@ +PATH_SEPARATOR = @PATH_SEPARATOR@ +POSTGRES_CPPFLAGS = @POSTGRES_CPPFLAGS@ +POSTGRES_LDFLAGS = @POSTGRES_LDFLAGS@ +POSUB = @POSUB@ +PYTHON = @PYTHON@ +PYTHON_EXEC_PREFIX = @PYTHON_EXEC_PREFIX@ +PYTHON_PLATFORM = @PYTHON_PLATFORM@ +PYTHON_PREFIX = @PYTHON_PREFIX@ +PYTHON_VERSION = @PYTHON_VERSION@ +RANLIB = @RANLIB@ +SED = @SED@ +SET_MAKE = @SET_MAKE@ +SHELL = @SHELL@ +SQLITE_CPPFLAGS = @SQLITE_CPPFLAGS@ +SQLITE_LDFLAGS = @SQLITE_LDFLAGS@ +STRIP = @STRIP@ +SUDO_BINARY = @SUDO_BINARY@ +UNIXONLY = @UNIXONLY@ +USE_NLS = @USE_NLS@ +VERSION = @VERSION@ +XGETTEXT = @XGETTEXT@ +XGETTEXT_015 = @XGETTEXT_015@ +XMKMF = @XMKMF@ +X_CFLAGS = @X_CFLAGS@ +X_EXTRA_LIBS = @X_EXTRA_LIBS@ +X_LIBS = @X_LIBS@ +X_PRE_LIBS = @X_PRE_LIBS@ +_libcurl_config = @_libcurl_config@ +abs_builddir = @abs_builddir@ +abs_srcdir = @abs_srcdir@ +abs_top_builddir = @abs_top_builddir@ +abs_top_srcdir = @abs_top_srcdir@ +ac_ct_CC = @ac_ct_CC@ +ac_ct_CXX = @ac_ct_CXX@ +ac_ct_DUMPBIN = @ac_ct_DUMPBIN@ +ac_ct_OBJC = @ac_ct_OBJC@ +am__include = @am__include@ +am__leading_dot = @am__leading_dot@ +am__quote = @am__quote@ +am__tar = @am__tar@ +am__untar = @am__untar@ +bindir = @bindir@ +build = @build@ +build_alias = @build_alias@ +build_cpu = @build_cpu@ +build_os = @build_os@ +build_target = @build_target@ +build_vendor = @build_vendor@ +builddir = @builddir@ +datadir = @datadir@ +datarootdir = @datarootdir@ +docdir = @docdir@ +dvidir = @dvidir@ +exec_prefix = @exec_prefix@ +host = @host@ +host_alias = @host_alias@ +host_cpu = @host_cpu@ +host_os = @host_os@ +host_vendor = @host_vendor@ +htmldir = @htmldir@ +includedir = @includedir@ +infodir = @infodir@ +install_sh = @install_sh@ +libdir = @libdir@ +libexecdir = @libexecdir@ +localedir = @localedir@ +localstatedir = @localstatedir@ +lt_ECHO = @lt_ECHO@ +ltdl_LIBOBJS = @ltdl_LIBOBJS@ +ltdl_LTLIBOBJS = @ltdl_LTLIBOBJS@ +mandir = @mandir@ +mkdir_p = @mkdir_p@ +oldincludedir = @oldincludedir@ +pdfdir = @pdfdir@ +pkgpyexecdir = @pkgpyexecdir@ +pkgpythondir = @pkgpythondir@ +prefix = @prefix@ +program_transform_name = @program_transform_name@ +psdir = @psdir@ +pyexecdir = @pyexecdir@ +pythondir = @pythondir@ +sbindir = @sbindir@ +sharedstatedir = @sharedstatedir@ +srcdir = @srcdir@ +subdirs = @subdirs@ +sys_symbol_underscore = @sys_symbol_underscore@ +sysconfdir = @sysconfdir@ +target = @target@ +target_alias = @target_alias@ +target_cpu = @target_cpu@ +target_os = @target_os@ +target_vendor = @target_vendor@ +top_build_prefix = @top_build_prefix@ +top_builddir = @top_builddir@ +top_srcdir = @top_srcdir@ +INCLUDES = -I$(top_srcdir)/src/include +@MINGW_TRUE@WINFLAGS = -Wl,--no-undefined -Wl,--export-all-symbols +@USE_COVERAGE_TRUE@AM_CFLAGS = --coverage -O0 +@USE_COVERAGE_TRUE@XLIB = -lgcov +pkgcfgdir = $(pkgdatadir)/config.d/ +pkgcfg_DATA = \ + fs.conf + +plugindir = $(libdir)/gnunet +lib_LTLIBRARIES = libgnunetfs.la +plugin_LTLIBRARIES = \ + libgnunet_plugin_block_fs.la + +noinst_LIBRARIES = libgnunetfstest.a +libgnunetfs_la_SOURCES = \ + fs_api.c fs_api.h fs.h \ + fs_directory.c \ + fs_dirmetascan.c \ + fs_download.c \ + fs_file_information.c \ + fs_getopt.c \ + fs_list_indexed.c \ + fs_publish.c \ + fs_publish_ksk.c \ + fs_misc.c \ + fs_namespace.c \ + fs_namespace_advertise.c \ + fs_search.c \ + fs_sharetree.c \ + fs_tree.c fs_tree.h \ + fs_unindex.c \ + fs_uri.c + +libgnunetfs_la_LIBADD = \ + $(top_builddir)/src/datastore/libgnunetdatastore.la \ + $(top_builddir)/src/util/libgnunetutil.la \ + $(GN_LIBINTL) $(XLIB) -lunistring -lextractor + +libgnunetfs_la_LDFLAGS = \ + $(GN_LIB_LDFLAGS) $(WINFLAGS) \ + -version-info 2:0:0 + +libgnunetfstest_a_SOURCES = \ + fs_test_lib.c fs_test_lib.h + +libgnunetfstest_a_LIBADD = \ + $(top_builddir)/src/testing/libgnunettesting.la + +bin_SCRIPTS = \ + gnunet-download-manager.scm + +gnunet_directory_SOURCES = \ + gnunet-directory.c + +gnunet_directory_LDADD = \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la \ + -lextractor \ + $(GN_LIBINTL) + +gnunet_directory_DEPENDENCIES = \ + libgnunetfs.la + +gnunet_fs_SOURCES = \ + gnunet-fs.c + +gnunet_fs_LDADD = \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la \ + -lextractor \ + $(GN_LIBINTL) + +gnunet_fs_DEPENDENCIES = \ + libgnunetfs.la + +gnunet_download_SOURCES = \ + gnunet-download.c + +gnunet_download_LDADD = \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la \ + $(GN_LIBINTL) + +gnunet_download_DEPENDENCIES = \ + libgnunetfs.la + +gnunet_publish_SOURCES = \ + gnunet-publish.c + +gnunet_publish_LDADD = \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la \ + -lextractor \ + $(GN_LIBINTL) + +gnunet_publish_DEPENDENCIES = \ + libgnunetfs.la + +gnunet_helper_fs_publish_SOURCES = \ + gnunet-helper-fs-publish.c + +gnunet_helper_fs_publish_LDADD = \ + $(top_builddir)/src/util/libgnunetutil.la \ + -lextractor \ + $(GN_LIBINTL) + +gnunet_helper_fs_publish_DEPENDENCIES = \ + libgnunetfs.la + +gnunet_pseudonym_SOURCES = \ + gnunet-pseudonym.c + +gnunet_pseudonym_LDADD = \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la \ + -lextractor \ + $(GN_LIBINTL) + +gnunet_pseudonym_DEPENDENCIES = \ + libgnunetfs.la + +gnunet_search_SOURCES = \ + gnunet-search.c + +gnunet_search_LDADD = \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la \ + -lextractor \ + $(GN_LIBINTL) + +gnunet_search_DEPENDENCIES = \ + libgnunetfs.la + +gnunet_service_fs_SOURCES = \ + gnunet-service-fs.c gnunet-service-fs.h \ + gnunet-service-fs_cp.c gnunet-service-fs_cp.h \ + gnunet-service-fs_indexing.c gnunet-service-fs_indexing.h \ + gnunet-service-fs_lc.c gnunet-service-fs_lc.h \ + gnunet-service-fs_pe.c gnunet-service-fs_pe.h \ + gnunet-service-fs_pr.c gnunet-service-fs_pr.h \ + gnunet-service-fs_push.c gnunet-service-fs_push.h \ + gnunet-service-fs_put.c gnunet-service-fs_put.h + +gnunet_service_fs_LDADD = \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/dht/libgnunetdht.la \ + $(top_builddir)/src/block/libgnunetblock.la \ + $(top_builddir)/src/datastore/libgnunetdatastore.la \ + $(top_builddir)/src/statistics/libgnunetstatistics.la \ + $(top_builddir)/src/ats/libgnunetats.la \ + $(top_builddir)/src/core/libgnunetcore.la \ + $(top_builddir)/src/util/libgnunetutil.la \ + $(GN_LIBINTL) -lm + +gnunet_service_fs_DEPENDENCIES = \ + libgnunetfs.la + +gnunet_unindex_SOURCES = \ + gnunet-unindex.c + +gnunet_unindex_LDADD = \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la \ + $(GN_LIBINTL) + +gnunet_unindex_DEPENDENCIES = \ + libgnunetfs.la + +libgnunet_plugin_block_fs_la_SOURCES = \ + plugin_block_fs.c + +libgnunet_plugin_block_fs_la_LIBADD = \ + $(top_builddir)/src/block/libgnunetblock.la \ + $(top_builddir)/src/util/libgnunetutil.la + +libgnunet_plugin_block_fs_la_LDFLAGS = \ + $(GN_PLUGIN_LDFLAGS) + +libgnunet_plugin_block_fs_la_DEPENDENCIES = \ + $(top_builddir)/src/block/libgnunetblock.la + +@HAVE_BENCHMARKS_TRUE@FS_BENCHMARKS = \ +@HAVE_BENCHMARKS_TRUE@ perf_gnunet_service_fs_p2p \ +@HAVE_BENCHMARKS_TRUE@ perf_gnunet_service_fs_p2p_dht \ +@HAVE_BENCHMARKS_TRUE@ perf_gnunet_service_fs_p2p_index \ +@HAVE_BENCHMARKS_TRUE@ perf_gnunet_service_fs_p2p_trust + +@HAVE_PYTHON_PEXPECT_TRUE@check_SCRIPTS = \ +@HAVE_PYTHON_PEXPECT_TRUE@ test_gnunet_fs_psd.py \ +@HAVE_PYTHON_PEXPECT_TRUE@ test_gnunet_fs_rec.py \ +@HAVE_PYTHON_PEXPECT_TRUE@ test_gnunet_fs_idx.py \ +@HAVE_PYTHON_PEXPECT_TRUE@ test_gnunet_fs_ns.py + +test_fs_directory_SOURCES = \ + test_fs_directory.c + +test_fs_directory_LDADD = \ + -lextractor \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la + +test_fs_download_SOURCES = \ + test_fs_download.c + +test_fs_download_LDADD = \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la + +test_fs_download_indexed_SOURCES = \ + test_fs_download_indexed.c + +test_fs_download_indexed_LDADD = \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la + +test_fs_download_persistence_SOURCES = \ + test_fs_download_persistence.c + +test_fs_download_persistence_LDADD = \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la + +test_fs_file_information_SOURCES = \ + test_fs_file_information.c + +test_fs_file_information_LDADD = \ + -lextractor \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la + +test_fs_getopt_SOURCES = \ + test_fs_getopt.c + +test_fs_getopt_LDADD = \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la + +test_fs_list_indexed_SOURCES = \ + test_fs_list_indexed.c + +test_fs_list_indexed_LDADD = \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la + +test_fs_namespace_SOURCES = \ + test_fs_namespace.c + +test_fs_namespace_LDADD = \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la + +test_fs_namespace_list_updateable_SOURCES = \ + test_fs_namespace_list_updateable.c + +test_fs_namespace_list_updateable_LDADD = \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la + +test_fs_publish_SOURCES = \ + test_fs_publish.c + +test_fs_publish_LDADD = \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la + +test_fs_publish_persistence_SOURCES = \ + test_fs_publish_persistence.c + +test_fs_publish_persistence_LDADD = \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la + +test_fs_search_SOURCES = \ + test_fs_search.c + +test_fs_search_LDADD = $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la + +test_fs_search_persistence_SOURCES = \ + test_fs_search_persistence.c + +test_fs_search_persistence_LDADD = $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la + +test_fs_start_stop_SOURCES = \ + test_fs_start_stop.c + +test_fs_start_stop_LDADD = \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la + +test_fs_unindex_SOURCES = \ + test_fs_unindex.c + +test_fs_unindex_LDADD = \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la + +test_fs_unindex_persistence_SOURCES = \ + test_fs_unindex_persistence.c + +test_fs_unindex_persistence_LDADD = \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la + +test_fs_uri_SOURCES = \ + test_fs_uri.c + +test_fs_uri_LDADD = \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la + +test_fs_test_lib_SOURCES = \ + test_fs_test_lib.c + +test_fs_test_lib_LDADD = \ + $(top_builddir)/src/fs/libgnunetfstest.a \ + $(top_builddir)/src/testing/libgnunettesting.la \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la + +test_gnunet_service_fs_p2p_SOURCES = \ + test_gnunet_service_fs_p2p.c + +test_gnunet_service_fs_p2p_LDADD = \ + $(top_builddir)/src/fs/libgnunetfstest.a \ + $(top_builddir)/src/testing/libgnunettesting.la \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la + +test_gnunet_service_fs_migration_SOURCES = \ + test_gnunet_service_fs_migration.c + +test_gnunet_service_fs_migration_LDADD = \ + $(top_builddir)/src/fs/libgnunetfstest.a \ + $(top_builddir)/src/testing/libgnunettesting.la \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la + +perf_gnunet_service_fs_p2p_SOURCES = \ + perf_gnunet_service_fs_p2p.c + +perf_gnunet_service_fs_p2p_LDADD = \ + $(top_builddir)/src/fs/libgnunetfstest.a \ + $(top_builddir)/src/statistics/libgnunetstatistics.la \ + $(top_builddir)/src/testing/libgnunettesting.la \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la + +perf_gnunet_service_fs_p2p_index_SOURCES = \ + perf_gnunet_service_fs_p2p.c + +perf_gnunet_service_fs_p2p_index_LDADD = \ + $(top_builddir)/src/fs/libgnunetfstest.a \ + $(top_builddir)/src/statistics/libgnunetstatistics.la \ + $(top_builddir)/src/testing/libgnunettesting.la \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la + +perf_gnunet_service_fs_p2p_dht_SOURCES = \ + perf_gnunet_service_fs_p2p.c + +perf_gnunet_service_fs_p2p_dht_LDADD = \ + $(top_builddir)/src/fs/libgnunetfstest.a \ + $(top_builddir)/src/statistics/libgnunetstatistics.la \ + $(top_builddir)/src/testing/libgnunettesting.la \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la + +perf_gnunet_service_fs_p2p_trust_SOURCES = \ + perf_gnunet_service_fs_p2p_trust.c + +perf_gnunet_service_fs_p2p_trust_LDADD = \ + $(top_builddir)/src/fs/libgnunetfstest.a \ + $(top_builddir)/src/statistics/libgnunetstatistics.la \ + $(top_builddir)/src/testing/libgnunettesting.la \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la + +do_subst = $(SED) -e 's,[@]PYTHON[@],$(PYTHON),g' +EXTRA_DIST = \ + test_fs_defaults.conf \ + fs_test_lib_data.conf \ + test_fs_data.conf \ + test_fs_download_data.conf \ + test_fs_file_information_data.conf \ + fs_test_lib_data.conf \ + test_fs_list_indexed_data.conf \ + test_fs_namespace_data.conf \ + test_fs_publish_data.conf \ + test_fs_search_data.conf \ + test_fs_unindex_data.conf \ + test_fs_uri_data.conf \ + test_gnunet_service_fs_migration_data.conf \ + test_gnunet_fs_idx_data.conf \ + test_gnunet_fs_ns_data.conf \ + test_gnunet_fs_psd_data.conf \ + test_gnunet_fs_rec_data.conf \ + test_gnunet_fs_rec_data.tgz \ + test_gnunet_fs_psd.py.in \ + test_gnunet_fs_rec.py.in \ + test_gnunet_fs_ns.py.in \ + test_gnunet_fs_idx.py.in \ + $(bin_SCRIPTS) + +CLEANFILES = $(check_SCRIPTS) +all: all-am + +.SUFFIXES: +.SUFFIXES: .c .lo .o .obj +$(srcdir)/Makefile.in: $(srcdir)/Makefile.am $(am__configure_deps) + @for dep in $?; do \ + case '$(am__configure_deps)' in \ + *$$dep*) \ + ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \ + && { if test -f $@; then exit 0; else break; fi; }; \ + exit 1;; \ + esac; \ + done; \ + echo ' cd $(top_srcdir) && $(AUTOMAKE) --gnu src/fs/Makefile'; \ + $(am__cd) $(top_srcdir) && \ + $(AUTOMAKE) --gnu src/fs/Makefile +.PRECIOUS: Makefile +Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status + @case '$?' in \ + *config.status*) \ + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \ + *) \ + echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe)'; \ + cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe);; \ + esac; + +$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh + +$(top_srcdir)/configure: $(am__configure_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(ACLOCAL_M4): $(am__aclocal_m4_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(am__aclocal_m4_deps): +fs.conf: $(top_builddir)/config.status $(srcdir)/fs.conf.in + cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ + +clean-noinstLIBRARIES: + -test -z "$(noinst_LIBRARIES)" || rm -f $(noinst_LIBRARIES) +libgnunetfstest.a: $(libgnunetfstest_a_OBJECTS) $(libgnunetfstest_a_DEPENDENCIES) + $(AM_V_at)-rm -f libgnunetfstest.a + $(AM_V_AR)$(libgnunetfstest_a_AR) libgnunetfstest.a $(libgnunetfstest_a_OBJECTS) $(libgnunetfstest_a_LIBADD) + $(AM_V_at)$(RANLIB) libgnunetfstest.a +install-libLTLIBRARIES: $(lib_LTLIBRARIES) + @$(NORMAL_INSTALL) + test -z "$(libdir)" || $(MKDIR_P) "$(DESTDIR)$(libdir)" + @list='$(lib_LTLIBRARIES)'; test -n "$(libdir)" || list=; \ + list2=; for p in $$list; do \ + if test -f $$p; then \ + list2="$$list2 $$p"; \ + else :; fi; \ + done; \ + test -z "$$list2" || { \ + echo " $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL) $(INSTALL_STRIP_FLAG) $$list2 '$(DESTDIR)$(libdir)'"; \ + $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL) $(INSTALL_STRIP_FLAG) $$list2 "$(DESTDIR)$(libdir)"; \ + } + +uninstall-libLTLIBRARIES: + @$(NORMAL_UNINSTALL) + @list='$(lib_LTLIBRARIES)'; test -n "$(libdir)" || list=; \ + for p in $$list; do \ + $(am__strip_dir) \ + echo " $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=uninstall rm -f '$(DESTDIR)$(libdir)/$$f'"; \ + $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=uninstall rm -f "$(DESTDIR)$(libdir)/$$f"; \ + done + +clean-libLTLIBRARIES: + -test -z "$(lib_LTLIBRARIES)" || rm -f $(lib_LTLIBRARIES) + @list='$(lib_LTLIBRARIES)'; for p in $$list; do \ + dir="`echo $$p | sed -e 's|/[^/]*$$||'`"; \ + test "$$dir" != "$$p" || dir=.; \ + echo "rm -f \"$${dir}/so_locations\""; \ + rm -f "$${dir}/so_locations"; \ + done +install-pluginLTLIBRARIES: $(plugin_LTLIBRARIES) + @$(NORMAL_INSTALL) + test -z "$(plugindir)" || $(MKDIR_P) "$(DESTDIR)$(plugindir)" + @list='$(plugin_LTLIBRARIES)'; test -n "$(plugindir)" || list=; \ + list2=; for p in $$list; do \ + if test -f $$p; then \ + list2="$$list2 $$p"; \ + else :; fi; \ + done; \ + test -z "$$list2" || { \ + echo " $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL) $(INSTALL_STRIP_FLAG) $$list2 '$(DESTDIR)$(plugindir)'"; \ + $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL) $(INSTALL_STRIP_FLAG) $$list2 "$(DESTDIR)$(plugindir)"; \ + } + +uninstall-pluginLTLIBRARIES: + @$(NORMAL_UNINSTALL) + @list='$(plugin_LTLIBRARIES)'; test -n "$(plugindir)" || list=; \ + for p in $$list; do \ + $(am__strip_dir) \ + echo " $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=uninstall rm -f '$(DESTDIR)$(plugindir)/$$f'"; \ + $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=uninstall rm -f "$(DESTDIR)$(plugindir)/$$f"; \ + done + +clean-pluginLTLIBRARIES: + -test -z "$(plugin_LTLIBRARIES)" || rm -f $(plugin_LTLIBRARIES) + @list='$(plugin_LTLIBRARIES)'; for p in $$list; do \ + dir="`echo $$p | sed -e 's|/[^/]*$$||'`"; \ + test "$$dir" != "$$p" || dir=.; \ + echo "rm -f \"$${dir}/so_locations\""; \ + rm -f "$${dir}/so_locations"; \ + done +libgnunet_plugin_block_fs.la: $(libgnunet_plugin_block_fs_la_OBJECTS) $(libgnunet_plugin_block_fs_la_DEPENDENCIES) + $(AM_V_CCLD)$(libgnunet_plugin_block_fs_la_LINK) -rpath $(plugindir) $(libgnunet_plugin_block_fs_la_OBJECTS) $(libgnunet_plugin_block_fs_la_LIBADD) $(LIBS) +libgnunetfs.la: $(libgnunetfs_la_OBJECTS) $(libgnunetfs_la_DEPENDENCIES) + $(AM_V_CCLD)$(libgnunetfs_la_LINK) -rpath $(libdir) $(libgnunetfs_la_OBJECTS) $(libgnunetfs_la_LIBADD) $(LIBS) +install-binPROGRAMS: $(bin_PROGRAMS) + @$(NORMAL_INSTALL) + test -z "$(bindir)" || $(MKDIR_P) "$(DESTDIR)$(bindir)" + @list='$(bin_PROGRAMS)'; test -n "$(bindir)" || list=; \ + for p in $$list; do echo "$$p $$p"; done | \ + sed 's/$(EXEEXT)$$//' | \ + while read p p1; do if test -f $$p || test -f $$p1; \ + then echo "$$p"; echo "$$p"; else :; fi; \ + done | \ + sed -e 'p;s,.*/,,;n;h' -e 's|.*|.|' \ + -e 'p;x;s,.*/,,;s/$(EXEEXT)$$//;$(transform);s/$$/$(EXEEXT)/' | \ + sed 'N;N;N;s,\n, ,g' | \ + $(AWK) 'BEGIN { files["."] = ""; dirs["."] = 1 } \ + { d=$$3; if (dirs[d] != 1) { print "d", d; dirs[d] = 1 } \ + if ($$2 == $$4) files[d] = files[d] " " $$1; \ + else { print "f", $$3 "/" $$4, $$1; } } \ + END { for (d in files) print "f", d, files[d] }' | \ + while read type dir files; do \ + if test "$$dir" = .; then dir=; else dir=/$$dir; fi; \ + test -z "$$files" || { \ + echo " $(INSTALL_PROGRAM_ENV) $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL_PROGRAM) $$files '$(DESTDIR)$(bindir)$$dir'"; \ + $(INSTALL_PROGRAM_ENV) $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL_PROGRAM) $$files "$(DESTDIR)$(bindir)$$dir" || exit $$?; \ + } \ + ; done + +uninstall-binPROGRAMS: + @$(NORMAL_UNINSTALL) + @list='$(bin_PROGRAMS)'; test -n "$(bindir)" || list=; \ + files=`for p in $$list; do echo "$$p"; done | \ + sed -e 'h;s,^.*/,,;s/$(EXEEXT)$$//;$(transform)' \ + -e 's/$$/$(EXEEXT)/' `; \ + test -n "$$list" || exit 0; \ + echo " ( cd '$(DESTDIR)$(bindir)' && rm -f" $$files ")"; \ + cd "$(DESTDIR)$(bindir)" && rm -f $$files + +clean-binPROGRAMS: + @list='$(bin_PROGRAMS)'; test -n "$$list" || exit 0; \ + echo " rm -f" $$list; \ + rm -f $$list || exit $$?; \ + test -n "$(EXEEXT)" || exit 0; \ + list=`for p in $$list; do echo "$$p"; done | sed 's/$(EXEEXT)$$//'`; \ + echo " rm -f" $$list; \ + rm -f $$list + +clean-checkPROGRAMS: + @list='$(check_PROGRAMS)'; test -n "$$list" || exit 0; \ + echo " rm -f" $$list; \ + rm -f $$list || exit $$?; \ + test -n "$(EXEEXT)" || exit 0; \ + list=`for p in $$list; do echo "$$p"; done | sed 's/$(EXEEXT)$$//'`; \ + echo " rm -f" $$list; \ + rm -f $$list +gnunet-directory$(EXEEXT): $(gnunet_directory_OBJECTS) $(gnunet_directory_DEPENDENCIES) + @rm -f gnunet-directory$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(gnunet_directory_OBJECTS) $(gnunet_directory_LDADD) $(LIBS) +gnunet-download$(EXEEXT): $(gnunet_download_OBJECTS) $(gnunet_download_DEPENDENCIES) + @rm -f gnunet-download$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(gnunet_download_OBJECTS) $(gnunet_download_LDADD) $(LIBS) +gnunet-fs$(EXEEXT): $(gnunet_fs_OBJECTS) $(gnunet_fs_DEPENDENCIES) + @rm -f gnunet-fs$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(gnunet_fs_OBJECTS) $(gnunet_fs_LDADD) $(LIBS) +gnunet-helper-fs-publish$(EXEEXT): $(gnunet_helper_fs_publish_OBJECTS) $(gnunet_helper_fs_publish_DEPENDENCIES) + @rm -f gnunet-helper-fs-publish$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(gnunet_helper_fs_publish_OBJECTS) $(gnunet_helper_fs_publish_LDADD) $(LIBS) +gnunet-pseudonym$(EXEEXT): $(gnunet_pseudonym_OBJECTS) $(gnunet_pseudonym_DEPENDENCIES) + @rm -f gnunet-pseudonym$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(gnunet_pseudonym_OBJECTS) $(gnunet_pseudonym_LDADD) $(LIBS) +gnunet-publish$(EXEEXT): $(gnunet_publish_OBJECTS) $(gnunet_publish_DEPENDENCIES) + @rm -f gnunet-publish$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(gnunet_publish_OBJECTS) $(gnunet_publish_LDADD) $(LIBS) +gnunet-search$(EXEEXT): $(gnunet_search_OBJECTS) $(gnunet_search_DEPENDENCIES) + @rm -f gnunet-search$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(gnunet_search_OBJECTS) $(gnunet_search_LDADD) $(LIBS) +gnunet-service-fs$(EXEEXT): $(gnunet_service_fs_OBJECTS) $(gnunet_service_fs_DEPENDENCIES) + @rm -f gnunet-service-fs$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(gnunet_service_fs_OBJECTS) $(gnunet_service_fs_LDADD) $(LIBS) +gnunet-unindex$(EXEEXT): $(gnunet_unindex_OBJECTS) $(gnunet_unindex_DEPENDENCIES) + @rm -f gnunet-unindex$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(gnunet_unindex_OBJECTS) $(gnunet_unindex_LDADD) $(LIBS) +perf_gnunet_service_fs_p2p$(EXEEXT): $(perf_gnunet_service_fs_p2p_OBJECTS) $(perf_gnunet_service_fs_p2p_DEPENDENCIES) + @rm -f perf_gnunet_service_fs_p2p$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(perf_gnunet_service_fs_p2p_OBJECTS) $(perf_gnunet_service_fs_p2p_LDADD) $(LIBS) +perf_gnunet_service_fs_p2p_dht$(EXEEXT): $(perf_gnunet_service_fs_p2p_dht_OBJECTS) $(perf_gnunet_service_fs_p2p_dht_DEPENDENCIES) + @rm -f perf_gnunet_service_fs_p2p_dht$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(perf_gnunet_service_fs_p2p_dht_OBJECTS) $(perf_gnunet_service_fs_p2p_dht_LDADD) $(LIBS) +perf_gnunet_service_fs_p2p_index$(EXEEXT): $(perf_gnunet_service_fs_p2p_index_OBJECTS) $(perf_gnunet_service_fs_p2p_index_DEPENDENCIES) + @rm -f perf_gnunet_service_fs_p2p_index$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(perf_gnunet_service_fs_p2p_index_OBJECTS) $(perf_gnunet_service_fs_p2p_index_LDADD) $(LIBS) +perf_gnunet_service_fs_p2p_trust$(EXEEXT): $(perf_gnunet_service_fs_p2p_trust_OBJECTS) $(perf_gnunet_service_fs_p2p_trust_DEPENDENCIES) + @rm -f perf_gnunet_service_fs_p2p_trust$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(perf_gnunet_service_fs_p2p_trust_OBJECTS) $(perf_gnunet_service_fs_p2p_trust_LDADD) $(LIBS) +test_fs_directory$(EXEEXT): $(test_fs_directory_OBJECTS) $(test_fs_directory_DEPENDENCIES) + @rm -f test_fs_directory$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_fs_directory_OBJECTS) $(test_fs_directory_LDADD) $(LIBS) +test_fs_download$(EXEEXT): $(test_fs_download_OBJECTS) $(test_fs_download_DEPENDENCIES) + @rm -f test_fs_download$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_fs_download_OBJECTS) $(test_fs_download_LDADD) $(LIBS) +test_fs_download_indexed$(EXEEXT): $(test_fs_download_indexed_OBJECTS) $(test_fs_download_indexed_DEPENDENCIES) + @rm -f test_fs_download_indexed$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_fs_download_indexed_OBJECTS) $(test_fs_download_indexed_LDADD) $(LIBS) +test_fs_download_persistence$(EXEEXT): $(test_fs_download_persistence_OBJECTS) $(test_fs_download_persistence_DEPENDENCIES) + @rm -f test_fs_download_persistence$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_fs_download_persistence_OBJECTS) $(test_fs_download_persistence_LDADD) $(LIBS) +test_fs_file_information$(EXEEXT): $(test_fs_file_information_OBJECTS) $(test_fs_file_information_DEPENDENCIES) + @rm -f test_fs_file_information$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_fs_file_information_OBJECTS) $(test_fs_file_information_LDADD) $(LIBS) +test_fs_getopt$(EXEEXT): $(test_fs_getopt_OBJECTS) $(test_fs_getopt_DEPENDENCIES) + @rm -f test_fs_getopt$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_fs_getopt_OBJECTS) $(test_fs_getopt_LDADD) $(LIBS) +test_fs_list_indexed$(EXEEXT): $(test_fs_list_indexed_OBJECTS) $(test_fs_list_indexed_DEPENDENCIES) + @rm -f test_fs_list_indexed$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_fs_list_indexed_OBJECTS) $(test_fs_list_indexed_LDADD) $(LIBS) +test_fs_namespace$(EXEEXT): $(test_fs_namespace_OBJECTS) $(test_fs_namespace_DEPENDENCIES) + @rm -f test_fs_namespace$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_fs_namespace_OBJECTS) $(test_fs_namespace_LDADD) $(LIBS) +test_fs_namespace_list_updateable$(EXEEXT): $(test_fs_namespace_list_updateable_OBJECTS) $(test_fs_namespace_list_updateable_DEPENDENCIES) + @rm -f test_fs_namespace_list_updateable$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_fs_namespace_list_updateable_OBJECTS) $(test_fs_namespace_list_updateable_LDADD) $(LIBS) +test_fs_publish$(EXEEXT): $(test_fs_publish_OBJECTS) $(test_fs_publish_DEPENDENCIES) + @rm -f test_fs_publish$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_fs_publish_OBJECTS) $(test_fs_publish_LDADD) $(LIBS) +test_fs_publish_persistence$(EXEEXT): $(test_fs_publish_persistence_OBJECTS) $(test_fs_publish_persistence_DEPENDENCIES) + @rm -f test_fs_publish_persistence$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_fs_publish_persistence_OBJECTS) $(test_fs_publish_persistence_LDADD) $(LIBS) +test_fs_search$(EXEEXT): $(test_fs_search_OBJECTS) $(test_fs_search_DEPENDENCIES) + @rm -f test_fs_search$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_fs_search_OBJECTS) $(test_fs_search_LDADD) $(LIBS) +test_fs_search_persistence$(EXEEXT): $(test_fs_search_persistence_OBJECTS) $(test_fs_search_persistence_DEPENDENCIES) + @rm -f test_fs_search_persistence$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_fs_search_persistence_OBJECTS) $(test_fs_search_persistence_LDADD) $(LIBS) +test_fs_start_stop$(EXEEXT): $(test_fs_start_stop_OBJECTS) $(test_fs_start_stop_DEPENDENCIES) + @rm -f test_fs_start_stop$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_fs_start_stop_OBJECTS) $(test_fs_start_stop_LDADD) $(LIBS) +test_fs_test_lib$(EXEEXT): $(test_fs_test_lib_OBJECTS) $(test_fs_test_lib_DEPENDENCIES) + @rm -f test_fs_test_lib$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_fs_test_lib_OBJECTS) $(test_fs_test_lib_LDADD) $(LIBS) +test_fs_unindex$(EXEEXT): $(test_fs_unindex_OBJECTS) $(test_fs_unindex_DEPENDENCIES) + @rm -f test_fs_unindex$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_fs_unindex_OBJECTS) $(test_fs_unindex_LDADD) $(LIBS) +test_fs_unindex_persistence$(EXEEXT): $(test_fs_unindex_persistence_OBJECTS) $(test_fs_unindex_persistence_DEPENDENCIES) + @rm -f test_fs_unindex_persistence$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_fs_unindex_persistence_OBJECTS) $(test_fs_unindex_persistence_LDADD) $(LIBS) +test_fs_uri$(EXEEXT): $(test_fs_uri_OBJECTS) $(test_fs_uri_DEPENDENCIES) + @rm -f test_fs_uri$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_fs_uri_OBJECTS) $(test_fs_uri_LDADD) $(LIBS) +test_gnunet_service_fs_migration$(EXEEXT): $(test_gnunet_service_fs_migration_OBJECTS) $(test_gnunet_service_fs_migration_DEPENDENCIES) + @rm -f test_gnunet_service_fs_migration$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_gnunet_service_fs_migration_OBJECTS) $(test_gnunet_service_fs_migration_LDADD) $(LIBS) +test_gnunet_service_fs_p2p$(EXEEXT): $(test_gnunet_service_fs_p2p_OBJECTS) $(test_gnunet_service_fs_p2p_DEPENDENCIES) + @rm -f test_gnunet_service_fs_p2p$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_gnunet_service_fs_p2p_OBJECTS) $(test_gnunet_service_fs_p2p_LDADD) $(LIBS) +install-binSCRIPTS: $(bin_SCRIPTS) + @$(NORMAL_INSTALL) + test -z "$(bindir)" || $(MKDIR_P) "$(DESTDIR)$(bindir)" + @list='$(bin_SCRIPTS)'; test -n "$(bindir)" || list=; \ + for p in $$list; do \ + if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \ + if test -f "$$d$$p"; then echo "$$d$$p"; echo "$$p"; else :; fi; \ + done | \ + sed -e 'p;s,.*/,,;n' \ + -e 'h;s|.*|.|' \ + -e 'p;x;s,.*/,,;$(transform)' | sed 'N;N;N;s,\n, ,g' | \ + $(AWK) 'BEGIN { files["."] = ""; dirs["."] = 1; } \ + { d=$$3; if (dirs[d] != 1) { print "d", d; dirs[d] = 1 } \ + if ($$2 == $$4) { files[d] = files[d] " " $$1; \ + if (++n[d] == $(am__install_max)) { \ + print "f", d, files[d]; n[d] = 0; files[d] = "" } } \ + else { print "f", d "/" $$4, $$1 } } \ + END { for (d in files) print "f", d, files[d] }' | \ + while read type dir files; do \ + if test "$$dir" = .; then dir=; else dir=/$$dir; fi; \ + test -z "$$files" || { \ + echo " $(INSTALL_SCRIPT) $$files '$(DESTDIR)$(bindir)$$dir'"; \ + $(INSTALL_SCRIPT) $$files "$(DESTDIR)$(bindir)$$dir" || exit $$?; \ + } \ + ; done + +uninstall-binSCRIPTS: + @$(NORMAL_UNINSTALL) + @list='$(bin_SCRIPTS)'; test -n "$(bindir)" || exit 0; \ + files=`for p in $$list; do echo "$$p"; done | \ + sed -e 's,.*/,,;$(transform)'`; \ + test -n "$$list" || exit 0; \ + echo " ( cd '$(DESTDIR)$(bindir)' && rm -f" $$files ")"; \ + cd "$(DESTDIR)$(bindir)" && rm -f $$files + +mostlyclean-compile: + -rm -f *.$(OBJEXT) + +distclean-compile: + -rm -f *.tab.c + +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fs_api.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fs_directory.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fs_dirmetascan.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fs_download.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fs_file_information.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fs_getopt.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fs_list_indexed.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fs_misc.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fs_namespace.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fs_namespace_advertise.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fs_publish.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fs_publish_ksk.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fs_search.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fs_sharetree.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fs_test_lib.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fs_tree.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fs_unindex.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/fs_uri.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/gnunet-directory.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/gnunet-download.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/gnunet-fs.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/gnunet-helper-fs-publish.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/gnunet-pseudonym.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/gnunet-publish.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/gnunet-search.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/gnunet-service-fs.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/gnunet-service-fs_cp.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/gnunet-service-fs_indexing.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/gnunet-service-fs_lc.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/gnunet-service-fs_pe.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/gnunet-service-fs_pr.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/gnunet-service-fs_push.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/gnunet-service-fs_put.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/gnunet-unindex.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/perf_gnunet_service_fs_p2p.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/perf_gnunet_service_fs_p2p_trust.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/plugin_block_fs.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test_fs_directory.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test_fs_download.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test_fs_download_indexed.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test_fs_download_persistence.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test_fs_file_information.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test_fs_getopt.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test_fs_list_indexed.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test_fs_namespace.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test_fs_namespace_list_updateable.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test_fs_publish.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test_fs_publish_persistence.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test_fs_search.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test_fs_search_persistence.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test_fs_start_stop.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test_fs_test_lib.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test_fs_unindex.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test_fs_unindex_persistence.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test_fs_uri.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test_gnunet_service_fs_migration.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test_gnunet_service_fs_p2p.Po@am__quote@ + +.c.o: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(COMPILE) -c $< + +.c.obj: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(COMPILE) -c `$(CYGPATH_W) '$<'` + +.c.lo: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LTCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo +@am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(LTCOMPILE) -c -o $@ $< + +mostlyclean-libtool: + -rm -f *.lo + +clean-libtool: + -rm -rf .libs _libs +install-pkgcfgDATA: $(pkgcfg_DATA) + @$(NORMAL_INSTALL) + test -z "$(pkgcfgdir)" || $(MKDIR_P) "$(DESTDIR)$(pkgcfgdir)" + @list='$(pkgcfg_DATA)'; test -n "$(pkgcfgdir)" || list=; \ + for p in $$list; do \ + if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \ + echo "$$d$$p"; \ + done | $(am__base_list) | \ + while read files; do \ + echo " $(INSTALL_DATA) $$files '$(DESTDIR)$(pkgcfgdir)'"; \ + $(INSTALL_DATA) $$files "$(DESTDIR)$(pkgcfgdir)" || exit $$?; \ + done + +uninstall-pkgcfgDATA: + @$(NORMAL_UNINSTALL) + @list='$(pkgcfg_DATA)'; test -n "$(pkgcfgdir)" || list=; \ + files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \ + test -n "$$files" || exit 0; \ + echo " ( cd '$(DESTDIR)$(pkgcfgdir)' && rm -f" $$files ")"; \ + cd "$(DESTDIR)$(pkgcfgdir)" && rm -f $$files + +ID: $(HEADERS) $(SOURCES) $(LISP) $(TAGS_FILES) + list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | \ + $(AWK) '{ files[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in files) print i; }; }'`; \ + mkid -fID $$unique +tags: TAGS + +TAGS: $(HEADERS) $(SOURCES) $(TAGS_DEPENDENCIES) \ + $(TAGS_FILES) $(LISP) + set x; \ + here=`pwd`; \ + list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | \ + $(AWK) '{ files[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in files) print i; }; }'`; \ + shift; \ + if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \ + test -n "$$unique" || unique=$$empty_fix; \ + if test $$# -gt 0; then \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + "$$@" $$unique; \ + else \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + $$unique; \ + fi; \ + fi +ctags: CTAGS +CTAGS: $(HEADERS) $(SOURCES) $(TAGS_DEPENDENCIES) \ + $(TAGS_FILES) $(LISP) + list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | \ + $(AWK) '{ files[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in files) print i; }; }'`; \ + test -z "$(CTAGS_ARGS)$$unique" \ + || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \ + $$unique + +GTAGS: + here=`$(am__cd) $(top_builddir) && pwd` \ + && $(am__cd) $(top_srcdir) \ + && gtags -i $(GTAGS_ARGS) "$$here" + +distclean-tags: + -rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags + +check-TESTS: $(TESTS) + @failed=0; all=0; xfail=0; xpass=0; skip=0; \ + srcdir=$(srcdir); export srcdir; \ + list=' $(TESTS) '; \ + $(am__tty_colors); \ + if test -n "$$list"; then \ + for tst in $$list; do \ + if test -f ./$$tst; then dir=./; \ + elif test -f $$tst; then dir=; \ + else dir="$(srcdir)/"; fi; \ + if $(TESTS_ENVIRONMENT) $${dir}$$tst; then \ + all=`expr $$all + 1`; \ + case " $(XFAIL_TESTS) " in \ + *[\ \ ]$$tst[\ \ ]*) \ + xpass=`expr $$xpass + 1`; \ + failed=`expr $$failed + 1`; \ + col=$$red; res=XPASS; \ + ;; \ + *) \ + col=$$grn; res=PASS; \ + ;; \ + esac; \ + elif test $$? -ne 77; then \ + all=`expr $$all + 1`; \ + case " $(XFAIL_TESTS) " in \ + *[\ \ ]$$tst[\ \ ]*) \ + xfail=`expr $$xfail + 1`; \ + col=$$lgn; res=XFAIL; \ + ;; \ + *) \ + failed=`expr $$failed + 1`; \ + col=$$red; res=FAIL; \ + ;; \ + esac; \ + else \ + skip=`expr $$skip + 1`; \ + col=$$blu; res=SKIP; \ + fi; \ + echo "$${col}$$res$${std}: $$tst"; \ + done; \ + if test "$$all" -eq 1; then \ + tests="test"; \ + All=""; \ + else \ + tests="tests"; \ + All="All "; \ + fi; \ + if test "$$failed" -eq 0; then \ + if test "$$xfail" -eq 0; then \ + banner="$$All$$all $$tests passed"; \ + else \ + if test "$$xfail" -eq 1; then failures=failure; else failures=failures; fi; \ + banner="$$All$$all $$tests behaved as expected ($$xfail expected $$failures)"; \ + fi; \ + else \ + if test "$$xpass" -eq 0; then \ + banner="$$failed of $$all $$tests failed"; \ + else \ + if test "$$xpass" -eq 1; then passes=pass; else passes=passes; fi; \ + banner="$$failed of $$all $$tests did not behave as expected ($$xpass unexpected $$passes)"; \ + fi; \ + fi; \ + dashes="$$banner"; \ + skipped=""; \ + if test "$$skip" -ne 0; then \ + if test "$$skip" -eq 1; then \ + skipped="($$skip test was not run)"; \ + else \ + skipped="($$skip tests were not run)"; \ + fi; \ + test `echo "$$skipped" | wc -c` -le `echo "$$banner" | wc -c` || \ + dashes="$$skipped"; \ + fi; \ + report=""; \ + if test "$$failed" -ne 0 && test -n "$(PACKAGE_BUGREPORT)"; then \ + report="Please report to $(PACKAGE_BUGREPORT)"; \ + test `echo "$$report" | wc -c` -le `echo "$$banner" | wc -c` || \ + dashes="$$report"; \ + fi; \ + dashes=`echo "$$dashes" | sed s/./=/g`; \ + if test "$$failed" -eq 0; then \ + echo "$$grn$$dashes"; \ + else \ + echo "$$red$$dashes"; \ + fi; \ + echo "$$banner"; \ + test -z "$$skipped" || echo "$$skipped"; \ + test -z "$$report" || echo "$$report"; \ + echo "$$dashes$$std"; \ + test "$$failed" -eq 0; \ + else :; fi + +distdir: $(DISTFILES) + @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + list='$(DISTFILES)'; \ + dist_files=`for file in $$list; do echo $$file; done | \ + sed -e "s|^$$srcdirstrip/||;t" \ + -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \ + case $$dist_files in \ + */*) $(MKDIR_P) `echo "$$dist_files" | \ + sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \ + sort -u` ;; \ + esac; \ + for file in $$dist_files; do \ + if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \ + if test -d $$d/$$file; then \ + dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \ + if test -d "$(distdir)/$$file"; then \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \ + cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \ + else \ + test -f "$(distdir)/$$file" \ + || cp -p $$d/$$file "$(distdir)/$$file" \ + || exit 1; \ + fi; \ + done +check-am: all-am + $(MAKE) $(AM_MAKEFLAGS) $(check_PROGRAMS) $(check_SCRIPTS) + $(MAKE) $(AM_MAKEFLAGS) check-TESTS +check: check-am +all-am: Makefile $(LIBRARIES) $(LTLIBRARIES) $(PROGRAMS) $(SCRIPTS) \ + $(DATA) +install-binPROGRAMS: install-libLTLIBRARIES + +installdirs: + for dir in "$(DESTDIR)$(libdir)" "$(DESTDIR)$(plugindir)" "$(DESTDIR)$(bindir)" "$(DESTDIR)$(bindir)" "$(DESTDIR)$(pkgcfgdir)"; do \ + test -z "$$dir" || $(MKDIR_P) "$$dir"; \ + done +install: install-am +install-exec: install-exec-am +install-data: install-data-am +uninstall: uninstall-am + +install-am: all-am + @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am + +installcheck: installcheck-am +install-strip: + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + `test -z '$(STRIP)' || \ + echo "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'"` install +mostlyclean-generic: + +clean-generic: + -test -z "$(CLEANFILES)" || rm -f $(CLEANFILES) + +distclean-generic: + -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES) + -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES) + +maintainer-clean-generic: + @echo "This command is intended for maintainers to use" + @echo "it deletes files that may require special tools to rebuild." +clean: clean-am + +clean-am: clean-binPROGRAMS clean-checkPROGRAMS clean-generic \ + clean-libLTLIBRARIES clean-libtool clean-noinstLIBRARIES \ + clean-pluginLTLIBRARIES mostlyclean-am + +distclean: distclean-am + -rm -rf ./$(DEPDIR) + -rm -f Makefile +distclean-am: clean-am distclean-compile distclean-generic \ + distclean-tags + +dvi: dvi-am + +dvi-am: + +html: html-am + +html-am: + +info: info-am + +info-am: + +install-data-am: install-pkgcfgDATA install-pluginLTLIBRARIES + +install-dvi: install-dvi-am + +install-dvi-am: + +install-exec-am: install-binPROGRAMS install-binSCRIPTS \ + install-libLTLIBRARIES + +install-html: install-html-am + +install-html-am: + +install-info: install-info-am + +install-info-am: + +install-man: + +install-pdf: install-pdf-am + +install-pdf-am: + +install-ps: install-ps-am + +install-ps-am: + +installcheck-am: + +maintainer-clean: maintainer-clean-am + -rm -rf ./$(DEPDIR) + -rm -f Makefile +maintainer-clean-am: distclean-am maintainer-clean-generic + +mostlyclean: mostlyclean-am + +mostlyclean-am: mostlyclean-compile mostlyclean-generic \ + mostlyclean-libtool + +pdf: pdf-am + +pdf-am: + +ps: ps-am + +ps-am: + +uninstall-am: uninstall-binPROGRAMS uninstall-binSCRIPTS \ + uninstall-libLTLIBRARIES uninstall-pkgcfgDATA \ + uninstall-pluginLTLIBRARIES + +.MAKE: check-am install-am install-strip + +.PHONY: CTAGS GTAGS all all-am check check-TESTS check-am clean \ + clean-binPROGRAMS clean-checkPROGRAMS clean-generic \ + clean-libLTLIBRARIES clean-libtool clean-noinstLIBRARIES \ + clean-pluginLTLIBRARIES ctags distclean distclean-compile \ + distclean-generic distclean-libtool distclean-tags distdir dvi \ + dvi-am html html-am info info-am install install-am \ + install-binPROGRAMS install-binSCRIPTS install-data \ + install-data-am install-dvi install-dvi-am install-exec \ + install-exec-am install-html install-html-am install-info \ + install-info-am install-libLTLIBRARIES install-man install-pdf \ + install-pdf-am install-pkgcfgDATA install-pluginLTLIBRARIES \ + install-ps install-ps-am install-strip installcheck \ + installcheck-am installdirs maintainer-clean \ + maintainer-clean-generic mostlyclean mostlyclean-compile \ + mostlyclean-generic mostlyclean-libtool pdf pdf-am ps ps-am \ + tags uninstall uninstall-am uninstall-binPROGRAMS \ + uninstall-binSCRIPTS uninstall-libLTLIBRARIES \ + uninstall-pkgcfgDATA uninstall-pluginLTLIBRARIES + + +test_gnunet_fs_psd.py: test_gnunet_fs_psd.py.in Makefile + $(do_subst) < $(srcdir)/test_gnunet_fs_psd.py.in > test_gnunet_fs_psd.py + chmod +x test_gnunet_fs_psd.py + +test_gnunet_fs_rec.py: test_gnunet_fs_rec.py.in Makefile + $(do_subst) < $(srcdir)/test_gnunet_fs_rec.py.in > test_gnunet_fs_rec.py + chmod +x test_gnunet_fs_rec.py + +test_gnunet_fs_ns.py: test_gnunet_fs_ns.py.in Makefile + $(do_subst) < $(srcdir)/test_gnunet_fs_ns.py.in > test_gnunet_fs_ns.py + chmod +x test_gnunet_fs_ns.py + +test_gnunet_fs_idx.py: test_gnunet_fs_idx.py.in Makefile + $(do_subst) < $(srcdir)/test_gnunet_fs_idx.py.in > test_gnunet_fs_idx.py + chmod +x test_gnunet_fs_idx.py + +# Tell versions [3.59,3.63) of GNU make to not export all variables. +# Otherwise a system limit (for SysV at least) may be exceeded. +.NOEXPORT: diff --git a/src/fs/fs.conf.in b/src/fs/fs.conf.in new file mode 100644 index 0000000..48c8b52 --- /dev/null +++ b/src/fs/fs.conf.in @@ -0,0 +1,32 @@ +[fs] +AUTOSTART = YES +INDEXDB = $SERVICEHOME/idxinfo.lst +TRUST = $SERVICEHOME/data/credit/ +IDENTITY_DIR = $SERVICEHOME/identities/ +STATE_DIR = $SERVICEHOME/persistence/ +UPDATE_DIR = $SERVICEHOME/updates/ +@UNIXONLY@ PORT = 2094 +HOSTNAME = localhost +HOME = $SERVICEHOME +CONFIG = $DEFAULTCONFIG +BINARY = gnunet-service-fs +ACCEPT_FROM = 127.0.0.1; +ACCEPT_FROM6 = ::1; + +DELAY = YES +CONTENT_CACHING = YES +CONTENT_PUSHING = YES + +UNIXPATH = /tmp/gnunet-service-fs.sock +UNIX_MATCH_UID = NO +UNIX_MATCH_GID = YES +# DISABLE_SOCKET_FORWARDING = NO +# DEBUG = YES +MAX_PENDING_REQUESTS = 65536 +# Maximum frequency we're allowed to poll the datastore +# for content for migration (can be used to reduce +# GNUnet's disk-IO rate) +MIN_MIGRATION_DELAY = 100 ms +EXPECTED_NEIGHBOUR_COUNT = 128 + + diff --git a/src/fs/fs.h b/src/fs/fs.h new file mode 100644 index 0000000..059b892 --- /dev/null +++ b/src/fs/fs.h @@ -0,0 +1,332 @@ +/* + This file is part of GNUnet. + (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/fs.h + * @brief definitions for the entire fs module + * @author Igor Wronsky, Christian Grothoff + */ +#ifndef FS_H +#define FS_H + +#include "gnunet_constants.h" +#include "gnunet_datastore_service.h" +#include "gnunet_dht_service.h" +#include "gnunet_fs_service.h" +#include "gnunet_block_lib.h" +#include "block_fs.h" + + +/** + * Size of the individual blocks used for file-sharing. + */ +#define DBLOCK_SIZE (32*1024) + +/** + * Blocksize to use when hashing files for indexing (blocksize for IO, + * not for the DBlocks). Larger blocksizes can be more efficient but + * will be more disruptive as far as the scheduler is concerned. + */ +#define HASHING_BLOCKSIZE (1024 * 128) + + +/** + * @brief content hash key + */ +struct ContentHashKey +{ + /** + * Hash of the original content, used for encryption. + */ + GNUNET_HashCode key; + + /** + * Hash of the encrypted content, used for querying. + */ + GNUNET_HashCode query; +}; + + +GNUNET_NETWORK_STRUCT_BEGIN + +/** + * Message sent from a GNUnet (fs) publishing activity to the + * gnunet-fs-service to initiate indexing of a file. The service is + * supposed to check if the specified file is available and has the + * same cryptographic hash. It should then respond with either a + * confirmation or a denial. + * + * On OSes where this works, it is considered acceptable if the + * service only checks that the path, device and inode match (it can + * then be assumed that the hash will also match without actually + * computing it; this is an optimization that should be safe given + * that the client is not our adversary). + */ +struct IndexStartMessage +{ + + /** + * Message type will be GNUNET_MESSAGE_TYPE_FS_INDEX_START. + */ + struct GNUNET_MessageHeader header; + + /** + * For alignment. + */ + uint32_t reserved GNUNET_PACKED; + + /** + * ID of device containing the file, as seen by the client. This + * device ID is obtained using a call like "statvfs" (and converting + * the "f_fsid" field to a 32-bit big-endian number). Use 0 if the + * OS does not support this, in which case the service must do a + * full hash recomputation. + */ + uint64_t device GNUNET_PACKED; + + /** + * Inode of the file on the given device, as seen by the client + * ("st_ino" field from "struct stat"). Use 0 if the OS does not + * support this, in which case the service must do a full hash + * recomputation. + */ + uint64_t inode GNUNET_PACKED; + + /** + * Hash of the file that we would like to index. + */ + GNUNET_HashCode file_id; + + /* this is followed by a 0-terminated + * filename of a file with the hash + * "file_id" as seen by the client */ + +}; + + +/** + * Message send by FS service in response to a request + * asking for a list of all indexed files. + */ +struct IndexInfoMessage +{ + /** + * Message type will be + * GNUNET_MESSAGE_TYPE_FS_INDEX_LIST_ENTRY. + */ + struct GNUNET_MessageHeader header; + + /** + * Always zero. + */ + uint32_t reserved GNUNET_PACKED; + + /** + * Hash of the indexed file. + */ + GNUNET_HashCode file_id; + + /* this is followed by a 0-terminated + * filename of a file with the hash + * "file_id" as seen by the client */ + +}; + + +/** + * Message sent from a GNUnet (fs) unindexing activity to the + * gnunet-service-fs to indicate that a file will be unindexed. The + * service is supposed to remove the file from the list of indexed + * files and response with a confirmation message (even if the file + * was already not on the list). + */ +struct UnindexMessage +{ + + /** + * Message type will be + * GNUNET_MESSAGE_TYPE_FS_UNINDEX. + */ + struct GNUNET_MessageHeader header; + + /** + * Always zero. + */ + uint32_t reserved GNUNET_PACKED; + + /** + * Hash of the file that we will unindex. + */ + GNUNET_HashCode file_id; + +}; + + +/** + * No options. + */ +#define SEARCH_MESSAGE_OPTION_NONE 0 + +/** + * Only search the local datastore (no network) + */ +#define SEARCH_MESSAGE_OPTION_LOOPBACK_ONLY 1 + +/** + * Request is too large to fit in 64k format. The list of + * already-known search results will be continued in another message + * for the same type/query/target and additional already-known results + * following this one). + */ +#define SEARCH_MESSAGE_OPTION_CONTINUED 2 + + +/** + * Message sent from a GNUnet (fs) search activity to the + * gnunet-service-fs to start a search. + */ +struct SearchMessage +{ + + /** + * Message type will be + * GNUNET_MESSAGE_TYPE_FS_START_SEARCH. + */ + struct GNUNET_MessageHeader header; + + /** + * Bitmask with options. Zero for no options, one for + * loopback-only, two for 'to be continued' (with a second search + * message for the same type/query/target and additional + * already-known results following this one). See + * SEARCH_MESSAGE_OPTION_ defines. + * + * Other bits are currently not defined. + */ + uint32_t options GNUNET_PACKED; + + /** + * Type of the content that we're looking for. + */ + uint32_t type GNUNET_PACKED; + + /** + * Desired anonymity level, big-endian. + */ + uint32_t anonymity_level GNUNET_PACKED; + + /** + * If the request is for a DBLOCK or IBLOCK, this is the identity of + * the peer that is known to have a response. Set to all-zeros if + * such a target is not known (note that even if OUR anonymity + * level is >0 we may happen to know the responder's identity; + * nevertheless, we should probably not use it for a DHT-lookup + * or similar blunt actions in order to avoid exposing ourselves). + * <p> + * If the request is for an SBLOCK, this is the identity of the + * pseudonym to which the SBLOCK belongs. + * <p> + * If the request is for a KBLOCK, "target" must be all zeros. + */ + GNUNET_HashCode target; + + /** + * Hash of the keyword (aka query) for KBLOCKs; Hash of + * the CHK-encoded block for DBLOCKS and IBLOCKS (aka query) + * and hash of the identifier XORed with the target for + * SBLOCKS (aka query). + */ + GNUNET_HashCode query; + + /* this is followed by the hash codes of already-known + * results (which should hence be excluded from what + * the service returns); naturally, this only applies + * to queries that can have multiple results, such as + * those for KBLOCKS (KSK) and SBLOCKS (SKS) */ +}; + + +/** + * Response from FS service with a result for a previous FS search. + * Note that queries for DBLOCKS and IBLOCKS that have received a + * single response are considered done. This message is transmitted + * between peers. + */ +struct PutMessage +{ + + /** + * Message type will be GNUNET_MESSAGE_TYPE_FS_PUT. + */ + struct GNUNET_MessageHeader header; + + /** + * Type of the block (in big endian). Should never be zero. + */ + uint32_t type GNUNET_PACKED; + + /** + * When does this result expire? + */ + struct GNUNET_TIME_AbsoluteNBO expiration; + + /* this is followed by the actual encrypted content */ + +}; + +/** + * Response from FS service with a result for a previous FS search. + * Note that queries for DBLOCKS and IBLOCKS that have received a + * single response are considered done. This message is transmitted + * between the service and a client. + */ +struct ClientPutMessage +{ + + /** + * Message type will be GNUNET_MESSAGE_TYPE_FS_PUT. + */ + struct GNUNET_MessageHeader header; + + /** + * Type of the block (in big endian). Should never be zero. + */ + uint32_t type GNUNET_PACKED; + + /** + * When does this result expire? + */ + struct GNUNET_TIME_AbsoluteNBO expiration; + + /** + * When was the last time we've tried to download this block? + * (FOREVER if unknown/not relevant) + */ + struct GNUNET_TIME_AbsoluteNBO last_transmission; + + /* this is followed by the actual encrypted content */ + +}; +GNUNET_NETWORK_STRUCT_END + + +#endif + +/* end of fs.h */ diff --git a/src/fs/fs_api.c b/src/fs/fs_api.c new file mode 100644 index 0000000..1df9b2e --- /dev/null +++ b/src/fs/fs_api.c @@ -0,0 +1,2824 @@ +/* + This file is part of GNUnet. + (C) 2001, 2002, 2003, 2004, 2005, 2006, 2008, 2009, 2010, 2011 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/fs_api.c + * @brief main FS functions (master initialization, serialization, deserialization, shared code) + * @author Christian Grothoff + */ + +#include "platform.h" +#include "gnunet_util_lib.h" +#include "gnunet_fs_service.h" +#include "fs_api.h" +#include "fs_tree.h" + + +/** + * Start the given job (send signal, remove from pending queue, update + * counters and state). + * + * @param qe job to start + */ +static void +start_job (struct GNUNET_FS_QueueEntry *qe) +{ + GNUNET_assert (NULL == qe->client); + qe->client = GNUNET_CLIENT_connect ("fs", qe->h->cfg); + if (qe->client == NULL) + { + GNUNET_break (0); + return; + } + qe->start (qe->cls, qe->client); + qe->start_times++; + qe->h->active_blocks += qe->blocks; + qe->start_time = GNUNET_TIME_absolute_get (); + GNUNET_CONTAINER_DLL_remove (qe->h->pending_head, qe->h->pending_tail, qe); + GNUNET_CONTAINER_DLL_insert_after (qe->h->running_head, qe->h->running_tail, + qe->h->running_tail, qe); +} + + +/** + * Stop the given job (send signal, remove from active queue, update + * counters and state). + * + * @param qe job to stop + */ +static void +stop_job (struct GNUNET_FS_QueueEntry *qe) +{ + qe->client = NULL; + qe->stop (qe->cls); + qe->h->active_downloads--; + qe->h->active_blocks -= qe->blocks; + qe->run_time = + GNUNET_TIME_relative_add (qe->run_time, + GNUNET_TIME_absolute_get_duration + (qe->start_time)); + GNUNET_CONTAINER_DLL_remove (qe->h->running_head, qe->h->running_tail, qe); + GNUNET_CONTAINER_DLL_insert_after (qe->h->pending_head, qe->h->pending_tail, + qe->h->pending_tail, qe); +} + + +/** + * Process the jobs in the job queue, possibly starting some + * and stopping others. + * + * @param cls the 'struct GNUNET_FS_Handle' + * @param tc scheduler context + */ +static void +process_job_queue (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct GNUNET_FS_Handle *h = cls; + struct GNUNET_FS_QueueEntry *qe; + struct GNUNET_FS_QueueEntry *next; + struct GNUNET_TIME_Relative run_time; + struct GNUNET_TIME_Relative restart_at; + struct GNUNET_TIME_Relative rst; + struct GNUNET_TIME_Absolute end_time; + + h->queue_job = GNUNET_SCHEDULER_NO_TASK; + next = h->pending_head; + while (NULL != (qe = next)) + { + next = qe->next; + if (h->running_head == NULL) + { + start_job (qe); + continue; + } + if ((qe->blocks + h->active_blocks <= h->max_parallel_requests) && + (h->active_downloads + 1 <= h->max_parallel_downloads)) + { + start_job (qe); + continue; + } + } + if (h->pending_head == NULL) + return; /* no need to stop anything */ + restart_at = GNUNET_TIME_UNIT_FOREVER_REL; + next = h->running_head; + while (NULL != (qe = next)) + { + next = qe->next; + run_time = + GNUNET_TIME_relative_multiply (h->avg_block_latency, + qe->blocks * qe->start_times); + end_time = GNUNET_TIME_absolute_add (qe->start_time, run_time); + rst = GNUNET_TIME_absolute_get_remaining (end_time); + restart_at = GNUNET_TIME_relative_min (rst, restart_at); + if (rst.rel_value > 0) + continue; + stop_job (qe); + } + h->queue_job = + GNUNET_SCHEDULER_add_delayed (restart_at, &process_job_queue, h); +} + + +/** + * Add a job to the queue. + * + * @param h handle to the overall FS state + * @param start function to call to begin the job + * @param stop function to call to pause the job, or on dequeue (if the job was running) + * @param cls closure for start and stop + * @param blocks number of blocks this jobs uses + * @return queue handle + */ +struct GNUNET_FS_QueueEntry * +GNUNET_FS_queue_ (struct GNUNET_FS_Handle *h, GNUNET_FS_QueueStart start, + GNUNET_FS_QueueStop stop, void *cls, unsigned int blocks) +{ + struct GNUNET_FS_QueueEntry *qe; + + qe = GNUNET_malloc (sizeof (struct GNUNET_FS_QueueEntry)); + qe->h = h; + qe->start = start; + qe->stop = stop; + qe->cls = cls; + qe->queue_time = GNUNET_TIME_absolute_get (); + qe->blocks = blocks; + GNUNET_CONTAINER_DLL_insert_after (h->pending_head, h->pending_tail, + h->pending_tail, qe); + if (h->queue_job != GNUNET_SCHEDULER_NO_TASK) + GNUNET_SCHEDULER_cancel (h->queue_job); + h->queue_job = GNUNET_SCHEDULER_add_now (&process_job_queue, h); + return qe; +} + + +/** + * Dequeue a job from the queue. + * @param qh handle for the job + */ +void +GNUNET_FS_dequeue_ (struct GNUNET_FS_QueueEntry *qh) +{ + struct GNUNET_FS_Handle *h; + + h = qh->h; + if (qh->client != NULL) + stop_job (qh); + GNUNET_CONTAINER_DLL_remove (h->pending_head, h->pending_tail, qh); + GNUNET_free (qh); + if (h->queue_job != GNUNET_SCHEDULER_NO_TASK) + GNUNET_SCHEDULER_cancel (h->queue_job); + h->queue_job = GNUNET_SCHEDULER_add_now (&process_job_queue, h); +} + + +/** + * Create a top-level activity entry. + * + * @param h global fs handle + * @param ssf suspend signal function to use + * @param ssf_cls closure for ssf + * @return fresh top-level activity handle + */ +struct TopLevelActivity * +GNUNET_FS_make_top (struct GNUNET_FS_Handle *h, SuspendSignalFunction ssf, + void *ssf_cls) +{ + struct TopLevelActivity *ret; + + ret = GNUNET_malloc (sizeof (struct TopLevelActivity)); + ret->ssf = ssf; + ret->ssf_cls = ssf_cls; + GNUNET_CONTAINER_DLL_insert (h->top_head, h->top_tail, ret); + return ret; +} + + +/** + * Destroy a top-level activity entry. + * + * @param h global fs handle + * @param top top level activity entry + */ +void +GNUNET_FS_end_top (struct GNUNET_FS_Handle *h, struct TopLevelActivity *top) +{ + GNUNET_CONTAINER_DLL_remove (h->top_head, h->top_tail, top); + GNUNET_free (top); +} + + + +/** + * Closure for "data_reader_file". + */ +struct FileInfo +{ + /** + * Name of the file to read. + */ + char *filename; + + /** + * File descriptor, NULL if it has not yet been opened. + */ + struct GNUNET_DISK_FileHandle *fd; +}; + + +/** + * Function that provides data by reading from a file. + * + * @param cls closure (points to the file information) + * @param offset offset to read from; it is possible + * that the caller might need to go backwards + * a bit at times + * @param max maximum number of bytes that should be + * copied to buf; readers are not allowed + * to provide less data unless there is an error; + * a value of "0" will be used at the end to allow + * the reader to clean up its internal state + * @param buf where the reader should write the data + * @param emsg location for the reader to store an error message + * @return number of bytes written, usually "max", 0 on error + */ +size_t +GNUNET_FS_data_reader_file_ (void *cls, uint64_t offset, size_t max, void *buf, + char **emsg) +{ + struct FileInfo *fi = cls; + ssize_t ret; + + if (max == 0) + { + if (fi->fd != NULL) + GNUNET_DISK_file_close (fi->fd); + GNUNET_free (fi->filename); + GNUNET_free (fi); + return 0; + } + if (fi->fd == NULL) + { + fi->fd = + GNUNET_DISK_file_open (fi->filename, GNUNET_DISK_OPEN_READ, + GNUNET_DISK_PERM_NONE); + if (fi->fd == NULL) + { + GNUNET_asprintf (emsg, _("Could not open file `%s': %s"), fi->filename, + STRERROR (errno)); + return 0; + } + } + GNUNET_DISK_file_seek (fi->fd, offset, GNUNET_DISK_SEEK_SET); + ret = GNUNET_DISK_file_read (fi->fd, buf, max); + if (ret == -1) + { + GNUNET_asprintf (emsg, _("Could not read file `%s': %s"), fi->filename, + STRERROR (errno)); + return 0; + } + if (ret != max) + { + GNUNET_asprintf (emsg, _("Short read reading from file `%s'!"), + fi->filename); + return 0; + } + return max; +} + + +/** + * Create the closure for the 'GNUNET_FS_data_reader_file_' callback. + * + * @param filename file to read + * @return closure to use, NULL on error + */ +void * +GNUNET_FS_make_file_reader_context_ (const char *filename) +{ + struct FileInfo *fi; + + fi = GNUNET_malloc (sizeof (struct FileInfo)); + fi->filename = GNUNET_STRINGS_filename_expand (filename); + if (fi->filename == NULL) + { + GNUNET_free (fi); + return NULL; + } + return fi; +} + + +/** + * Function that provides data by copying from a buffer. + * + * @param cls closure (points to the buffer) + * @param offset offset to read from; it is possible + * that the caller might need to go backwards + * a bit at times + * @param max maximum number of bytes that should be + * copied to buf; readers are not allowed + * to provide less data unless there is an error; + * a value of "0" will be used at the end to allow + * the reader to clean up its internal state + * @param buf where the reader should write the data + * @param emsg location for the reader to store an error message + * @return number of bytes written, usually "max", 0 on error + */ +size_t +GNUNET_FS_data_reader_copy_ (void *cls, uint64_t offset, size_t max, void *buf, + char **emsg) +{ + char *data = cls; + + if (max == 0) + { + GNUNET_free_non_null (data); + return 0; + } + memcpy (buf, &data[offset], max); + return max; +} + + +/** + * Return the full filename where we would store state information + * (for serialization/deserialization). + * + * @param h master context + * @param ext component of the path + * @param ent entity identifier (or emtpy string for the directory) + * @return NULL on error + */ +static char * +get_serialization_file_name (struct GNUNET_FS_Handle *h, const char *ext, + const char *ent) +{ + char *basename; + char *ret; + + if (0 == (h->flags & GNUNET_FS_FLAGS_PERSISTENCE)) + return NULL; /* persistence not requested */ + if (GNUNET_OK != + GNUNET_CONFIGURATION_get_value_filename (h->cfg, "fs", "STATE_DIR", + &basename)) + return NULL; + GNUNET_asprintf (&ret, "%s%s%s%s%s%s%s", basename, DIR_SEPARATOR_STR, + h->client_name, DIR_SEPARATOR_STR, ext, DIR_SEPARATOR_STR, + ent); + GNUNET_free (basename); + return ret; +} + + +/** + * Return the full filename where we would store state information + * (for serialization/deserialization) that is associated with a + * parent operation. + * + * @param h master context + * @param ext component of the path + * @param uni name of the parent operation + * @param ent entity identifier (or emtpy string for the directory) + * @return NULL on error + */ +static char * +get_serialization_file_name_in_dir (struct GNUNET_FS_Handle *h, const char *ext, + const char *uni, const char *ent) +{ + char *basename; + char *ret; + + if (0 == (h->flags & GNUNET_FS_FLAGS_PERSISTENCE)) + return NULL; /* persistence not requested */ + if (GNUNET_OK != + GNUNET_CONFIGURATION_get_value_filename (h->cfg, "fs", "STATE_DIR", + &basename)) + return NULL; + GNUNET_asprintf (&ret, "%s%s%s%s%s%s%s.dir%s%s", basename, DIR_SEPARATOR_STR, + h->client_name, DIR_SEPARATOR_STR, ext, DIR_SEPARATOR_STR, + uni, DIR_SEPARATOR_STR, ent); + GNUNET_free (basename); + return ret; +} + + +/** + * Return a read handle for deserialization. + * + * @param h master context + * @param ext component of the path + * @param ent entity identifier (or emtpy string for the directory) + * @return NULL on error + */ +static struct GNUNET_BIO_ReadHandle * +get_read_handle (struct GNUNET_FS_Handle *h, const char *ext, const char *ent) +{ + char *fn; + struct GNUNET_BIO_ReadHandle *ret; + + fn = get_serialization_file_name (h, ext, ent); + if (fn == NULL) + return NULL; + ret = GNUNET_BIO_read_open (fn); + GNUNET_free (fn); + return ret; +} + + +/** + * Return a write handle for serialization. + * + * @param h master context + * @param ext component of the path + * @param ent entity identifier (or emtpy string for the directory) + * @return NULL on error + */ +static struct GNUNET_BIO_WriteHandle * +get_write_handle (struct GNUNET_FS_Handle *h, const char *ext, const char *ent) +{ + char *fn; + struct GNUNET_BIO_WriteHandle *ret; + + fn = get_serialization_file_name (h, ext, ent); + if (fn == NULL) + { + return NULL; + } + ret = GNUNET_BIO_write_open (fn); + if (ret == NULL) + GNUNET_break (0); + GNUNET_free (fn); + return ret; +} + + +/** + * Return a write handle for serialization. + * + * @param h master context + * @param ext component of the path + * @param uni name of parent + * @param ent entity identifier (or emtpy string for the directory) + * @return NULL on error + */ +static struct GNUNET_BIO_WriteHandle * +get_write_handle_in_dir (struct GNUNET_FS_Handle *h, const char *ext, + const char *uni, const char *ent) +{ + char *fn; + struct GNUNET_BIO_WriteHandle *ret; + + fn = get_serialization_file_name_in_dir (h, ext, uni, ent); + if (fn == NULL) + return NULL; + ret = GNUNET_BIO_write_open (fn); + GNUNET_free (fn); + return ret; +} + + +/** + * Remove serialization/deserialization file from disk. + * + * @param h master context + * @param ext component of the path + * @param ent entity identifier + */ +void +GNUNET_FS_remove_sync_file_ (struct GNUNET_FS_Handle *h, const char *ext, + const char *ent) +{ + char *filename; + + if ((NULL == ent) || (0 == strlen (ent))) + { + GNUNET_break (0); + return; + } + filename = get_serialization_file_name (h, ext, ent); + if (filename != NULL) + { + if (0 != UNLINK (filename)) + GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", filename); + GNUNET_free (filename); + } +} + + +/** + * Remove serialization/deserialization file from disk. + * + * @param h master context + * @param ext component of the path + * @param uni parent name + * @param ent entity identifier + */ +static void +remove_sync_file_in_dir (struct GNUNET_FS_Handle *h, const char *ext, + const char *uni, const char *ent) +{ + char *filename; + + if ((NULL == ent) || (0 == strlen (ent))) + { + GNUNET_break (0); + return; + } + filename = get_serialization_file_name_in_dir (h, ext, uni, ent); + if (filename != NULL) + { + if (0 != UNLINK (filename)) + GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", filename); + GNUNET_free (filename); + } +} + + +/** + * Remove serialization/deserialization directory from disk. + * + * @param h master context + * @param ext component of the path + * @param uni unique name of parent + */ +void +GNUNET_FS_remove_sync_dir_ (struct GNUNET_FS_Handle *h, const char *ext, + const char *uni) +{ + char *dn; + + if (uni == NULL) + return; + dn = get_serialization_file_name_in_dir (h, ext, uni, ""); + if (dn == NULL) + return; + if ((GNUNET_OK == GNUNET_DISK_directory_test (dn)) && + (GNUNET_OK != GNUNET_DISK_directory_remove (dn))) + GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "rmdir", dn); + GNUNET_free (dn); +} + + +/** + * Serialize a 'start_time'. Since we use start-times to + * calculate the duration of some operation, we actually + * do not serialize the absolute time but the (relative) + * duration since the start time. When we then + * deserialize the start time, we take the current time and + * subtract that duration so that we get again an absolute + * time stamp that will result in correct performance + * calculations. + * + * @param wh handle for writing + * @param timestamp time to serialize + * @return GNUNET_OK on success + */ +static int +write_start_time (struct GNUNET_BIO_WriteHandle *wh, + struct GNUNET_TIME_Absolute timestamp) +{ + struct GNUNET_TIME_Relative dur; + + dur = GNUNET_TIME_absolute_get_duration (timestamp); + return GNUNET_BIO_write_int64 (wh, dur.rel_value); +} + + +/** + * Serialize a 'start_time'. Since we use start-times to + * calculate the duration of some operation, we actually + * do not serialize the absolute time but the (relative) + * duration since the start time. When we then + * deserialize the start time, we take the current time and + * subtract that duration so that we get again an absolute + * time stamp that will result in correct performance + * calculations. + * + * @param rh handle for reading + * @param timestamp where to write the deserialized timestamp + * @return GNUNET_OK on success + */ +static int +read_start_time (struct GNUNET_BIO_ReadHandle *rh, + struct GNUNET_TIME_Absolute *timestamp) +{ + struct GNUNET_TIME_Relative dur; + + if (GNUNET_OK != GNUNET_BIO_read_int64 (rh, &dur.rel_value)) + return GNUNET_SYSERR; + *timestamp = GNUNET_TIME_absolute_subtract (GNUNET_TIME_absolute_get (), dur); + return GNUNET_OK; +} + + +/** + * Using the given serialization filename, try to deserialize + * the file-information tree associated with it. + * + * @param h master context + * @param filename name of the file (without directory) with + * the infromation + * @return NULL on error + */ +static struct GNUNET_FS_FileInformation * +deserialize_file_information (struct GNUNET_FS_Handle *h, const char *filename); + + +/** + * Using the given serialization filename, try to deserialize + * the file-information tree associated with it. + * + * @param h master context + * @param fn name of the file (without directory) with + * the infromation + * @param rh handle for reading + * @return NULL on error + */ +static struct GNUNET_FS_FileInformation * +deserialize_fi_node (struct GNUNET_FS_Handle *h, const char *fn, + struct GNUNET_BIO_ReadHandle *rh) +{ + struct GNUNET_FS_FileInformation *ret; + struct GNUNET_FS_FileInformation *nxt; + char b; + char *ksks; + char *chks; + char *filename; + uint32_t dsize; + + if (GNUNET_OK != GNUNET_BIO_read (rh, "status flag", &b, sizeof (b))) + { + GNUNET_break (0); + return NULL; + } + ret = GNUNET_malloc (sizeof (struct GNUNET_FS_FileInformation)); + ret->h = h; + ksks = NULL; + chks = NULL; + filename = NULL; + if ((GNUNET_OK != GNUNET_BIO_read_meta_data (rh, "metadata", &ret->meta)) || + (GNUNET_OK != GNUNET_BIO_read_string (rh, "ksk-uri", &ksks, 32 * 1024)) || + ( (NULL != ksks) && + ( (NULL == (ret->keywords = GNUNET_FS_uri_parse (ksks, NULL))) || + (GNUNET_YES != GNUNET_FS_uri_test_ksk (ret->keywords)) ) ) || + (GNUNET_OK != GNUNET_BIO_read_string (rh, "chk-uri", &chks, 1024)) || + ( (NULL != chks) && + ( (NULL == (ret->chk_uri = GNUNET_FS_uri_parse (chks, NULL))) || + (GNUNET_YES != GNUNET_FS_uri_test_chk (ret->chk_uri))) ) || + (GNUNET_OK != read_start_time (rh, &ret->start_time)) || + (GNUNET_OK != GNUNET_BIO_read_string (rh, "emsg", &ret->emsg, 16 * 1024)) + || (GNUNET_OK != + GNUNET_BIO_read_string (rh, "fn", &ret->filename, 16 * 1024)) || + (GNUNET_OK != + GNUNET_BIO_read_int64 (rh, &ret->bo.expiration_time.abs_value)) || + (GNUNET_OK != GNUNET_BIO_read_int32 (rh, &ret->bo.anonymity_level)) || + (GNUNET_OK != GNUNET_BIO_read_int32 (rh, &ret->bo.content_priority)) || + (GNUNET_OK != GNUNET_BIO_read_int32 (rh, &ret->bo.replication_level))) + { + GNUNET_break (0); + goto cleanup; + } + switch (b) + { + case 0: /* file-insert */ + if (GNUNET_OK != GNUNET_BIO_read_int64 (rh, &ret->data.file.file_size)) + { + GNUNET_break (0); + goto cleanup; + } + ret->is_directory = GNUNET_NO; + ret->data.file.do_index = GNUNET_NO; + ret->data.file.have_hash = GNUNET_NO; + ret->data.file.index_start_confirmed = GNUNET_NO; + if (GNUNET_NO == ret->is_published) + { + if (NULL == ret->filename) + { + ret->data.file.reader = &GNUNET_FS_data_reader_copy_; + ret->data.file.reader_cls = + GNUNET_malloc_large (ret->data.file.file_size); + if (ret->data.file.reader_cls == NULL) + goto cleanup; + if (GNUNET_OK != + GNUNET_BIO_read (rh, "file-data", ret->data.file.reader_cls, + ret->data.file.file_size)) + { + GNUNET_break (0); + goto cleanup; + } + } + else + { + ret->data.file.reader = &GNUNET_FS_data_reader_file_; + ret->data.file.reader_cls = + GNUNET_FS_make_file_reader_context_ (ret->filename); + } + } + break; + case 1: /* file-index, no hash */ + if (NULL == ret->filename) + { + GNUNET_break (0); + goto cleanup; + } + if (GNUNET_OK != GNUNET_BIO_read_int64 (rh, &ret->data.file.file_size)) + { + GNUNET_break (0); + goto cleanup; + } + ret->is_directory = GNUNET_NO; + ret->data.file.do_index = GNUNET_YES; + ret->data.file.have_hash = GNUNET_NO; + ret->data.file.index_start_confirmed = GNUNET_NO; + ret->data.file.reader = &GNUNET_FS_data_reader_file_; + ret->data.file.reader_cls = + GNUNET_FS_make_file_reader_context_ (ret->filename); + break; + case 2: /* file-index-with-hash */ + if (NULL == ret->filename) + { + GNUNET_break (0); + goto cleanup; + } + if ((GNUNET_OK != GNUNET_BIO_read_int64 (rh, &ret->data.file.file_size)) || + (GNUNET_OK != + GNUNET_BIO_read (rh, "fileid", &ret->data.file.file_id, + sizeof (GNUNET_HashCode)))) + { + GNUNET_break (0); + goto cleanup; + } + ret->is_directory = GNUNET_NO; + ret->data.file.do_index = GNUNET_YES; + ret->data.file.have_hash = GNUNET_YES; + ret->data.file.index_start_confirmed = GNUNET_NO; + ret->data.file.reader = &GNUNET_FS_data_reader_file_; + ret->data.file.reader_cls = + GNUNET_FS_make_file_reader_context_ (ret->filename); + break; + case 3: /* file-index-with-hash-confirmed */ + if (NULL == ret->filename) + { + GNUNET_break (0); + goto cleanup; + } + if ((GNUNET_OK != GNUNET_BIO_read_int64 (rh, &ret->data.file.file_size)) || + (GNUNET_OK != + GNUNET_BIO_read (rh, "fileid", &ret->data.file.file_id, + sizeof (GNUNET_HashCode)))) + { + GNUNET_break (0); + goto cleanup; + } + ret->is_directory = GNUNET_NO; + ret->data.file.do_index = GNUNET_YES; + ret->data.file.have_hash = GNUNET_YES; + ret->data.file.index_start_confirmed = GNUNET_YES; + ret->data.file.reader = &GNUNET_FS_data_reader_file_; + ret->data.file.reader_cls = + GNUNET_FS_make_file_reader_context_ (ret->filename); + break; + case 4: /* directory */ + ret->is_directory = GNUNET_YES; + if ((GNUNET_OK != GNUNET_BIO_read_int32 (rh, &dsize)) || + (NULL == (ret->data.dir.dir_data = GNUNET_malloc_large (dsize))) || + (GNUNET_OK != + GNUNET_BIO_read (rh, "dir-data", ret->data.dir.dir_data, dsize)) || + (GNUNET_OK != + GNUNET_BIO_read_string (rh, "ent-filename", &filename, 16 * 1024))) + { + GNUNET_break (0); + goto cleanup; + } + ret->data.dir.dir_size = (uint32_t) dsize; + if (filename != NULL) + { + ret->data.dir.entries = deserialize_file_information (h, filename); + GNUNET_free (filename); + filename = NULL; + nxt = ret->data.dir.entries; + while (nxt != NULL) + { + nxt->dir = ret; + nxt = nxt->next; + } + } + break; + default: + GNUNET_break (0); + goto cleanup; + } + ret->serialization = GNUNET_strdup (fn); + if (GNUNET_OK != + GNUNET_BIO_read_string (rh, "nxt-filename", &filename, 16 * 1024)) + { + GNUNET_break (0); + goto cleanup; + } + if (filename != NULL) + { + ret->next = deserialize_file_information (h, filename); + GNUNET_free (filename); + filename = NULL; + } + GNUNET_free_non_null (ksks); + GNUNET_free_non_null (chks); + return ret; +cleanup: + GNUNET_free_non_null (ksks); + GNUNET_free_non_null (chks); + GNUNET_free_non_null (filename); + GNUNET_FS_file_information_destroy (ret, NULL, NULL); + return NULL; +} + + +/** + * Using the given serialization filename, try to deserialize + * the file-information tree associated with it. + * + * @param h master context + * @param filename name of the file (without directory) with + * the infromation + * @return NULL on error + */ +static struct GNUNET_FS_FileInformation * +deserialize_file_information (struct GNUNET_FS_Handle *h, const char *filename) +{ + struct GNUNET_FS_FileInformation *ret; + struct GNUNET_BIO_ReadHandle *rh; + char *emsg; + + rh = get_read_handle (h, GNUNET_FS_SYNC_PATH_FILE_INFO, filename); + if (rh == NULL) + return NULL; + ret = deserialize_fi_node (h, filename, rh); + if (GNUNET_OK != GNUNET_BIO_read_close (rh, &emsg)) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _("Failed to resume publishing information `%s': %s\n"), + filename, emsg); + GNUNET_free (emsg); + } + if (ret == NULL) + { + if (0 != UNLINK (filename)) + GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", filename); + } + return ret; +} + + +/** + * Given a serialization name (full absolute path), return the + * basename of the file (without the path), which must only + * consist of the 6 random characters. + * + * @param fullname name to extract the basename from + * @return copy of the basename, NULL on error + */ +static char * +get_serialization_short_name (const char *fullname) +{ + const char *end; + const char *nxt; + + end = NULL; + nxt = fullname; + /* FIXME: we could do this faster since we know + * the length of 'end'... */ + while ('\0' != *nxt) + { + if (DIR_SEPARATOR == *nxt) + end = nxt + 1; + nxt++; + } + if ((end == NULL) || (strlen (end) == 0)) + { + GNUNET_break (0); + return NULL; + } + GNUNET_break (6 == strlen (end)); + return GNUNET_strdup (end); +} + + +/** + * Create a new random name for serialization. Also checks if persistence + * is enabled and returns NULL if not. + * + * @param h master context + * @param ext component of the path + * @return NULL on errror + */ +static char * +make_serialization_file_name (struct GNUNET_FS_Handle *h, const char *ext) +{ + char *fn; + char *dn; + char *ret; + + if (0 == (h->flags & GNUNET_FS_FLAGS_PERSISTENCE)) + return NULL; /* persistence not requested */ + dn = get_serialization_file_name (h, ext, ""); + if (dn == NULL) + return NULL; + if (GNUNET_OK != GNUNET_DISK_directory_create_for_file (dn)) + { + GNUNET_free (dn); + return NULL; + } + fn = GNUNET_DISK_mktemp (dn); + GNUNET_free (dn); + if (fn == NULL) + return NULL; /* epic fail */ + ret = get_serialization_short_name (fn); + GNUNET_free (fn); + return ret; +} + + +/** + * Create a new random name for serialization. Also checks if persistence + * is enabled and returns NULL if not. + * + * @param h master context + * @param ext component of the path + * @param uni name of parent + * @return NULL on errror + */ +static char * +make_serialization_file_name_in_dir (struct GNUNET_FS_Handle *h, + const char *ext, const char *uni) +{ + char *fn; + char *dn; + char *ret; + + if (0 == (h->flags & GNUNET_FS_FLAGS_PERSISTENCE)) + return NULL; /* persistence not requested */ + dn = get_serialization_file_name_in_dir (h, ext, uni, ""); + if (dn == NULL) + return NULL; + if (GNUNET_OK != GNUNET_DISK_directory_create_for_file (dn)) + { + GNUNET_free (dn); + return NULL; + } + fn = GNUNET_DISK_mktemp (dn); + GNUNET_free (dn); + if (fn == NULL) + return NULL; /* epic fail */ + ret = get_serialization_short_name (fn); + GNUNET_free (fn); + return ret; +} + + +/** + * Copy all of the data from the reader to the write handle. + * + * @param wh write handle + * @param fi file with reader + * @return GNUNET_OK on success + */ +static int +copy_from_reader (struct GNUNET_BIO_WriteHandle *wh, + struct GNUNET_FS_FileInformation *fi) +{ + char buf[32 * 1024]; + uint64_t off; + size_t ret; + size_t left; + char *emsg; + + emsg = NULL; + off = 0; + while (off < fi->data.file.file_size) + { + left = GNUNET_MIN (sizeof (buf), fi->data.file.file_size - off); + ret = + fi->data.file.reader (fi->data.file.reader_cls, off, left, buf, &emsg); + if (ret == 0) + { + GNUNET_free (emsg); + return GNUNET_SYSERR; + } + if (GNUNET_OK != GNUNET_BIO_write (wh, buf, ret)) + return GNUNET_SYSERR; + off += ret; + } + return GNUNET_OK; +} + + +/** + * Create a temporary file on disk to store the current + * state of "fi" in. + * + * @param fi file information to sync with disk + */ +void +GNUNET_FS_file_information_sync_ (struct GNUNET_FS_FileInformation *fi) +{ + char *fn; + struct GNUNET_BIO_WriteHandle *wh; + char b; + char *ksks; + char *chks; + + if (NULL == fi->serialization) + fi->serialization = + make_serialization_file_name (fi->h, GNUNET_FS_SYNC_PATH_FILE_INFO); + if (NULL == fi->serialization) + return; + wh = get_write_handle (fi->h, GNUNET_FS_SYNC_PATH_FILE_INFO, + fi->serialization); + if (wh == NULL) + { + GNUNET_free (fi->serialization); + fi->serialization = NULL; + return; + } + if (GNUNET_YES == fi->is_directory) + b = 4; + else if (GNUNET_YES == fi->data.file.index_start_confirmed) + b = 3; + else if (GNUNET_YES == fi->data.file.have_hash) + b = 2; + else if (GNUNET_YES == fi->data.file.do_index) + b = 1; + else + b = 0; + if (fi->keywords != NULL) + ksks = GNUNET_FS_uri_to_string (fi->keywords); + else + ksks = NULL; + if (fi->chk_uri != NULL) + chks = GNUNET_FS_uri_to_string (fi->chk_uri); + else + chks = NULL; + if ((GNUNET_OK != GNUNET_BIO_write (wh, &b, sizeof (b))) || + (GNUNET_OK != GNUNET_BIO_write_meta_data (wh, fi->meta)) || + (GNUNET_OK != GNUNET_BIO_write_string (wh, ksks)) || + (GNUNET_OK != GNUNET_BIO_write_string (wh, chks)) || + (GNUNET_OK != write_start_time (wh, fi->start_time)) || + (GNUNET_OK != GNUNET_BIO_write_string (wh, fi->emsg)) || + (GNUNET_OK != GNUNET_BIO_write_string (wh, fi->filename)) || + (GNUNET_OK != + GNUNET_BIO_write_int64 (wh, fi->bo.expiration_time.abs_value)) || + (GNUNET_OK != GNUNET_BIO_write_int32 (wh, fi->bo.anonymity_level)) || + (GNUNET_OK != GNUNET_BIO_write_int32 (wh, fi->bo.content_priority)) || + (GNUNET_OK != GNUNET_BIO_write_int32 (wh, fi->bo.replication_level))) + { + GNUNET_break (0); + goto cleanup; + } + GNUNET_free_non_null (chks); + chks = NULL; + GNUNET_free_non_null (ksks); + ksks = NULL; + + switch (b) + { + case 0: /* file-insert */ + if (GNUNET_OK != GNUNET_BIO_write_int64 (wh, fi->data.file.file_size)) + { + GNUNET_break (0); + goto cleanup; + } + if ((GNUNET_NO == fi->is_published) && (NULL == fi->filename)) + if (GNUNET_OK != copy_from_reader (wh, fi)) + { + GNUNET_break (0); + goto cleanup; + } + break; + case 1: /* file-index, no hash */ + if (NULL == fi->filename) + { + GNUNET_break (0); + goto cleanup; + } + if (GNUNET_OK != GNUNET_BIO_write_int64 (wh, fi->data.file.file_size)) + { + GNUNET_break (0); + goto cleanup; + } + break; + case 2: /* file-index-with-hash */ + case 3: /* file-index-with-hash-confirmed */ + if (NULL == fi->filename) + { + GNUNET_break (0); + goto cleanup; + } + if ((GNUNET_OK != GNUNET_BIO_write_int64 (wh, fi->data.file.file_size)) || + (GNUNET_OK != + GNUNET_BIO_write (wh, &fi->data.file.file_id, + sizeof (GNUNET_HashCode)))) + { + GNUNET_break (0); + goto cleanup; + } + break; + case 4: /* directory */ + if ((GNUNET_OK != GNUNET_BIO_write_int32 (wh, fi->data.dir.dir_size)) || + (GNUNET_OK != + GNUNET_BIO_write (wh, fi->data.dir.dir_data, + (uint32_t) fi->data.dir.dir_size)) || + (GNUNET_OK != + GNUNET_BIO_write_string (wh, + (fi->data.dir.entries == + NULL) ? NULL : fi->data.dir. + entries->serialization))) + { + GNUNET_break (0); + goto cleanup; + } + break; + default: + GNUNET_assert (0); + goto cleanup; + } + if (GNUNET_OK != + GNUNET_BIO_write_string (wh, + (fi->next != + NULL) ? fi->next->serialization : NULL)) + { + GNUNET_break (0); + goto cleanup; + } + if (GNUNET_OK != GNUNET_BIO_write_close (wh)) + { + wh = NULL; + GNUNET_break (0); + goto cleanup; + } + return; /* done! */ +cleanup: + if (wh != NULL) + (void) GNUNET_BIO_write_close (wh); + GNUNET_free_non_null (chks); + GNUNET_free_non_null (ksks); + fn = get_serialization_file_name (fi->h, GNUNET_FS_SYNC_PATH_FILE_INFO, + fi->serialization); + if (NULL != fn) + { + if (0 != UNLINK (fn)) + GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", fn); + GNUNET_free (fn); + } + GNUNET_free (fi->serialization); + fi->serialization = NULL; +} + + + +/** + * Find the entry in the file information struct where the + * serialization filename matches the given name. + * + * @param pos file information to search + * @param srch filename to search for + * @return NULL if srch was not found in this subtree + */ +static struct GNUNET_FS_FileInformation * +find_file_position (struct GNUNET_FS_FileInformation *pos, const char *srch) +{ + struct GNUNET_FS_FileInformation *r; + + while (pos != NULL) + { + if (0 == strcmp (srch, pos->serialization)) + return pos; + if (pos->is_directory == GNUNET_YES) + { + r = find_file_position (pos->data.dir.entries, srch); + if (r != NULL) + return r; + } + pos = pos->next; + } + return NULL; +} + + +/** + * Signal the FS's progress function that we are resuming + * an upload. + * + * @param cls closure (of type "struct GNUNET_FS_PublishContext*") + * @param fi the entry in the publish-structure + * @param length length of the file or directory + * @param meta metadata for the file or directory (can be modified) + * @param uri pointer to the keywords that will be used for this entry (can be modified) + * @param bo block options (can be modified) + * @param do_index should we index? + * @param client_info pointer to client context set upon creation (can be modified) + * @return GNUNET_OK to continue (always) + */ +static int +fip_signal_resume (void *cls, struct GNUNET_FS_FileInformation *fi, + uint64_t length, struct GNUNET_CONTAINER_MetaData *meta, + struct GNUNET_FS_Uri **uri, + struct GNUNET_FS_BlockOptions *bo, int *do_index, + void **client_info) +{ + struct GNUNET_FS_PublishContext *pc = cls; + struct GNUNET_FS_ProgressInfo pi; + + if (GNUNET_YES == pc->skip_next_fi_callback) + { + pc->skip_next_fi_callback = GNUNET_NO; + return GNUNET_OK; + } + pi.status = GNUNET_FS_STATUS_PUBLISH_RESUME; + pi.value.publish.specifics.resume.message = pc->fi->emsg; + pi.value.publish.specifics.resume.chk_uri = pc->fi->chk_uri; + *client_info = GNUNET_FS_publish_make_status_ (&pi, pc, fi, 0); + if (GNUNET_YES == GNUNET_FS_meta_data_test_for_directory (meta)) + { + /* process entries in directory */ + pc->skip_next_fi_callback = GNUNET_YES; + GNUNET_FS_file_information_inspect (fi, &fip_signal_resume, pc); + } + return GNUNET_OK; +} + + +/** + * Function called with a filename of serialized publishing operation + * to deserialize. + * + * @param cls the 'struct GNUNET_FS_Handle*' + * @param filename complete filename (absolute path) + * @return GNUNET_OK (continue to iterate) + */ +static int +deserialize_publish_file (void *cls, const char *filename) +{ + struct GNUNET_FS_Handle *h = cls; + struct GNUNET_BIO_ReadHandle *rh; + struct GNUNET_FS_PublishContext *pc; + int32_t options; + int32_t all_done; + char *fi_root; + char *ns; + char *fi_pos; + char *emsg; + + pc = GNUNET_malloc (sizeof (struct GNUNET_FS_PublishContext)); + pc->h = h; + pc->serialization = get_serialization_short_name (filename); + fi_root = NULL; + fi_pos = NULL; + ns = NULL; + rh = GNUNET_BIO_read_open (filename); + if (rh == NULL) + { + GNUNET_break (0); + goto cleanup; + } + if ((GNUNET_OK != GNUNET_BIO_read_string (rh, "publish-nid", &pc->nid, 1024)) + || (GNUNET_OK != + GNUNET_BIO_read_string (rh, "publish-nuid", &pc->nuid, 1024)) || + (GNUNET_OK != GNUNET_BIO_read_int32 (rh, &options)) || + (GNUNET_OK != GNUNET_BIO_read_int32 (rh, &all_done)) || + (GNUNET_OK != + GNUNET_BIO_read_string (rh, "publish-firoot", &fi_root, 128)) || + (GNUNET_OK != GNUNET_BIO_read_string (rh, "publish-fipos", &fi_pos, 128)) + || (GNUNET_OK != GNUNET_BIO_read_string (rh, "publish-ns", &ns, 1024))) + { + GNUNET_break (0); + goto cleanup; + } + pc->options = options; + pc->all_done = all_done; + if (NULL == fi_root) + { + GNUNET_break (0); + goto cleanup; + } + pc->fi = deserialize_file_information (h, fi_root); + if (pc->fi == NULL) + { + GNUNET_break (0); + goto cleanup; + } + if (ns != NULL) + { + pc->namespace = GNUNET_FS_namespace_create (h, ns); + if (pc->namespace == NULL) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _ + ("Failed to recover namespace `%s', cannot resume publishing operation.\n"), + ns); + goto cleanup; + } + } + if ((0 == (pc->options & GNUNET_FS_PUBLISH_OPTION_SIMULATE_ONLY)) && + (GNUNET_YES != pc->all_done)) + { + pc->dsh = GNUNET_DATASTORE_connect (h->cfg); + if (NULL == pc->dsh) + goto cleanup; + } + if (fi_pos != NULL) + { + pc->fi_pos = find_file_position (pc->fi, fi_pos); + GNUNET_free (fi_pos); + fi_pos = NULL; + if (pc->fi_pos == NULL) + { + /* failed to find position for resuming, outch! Will start from root! */ + GNUNET_break (0); + if (pc->all_done != GNUNET_YES) + pc->fi_pos = pc->fi; + } + } + GNUNET_free (fi_root); + fi_root = NULL; + /* generate RESUME event(s) */ + GNUNET_FS_file_information_inspect (pc->fi, &fip_signal_resume, pc); + + /* re-start publishing (if needed)... */ + if (pc->all_done != GNUNET_YES) + { + GNUNET_assert (GNUNET_SCHEDULER_NO_TASK == pc->upload_task); + pc->upload_task = + GNUNET_SCHEDULER_add_with_priority + (GNUNET_SCHEDULER_PRIORITY_BACKGROUND, + &GNUNET_FS_publish_main_, pc); + } + if (GNUNET_OK != GNUNET_BIO_read_close (rh, &emsg)) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _("Failure while resuming publishing operation `%s': %s\n"), + filename, emsg); + GNUNET_free (emsg); + } + GNUNET_free_non_null (ns); + pc->top = GNUNET_FS_make_top (h, &GNUNET_FS_publish_signal_suspend_, pc); + return GNUNET_OK; +cleanup: + GNUNET_free_non_null (pc->nid); + GNUNET_free_non_null (pc->nuid); + GNUNET_free_non_null (fi_root); + GNUNET_free_non_null (fi_pos); + GNUNET_free_non_null (ns); + if ((rh != NULL) && (GNUNET_OK != GNUNET_BIO_read_close (rh, &emsg))) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _("Failed to resume publishing operation `%s': %s\n"), filename, + emsg); + GNUNET_free (emsg); + } + if (pc->fi != NULL) + GNUNET_FS_file_information_destroy (pc->fi, NULL, NULL); + if (0 != UNLINK (filename)) + GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", filename); + GNUNET_free (pc->serialization); + GNUNET_free (pc); + return GNUNET_OK; +} + + +/** + * Synchronize this publishing struct with its mirror + * on disk. Note that all internal FS-operations that change + * publishing structs should already call "sync" internally, + * so this function is likely not useful for clients. + * + * @param pc the struct to sync + */ +void +GNUNET_FS_publish_sync_ (struct GNUNET_FS_PublishContext *pc) +{ + struct GNUNET_BIO_WriteHandle *wh; + + if (NULL == pc->serialization) + pc->serialization = + make_serialization_file_name (pc->h, + GNUNET_FS_SYNC_PATH_MASTER_PUBLISH); + if (NULL == pc->serialization) + return; + if (NULL == pc->fi) + return; + if (NULL == pc->fi->serialization) + { + GNUNET_break (0); + return; + } + wh = get_write_handle (pc->h, GNUNET_FS_SYNC_PATH_MASTER_PUBLISH, + pc->serialization); + if (wh == NULL) + { + GNUNET_break (0); + goto cleanup; + } + if ((GNUNET_OK != GNUNET_BIO_write_string (wh, pc->nid)) || + (GNUNET_OK != GNUNET_BIO_write_string (wh, pc->nuid)) || + (GNUNET_OK != GNUNET_BIO_write_int32 (wh, pc->options)) || + (GNUNET_OK != GNUNET_BIO_write_int32 (wh, pc->all_done)) || + (GNUNET_OK != GNUNET_BIO_write_string (wh, pc->fi->serialization)) || + (GNUNET_OK != + GNUNET_BIO_write_string (wh, + (pc->fi_pos == + NULL) ? NULL : pc->fi_pos->serialization)) || + (GNUNET_OK != + GNUNET_BIO_write_string (wh, + (pc->namespace == + NULL) ? NULL : pc->namespace->name))) + { + GNUNET_break (0); + goto cleanup; + } + if (GNUNET_OK != GNUNET_BIO_write_close (wh)) + { + wh = NULL; + GNUNET_break (0); + goto cleanup; + } + return; +cleanup: + if (wh != NULL) + (void) GNUNET_BIO_write_close (wh); + GNUNET_FS_remove_sync_file_ (pc->h, GNUNET_FS_SYNC_PATH_MASTER_PUBLISH, + pc->serialization); + GNUNET_free (pc->serialization); + pc->serialization = NULL; +} + + +/** + * Synchronize this unindex struct with its mirror + * on disk. Note that all internal FS-operations that change + * publishing structs should already call "sync" internally, + * so this function is likely not useful for clients. + * + * @param uc the struct to sync + */ +void +GNUNET_FS_unindex_sync_ (struct GNUNET_FS_UnindexContext *uc) +{ + struct GNUNET_BIO_WriteHandle *wh; + + if (NULL == uc->serialization) + uc->serialization = + make_serialization_file_name (uc->h, + GNUNET_FS_SYNC_PATH_MASTER_UNINDEX); + if (NULL == uc->serialization) + return; + wh = get_write_handle (uc->h, GNUNET_FS_SYNC_PATH_MASTER_UNINDEX, + uc->serialization); + if (wh == NULL) + { + GNUNET_break (0); + goto cleanup; + } + if ((GNUNET_OK != GNUNET_BIO_write_string (wh, uc->filename)) || + (GNUNET_OK != GNUNET_BIO_write_int64 (wh, uc->file_size)) || + (GNUNET_OK != write_start_time (wh, uc->start_time)) || + (GNUNET_OK != GNUNET_BIO_write_int32 (wh, (uint32_t) uc->state)) || + ((uc->state == UNINDEX_STATE_FS_NOTIFY) && + (GNUNET_OK != + GNUNET_BIO_write (wh, &uc->file_id, sizeof (GNUNET_HashCode)))) || + ((uc->state == UNINDEX_STATE_ERROR) && + (GNUNET_OK != GNUNET_BIO_write_string (wh, uc->emsg)))) + { + GNUNET_break (0); + goto cleanup; + } + if (GNUNET_OK != GNUNET_BIO_write_close (wh)) + { + wh = NULL; + GNUNET_break (0); + goto cleanup; + } + return; +cleanup: + if (wh != NULL) + (void) GNUNET_BIO_write_close (wh); + GNUNET_FS_remove_sync_file_ (uc->h, GNUNET_FS_SYNC_PATH_MASTER_UNINDEX, + uc->serialization); + GNUNET_free (uc->serialization); + uc->serialization = NULL; +} + + +/** + * Serialize a download request. + * + * @param wh the 'struct GNUNET_BIO_WriteHandle*' + * @param dr the 'struct DownloadRequest' + * @return GNUNET_YES on success, GNUNET_NO on error + */ +static int +write_download_request (struct GNUNET_BIO_WriteHandle *wh, + struct DownloadRequest *dr) +{ + unsigned int i; + + if ((GNUNET_OK != GNUNET_BIO_write_int32 (wh, dr->state)) || + (GNUNET_OK != GNUNET_BIO_write_int64 (wh, dr->offset)) || + (GNUNET_OK != GNUNET_BIO_write_int32 (wh, dr->num_children)) || + (GNUNET_OK != GNUNET_BIO_write_int32 (wh, dr->depth))) + return GNUNET_NO; + if ((dr->state == BRS_CHK_SET) && + (GNUNET_OK != + GNUNET_BIO_write (wh, &dr->chk, sizeof (struct ContentHashKey)))) + return GNUNET_NO; + for (i = 0; i < dr->num_children; i++) + if (GNUNET_NO == write_download_request (wh, dr->children[i])) + return GNUNET_NO; + return GNUNET_YES; +} + + +/** + * Read a download request tree. + * + * @param rh stream to read from + * @return value the 'struct DownloadRequest', NULL on error + */ +static struct DownloadRequest * +read_download_request (struct GNUNET_BIO_ReadHandle *rh) +{ + struct DownloadRequest *dr; + unsigned int i; + + dr = GNUNET_malloc (sizeof (struct DownloadRequest)); + + if ((GNUNET_OK != GNUNET_BIO_read_int32 (rh, &dr->state)) || + (GNUNET_OK != GNUNET_BIO_read_int64 (rh, &dr->offset)) || + (GNUNET_OK != GNUNET_BIO_read_int32 (rh, &dr->num_children)) || + (dr->num_children > CHK_PER_INODE) || + (GNUNET_OK != GNUNET_BIO_read_int32 (rh, &dr->depth)) || ((dr->depth == 0) + && + (dr->num_children + > 0)) || + ((dr->depth > 0) && (dr->num_children == 0))) + { + GNUNET_break (0); + dr->num_children = 0; + goto cleanup; + } + if (dr->num_children > 0) + dr->children = + GNUNET_malloc (dr->num_children * sizeof (struct ContentHashKey)); + switch (dr->state) + { + case BRS_INIT: + case BRS_RECONSTRUCT_DOWN: + case BRS_RECONSTRUCT_META_UP: + case BRS_RECONSTRUCT_UP: + break; + case BRS_CHK_SET: + if (GNUNET_OK != + GNUNET_BIO_read (rh, "chk", &dr->chk, sizeof (struct ContentHashKey))) + goto cleanup; + break; + case BRS_DOWNLOAD_DOWN: + case BRS_DOWNLOAD_UP: + case BRS_ERROR: + break; + default: + GNUNET_break (0); + goto cleanup; + } + for (i = 0; i < dr->num_children; i++) + { + if (NULL == (dr->children[i] = read_download_request (rh))) + goto cleanup; + dr->children[i]->parent = dr; + } + return dr; +cleanup: + GNUNET_FS_free_download_request_ (dr); + return NULL; +} + + +/** + * Compute the name of the sync file (or directory) for the given download + * context. + * + * @param dc download context to compute for + * @param uni unique filename to use, use "" for the directory name + * @param ext extension to use, use ".dir" for our own subdirectory + * @return the expanded file name, NULL for none + */ +static char * +get_download_sync_filename (struct GNUNET_FS_DownloadContext *dc, + const char *uni, const char *ext) +{ + char *par; + char *epar; + + if (dc->parent == NULL) + return get_serialization_file_name (dc->h, + (dc->search != + NULL) ? + GNUNET_FS_SYNC_PATH_CHILD_DOWNLOAD : + GNUNET_FS_SYNC_PATH_MASTER_DOWNLOAD, + uni); + if (dc->parent->serialization == NULL) + return NULL; + par = get_download_sync_filename (dc->parent, dc->parent->serialization, ""); + if (par == NULL) + return NULL; + GNUNET_asprintf (&epar, "%s.dir%s%s%s", par, DIR_SEPARATOR_STR, uni, ext); + GNUNET_free (par); + return epar; +} + + +/** + * Synchronize this download struct with its mirror + * on disk. Note that all internal FS-operations that change + * publishing structs should already call "sync" internally, + * so this function is likely not useful for clients. + * + * @param dc the struct to sync + */ +void +GNUNET_FS_download_sync_ (struct GNUNET_FS_DownloadContext *dc) +{ + struct GNUNET_BIO_WriteHandle *wh; + char *uris; + char *fn; + char *dir; + + if (NULL == dc->serialization) + { + dir = get_download_sync_filename (dc, "", ""); + if (dir == NULL) + return; + if (GNUNET_OK != GNUNET_DISK_directory_create_for_file (dir)) + { + GNUNET_free (dir); + return; + } + fn = GNUNET_DISK_mktemp (dir); + GNUNET_free (dir); + if (fn == NULL) + return; + dc->serialization = get_serialization_short_name (fn); + } + else + { + fn = get_download_sync_filename (dc, dc->serialization, ""); + if (fn == NULL) + { + GNUNET_free (dc->serialization); + dc->serialization = NULL; + GNUNET_free (fn); + return; + } + } + wh = GNUNET_BIO_write_open (fn); + if (wh == NULL) + { + GNUNET_free (dc->serialization); + dc->serialization = NULL; + GNUNET_free (fn); + return; + } + GNUNET_assert ((GNUNET_YES == GNUNET_FS_uri_test_chk (dc->uri)) || + (GNUNET_YES == GNUNET_FS_uri_test_loc (dc->uri))); + uris = GNUNET_FS_uri_to_string (dc->uri); + if ((GNUNET_OK != GNUNET_BIO_write_string (wh, uris)) || + (GNUNET_OK != GNUNET_BIO_write_meta_data (wh, dc->meta)) || + (GNUNET_OK != GNUNET_BIO_write_string (wh, dc->emsg)) || + (GNUNET_OK != GNUNET_BIO_write_string (wh, dc->filename)) || + (GNUNET_OK != GNUNET_BIO_write_string (wh, dc->temp_filename)) || + (GNUNET_OK != GNUNET_BIO_write_int64 (wh, dc->old_file_size)) || + (GNUNET_OK != GNUNET_BIO_write_int64 (wh, dc->offset)) || + (GNUNET_OK != GNUNET_BIO_write_int64 (wh, dc->length)) || + (GNUNET_OK != GNUNET_BIO_write_int64 (wh, dc->completed)) || + (GNUNET_OK != write_start_time (wh, dc->start_time)) || + (GNUNET_OK != GNUNET_BIO_write_int32 (wh, dc->anonymity)) || + (GNUNET_OK != GNUNET_BIO_write_int32 (wh, (uint32_t) dc->options)) || + (GNUNET_OK != GNUNET_BIO_write_int32 (wh, (uint32_t) dc->has_finished))) + { + GNUNET_break (0); + goto cleanup; + } + if (NULL == dc->emsg) + { + GNUNET_assert (dc->top_request != NULL); + if (GNUNET_YES != write_download_request (wh, dc->top_request)) + { + GNUNET_break (0); + goto cleanup; + } + } + GNUNET_free_non_null (uris); + uris = NULL; + if (GNUNET_OK != GNUNET_BIO_write_close (wh)) + { + wh = NULL; + GNUNET_break (0); + goto cleanup; + } + GNUNET_free (fn); + return; +cleanup: + if (NULL != wh) + (void) GNUNET_BIO_write_close (wh); + GNUNET_free_non_null (uris); + if (0 != UNLINK (fn)) + GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", fn); + GNUNET_free (fn); + GNUNET_free (dc->serialization); + dc->serialization = NULL; +} + + +/** + * Synchronize this search result with its mirror + * on disk. Note that all internal FS-operations that change + * publishing structs should already call "sync" internally, + * so this function is likely not useful for clients. + * + * @param sr the struct to sync + */ +void +GNUNET_FS_search_result_sync_ (struct GNUNET_FS_SearchResult *sr) +{ + struct GNUNET_BIO_WriteHandle *wh; + char *uris; + + uris = NULL; + if (NULL == sr->serialization) + sr->serialization = + make_serialization_file_name_in_dir (sr->sc->h, + (sr->sc->psearch_result == + NULL) ? + GNUNET_FS_SYNC_PATH_MASTER_SEARCH : + GNUNET_FS_SYNC_PATH_CHILD_SEARCH, + sr->sc->serialization); + if (NULL == sr->serialization) + return; + wh = get_write_handle_in_dir (sr->sc->h, + (sr->sc->psearch_result == + NULL) ? GNUNET_FS_SYNC_PATH_MASTER_SEARCH : + GNUNET_FS_SYNC_PATH_CHILD_SEARCH, + sr->sc->serialization, sr->serialization); + if (wh == NULL) + { + GNUNET_break (0); + goto cleanup; + } + uris = GNUNET_FS_uri_to_string (sr->uri); + if ((GNUNET_OK != GNUNET_BIO_write_string (wh, uris)) || + (GNUNET_OK != + GNUNET_BIO_write_string (wh, + sr->download != + NULL ? sr->download->serialization : NULL)) || + (GNUNET_OK != + GNUNET_BIO_write_string (wh, + sr->update_search != + NULL ? sr->update_search->serialization : NULL)) + || (GNUNET_OK != GNUNET_BIO_write_meta_data (wh, sr->meta)) || + (GNUNET_OK != GNUNET_BIO_write (wh, &sr->key, sizeof (GNUNET_HashCode))) + || (GNUNET_OK != GNUNET_BIO_write_int32 (wh, sr->mandatory_missing)) || + (GNUNET_OK != GNUNET_BIO_write_int32 (wh, sr->optional_support)) || + (GNUNET_OK != GNUNET_BIO_write_int32 (wh, sr->availability_success)) || + (GNUNET_OK != GNUNET_BIO_write_int32 (wh, sr->availability_trials)) ) + { + GNUNET_break (0); + goto cleanup; + } + if ( (sr->uri != NULL) && + (sr->sc->uri->type == ksk) && + (GNUNET_OK != GNUNET_BIO_write (wh, sr->keyword_bitmap, + (sr->sc->uri->data.ksk.keywordCount + 7) / 8)) ) + { + GNUNET_break (0); + goto cleanup; + } + if (GNUNET_OK != GNUNET_BIO_write_close (wh)) + { + wh = NULL; + GNUNET_break (0); + goto cleanup; + } + GNUNET_free_non_null (uris); + return; +cleanup: + GNUNET_free_non_null (uris); + if (wh != NULL) + (void) GNUNET_BIO_write_close (wh); + remove_sync_file_in_dir (sr->sc->h, + (sr->sc->psearch_result == + NULL) ? GNUNET_FS_SYNC_PATH_MASTER_SEARCH : + GNUNET_FS_SYNC_PATH_CHILD_SEARCH, + sr->sc->serialization, sr->serialization); + GNUNET_free (sr->serialization); + sr->serialization = NULL; +} + + +/** + * Synchronize this search struct with its mirror + * on disk. Note that all internal FS-operations that change + * publishing structs should already call "sync" internally, + * so this function is likely not useful for clients. + * + * @param sc the struct to sync + */ +void +GNUNET_FS_search_sync_ (struct GNUNET_FS_SearchContext *sc) +{ + struct GNUNET_BIO_WriteHandle *wh; + char *uris; + char in_pause; + const char *category; + + category = + (sc->psearch_result == + NULL) ? GNUNET_FS_SYNC_PATH_MASTER_SEARCH : + GNUNET_FS_SYNC_PATH_CHILD_SEARCH; + if (NULL == sc->serialization) + sc->serialization = make_serialization_file_name (sc->h, category); + if (NULL == sc->serialization) + return; + uris = NULL; + wh = get_write_handle (sc->h, category, sc->serialization); + if (wh == NULL) + { + GNUNET_break (0); + goto cleanup; + } + GNUNET_assert ((GNUNET_YES == GNUNET_FS_uri_test_ksk (sc->uri)) || + (GNUNET_YES == GNUNET_FS_uri_test_sks (sc->uri))); + uris = GNUNET_FS_uri_to_string (sc->uri); + in_pause = (sc->task != GNUNET_SCHEDULER_NO_TASK) ? 'r' : '\0'; + if ((GNUNET_OK != GNUNET_BIO_write_string (wh, uris)) || + (GNUNET_OK != write_start_time (wh, sc->start_time)) || + (GNUNET_OK != GNUNET_BIO_write_string (wh, sc->emsg)) || + (GNUNET_OK != GNUNET_BIO_write_int32 (wh, (uint32_t) sc->options)) || + (GNUNET_OK != GNUNET_BIO_write (wh, &in_pause, sizeof (in_pause))) || + (GNUNET_OK != GNUNET_BIO_write_int32 (wh, sc->anonymity))) + { + GNUNET_break (0); + goto cleanup; + } + GNUNET_free (uris); + uris = NULL; + if (GNUNET_OK != GNUNET_BIO_write_close (wh)) + { + wh = NULL; + GNUNET_break (0); + goto cleanup; + } + return; +cleanup: + if (wh != NULL) + (void) GNUNET_BIO_write_close (wh); + GNUNET_free_non_null (uris); + GNUNET_FS_remove_sync_file_ (sc->h, category, sc->serialization); + GNUNET_free (sc->serialization); + sc->serialization = NULL; +} + + +/** + * Function called with a filename of serialized unindexing operation + * to deserialize. + * + * @param cls the 'struct GNUNET_FS_Handle*' + * @param filename complete filename (absolute path) + * @return GNUNET_OK (continue to iterate) + */ +static int +deserialize_unindex_file (void *cls, const char *filename) +{ + struct GNUNET_FS_Handle *h = cls; + struct GNUNET_BIO_ReadHandle *rh; + struct GNUNET_FS_UnindexContext *uc; + struct GNUNET_FS_ProgressInfo pi; + char *emsg; + uint32_t state; + + uc = GNUNET_malloc (sizeof (struct GNUNET_FS_UnindexContext)); + uc->h = h; + uc->serialization = get_serialization_short_name (filename); + rh = GNUNET_BIO_read_open (filename); + if (rh == NULL) + { + GNUNET_break (0); + goto cleanup; + } + if ((GNUNET_OK != + GNUNET_BIO_read_string (rh, "unindex-fn", &uc->filename, 10 * 1024)) || + (GNUNET_OK != GNUNET_BIO_read_int64 (rh, &uc->file_size)) || + (GNUNET_OK != read_start_time (rh, &uc->start_time)) || + (GNUNET_OK != GNUNET_BIO_read_int32 (rh, &state))) + { + GNUNET_break (0); + goto cleanup; + } + uc->state = (enum UnindexState) state; + switch (state) + { + case UNINDEX_STATE_HASHING: + break; + case UNINDEX_STATE_FS_NOTIFY: + if (GNUNET_OK != + GNUNET_BIO_read (rh, "unindex-hash", &uc->file_id, + sizeof (GNUNET_HashCode))) + { + GNUNET_break (0); + goto cleanup; + } + break; + case UNINDEX_STATE_DS_REMOVE: + break; + case UNINDEX_STATE_COMPLETE: + break; + case UNINDEX_STATE_ERROR: + if (GNUNET_OK != + GNUNET_BIO_read_string (rh, "unindex-emsg", &uc->emsg, 10 * 1024)) + { + GNUNET_break (0); + goto cleanup; + } + break; + default: + GNUNET_break (0); + goto cleanup; + } + uc->top = GNUNET_FS_make_top (h, &GNUNET_FS_unindex_signal_suspend_, uc); + pi.status = GNUNET_FS_STATUS_UNINDEX_RESUME; + pi.value.unindex.specifics.resume.message = uc->emsg; + GNUNET_FS_unindex_make_status_ (&pi, uc, + (uc->state == + UNINDEX_STATE_COMPLETE) ? uc->file_size : 0); + switch (uc->state) + { + case UNINDEX_STATE_HASHING: + uc->fhc = + GNUNET_CRYPTO_hash_file (GNUNET_SCHEDULER_PRIORITY_IDLE, uc->filename, + HASHING_BLOCKSIZE, + &GNUNET_FS_unindex_process_hash_, uc); + break; + case UNINDEX_STATE_FS_NOTIFY: + uc->state = UNINDEX_STATE_HASHING; + GNUNET_FS_unindex_process_hash_ (uc, &uc->file_id); + break; + case UNINDEX_STATE_DS_REMOVE: + GNUNET_FS_unindex_do_remove_ (uc); + break; + case UNINDEX_STATE_COMPLETE: + case UNINDEX_STATE_ERROR: + /* no need to resume any operation, we were done */ + break; + default: + break; + } + if (GNUNET_OK != GNUNET_BIO_read_close (rh, &emsg)) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _("Failure while resuming unindexing operation `%s': %s\n"), + filename, emsg); + GNUNET_free (emsg); + } + return GNUNET_OK; +cleanup: + GNUNET_free_non_null (uc->filename); + if ((rh != NULL) && (GNUNET_OK != GNUNET_BIO_read_close (rh, &emsg))) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _("Failed to resume unindexing operation `%s': %s\n"), filename, + emsg); + GNUNET_free (emsg); + } + if (uc->serialization != NULL) + GNUNET_FS_remove_sync_file_ (h, GNUNET_FS_SYNC_PATH_MASTER_UNINDEX, + uc->serialization); + GNUNET_free_non_null (uc->serialization); + GNUNET_free (uc); + return GNUNET_OK; +} + + +/** + * Deserialize a download. + * + * @param h overall context + * @param rh file to deserialize from + * @param parent parent download + * @param search associated search + * @param serialization name under which the search was serialized + */ +static void +deserialize_download (struct GNUNET_FS_Handle *h, + struct GNUNET_BIO_ReadHandle *rh, + struct GNUNET_FS_DownloadContext *parent, + struct GNUNET_FS_SearchResult *search, + const char *serialization); + + +/** + * Deserialize a search. + * + * @param h overall context + * @param rh file to deserialize from + * @param psearch_result parent search result + * @param serialization name under which the search was serialized + */ +static struct GNUNET_FS_SearchContext * +deserialize_search (struct GNUNET_FS_Handle *h, + struct GNUNET_BIO_ReadHandle *rh, + struct GNUNET_FS_SearchResult *psearch_result, + const char *serialization); + + +/** + * Function called with a filename of serialized search result + * to deserialize. + * + * @param cls the 'struct GNUNET_FS_SearchContext*' + * @param filename complete filename (absolute path) + * @return GNUNET_OK (continue to iterate) + */ +static int +deserialize_search_result (void *cls, const char *filename) +{ + struct GNUNET_FS_SearchContext *sc = cls; + char *ser; + char *uris; + char *emsg; + char *download; + char *update_srch; + struct GNUNET_BIO_ReadHandle *rh; + struct GNUNET_BIO_ReadHandle *drh; + struct GNUNET_FS_SearchResult *sr; + + ser = get_serialization_short_name (filename); + rh = GNUNET_BIO_read_open (filename); + if (rh == NULL) + { + if (ser != NULL) + { + remove_sync_file_in_dir (sc->h, + (sc->psearch_result == + NULL) ? GNUNET_FS_SYNC_PATH_MASTER_SEARCH : + GNUNET_FS_SYNC_PATH_CHILD_SEARCH, + sc->serialization, ser); + GNUNET_free (ser); + } + return GNUNET_OK; + } + emsg = NULL; + uris = NULL; + download = NULL; + update_srch = NULL; + sr = GNUNET_malloc (sizeof (struct GNUNET_FS_SearchResult)); + sr->sc = sc; + sr->serialization = ser; + if ((GNUNET_OK != GNUNET_BIO_read_string (rh, "result-uri", &uris, 10 * 1024)) + || (NULL == (sr->uri = GNUNET_FS_uri_parse (uris, &emsg))) || + (GNUNET_OK != GNUNET_BIO_read_string (rh, "download-lnk", &download, 16)) + || (GNUNET_OK != + GNUNET_BIO_read_string (rh, "search-lnk", &update_srch, 16)) || + (GNUNET_OK != GNUNET_BIO_read_meta_data (rh, "result-meta", &sr->meta)) || + (GNUNET_OK != + GNUNET_BIO_read (rh, "result-key", &sr->key, sizeof (GNUNET_HashCode))) + || (GNUNET_OK != GNUNET_BIO_read_int32 (rh, &sr->mandatory_missing)) || + (GNUNET_OK != GNUNET_BIO_read_int32 (rh, &sr->optional_support)) || + (GNUNET_OK != GNUNET_BIO_read_int32 (rh, &sr->availability_success)) || + (GNUNET_OK != GNUNET_BIO_read_int32 (rh, &sr->availability_trials))) + { + GNUNET_break (0); + goto cleanup; + } + if (sr->sc->uri->type == ksk) + { + sr->keyword_bitmap = GNUNET_malloc ((sr->sc->uri->data.ksk.keywordCount + 7) / 8); /* round up, count bits */ + if (GNUNET_OK != GNUNET_BIO_read (rh, "keyword-bitmap", + sr->keyword_bitmap, + (sr->sc->uri->data.ksk.keywordCount + 7) / 8)) + { + GNUNET_break (0); + goto cleanup; + } + } + GNUNET_free (uris); + if (download != NULL) + { + drh = get_read_handle (sc->h, GNUNET_FS_SYNC_PATH_CHILD_DOWNLOAD, download); + if (drh != NULL) + { + deserialize_download (sc->h, drh, NULL, sr, download); + if (GNUNET_OK != GNUNET_BIO_read_close (drh, &emsg)) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _("Failed to resume sub-download `%s': %s\n"), download, + emsg); + GNUNET_free (emsg); + } + } + GNUNET_free (download); + } + if (update_srch != NULL) + { + drh = + get_read_handle (sc->h, GNUNET_FS_SYNC_PATH_CHILD_SEARCH, update_srch); + if (drh != NULL) + { + deserialize_search (sc->h, drh, sr, update_srch); + if (GNUNET_OK != GNUNET_BIO_read_close (drh, &emsg)) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _("Failed to resume sub-search `%s': %s\n"), update_srch, + emsg); + GNUNET_free (emsg); + } + } + GNUNET_free (update_srch); + } + GNUNET_CONTAINER_multihashmap_put (sc->master_result_map, &sr->key, sr, + GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE); + if (GNUNET_OK != GNUNET_BIO_read_close (rh, &emsg)) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _("Failure while resuming search operation `%s': %s\n"), + filename, emsg); + GNUNET_free (emsg); + } + return GNUNET_OK; +cleanup: + GNUNET_free_non_null (download); + GNUNET_free_non_null (emsg); + GNUNET_free_non_null (uris); + GNUNET_free_non_null (update_srch); + if (sr->uri != NULL) + GNUNET_FS_uri_destroy (sr->uri); + if (sr->meta != NULL) + GNUNET_CONTAINER_meta_data_destroy (sr->meta); + GNUNET_free (sr->serialization); + GNUNET_free (sr); + if (GNUNET_OK != GNUNET_BIO_read_close (rh, &emsg)) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _("Failure while resuming search operation `%s': %s\n"), + filename, emsg); + GNUNET_free (emsg); + } + return GNUNET_OK; +} + + +/** + * Send the 'resume' signal to the callback; also actually + * resume the download (put it in the queue). Does this + * recursively for the top-level download and all child + * downloads. + * + * @param dc download to resume + */ +static void +signal_download_resume (struct GNUNET_FS_DownloadContext *dc) +{ + struct GNUNET_FS_DownloadContext *dcc; + struct GNUNET_FS_ProgressInfo pi; + + pi.status = GNUNET_FS_STATUS_DOWNLOAD_RESUME; + pi.value.download.specifics.resume.meta = dc->meta; + pi.value.download.specifics.resume.message = dc->emsg; + GNUNET_FS_download_make_status_ (&pi, dc); + dcc = dc->child_head; + while (NULL != dcc) + { + signal_download_resume (dcc); + dcc = dcc->next; + } + if (dc->pending_head != NULL) + GNUNET_FS_download_start_downloading_ (dc); +} + + +/** + * Signal resuming of a search to our clients (for the + * top level search and all sub-searches). + * + * @param sc search being resumed + */ +static void +signal_search_resume (struct GNUNET_FS_SearchContext *sc); + + +/** + * Iterator over search results signaling resume to the client for + * each result. + * + * @param cls closure, the 'struct GNUNET_FS_SearchContext' + * @param key current key code + * @param value value in the hash map, the 'struct GNUNET_FS_SearchResult' + * @return GNUNET_YES (we should continue to iterate) + */ +static int +signal_result_resume (void *cls, const GNUNET_HashCode * key, void *value) +{ + struct GNUNET_FS_SearchContext *sc = cls; + struct GNUNET_FS_ProgressInfo pi; + struct GNUNET_FS_SearchResult *sr = value; + + if (0 == sr->mandatory_missing) + { + pi.status = GNUNET_FS_STATUS_SEARCH_RESUME_RESULT; + pi.value.search.specifics.resume_result.meta = sr->meta; + pi.value.search.specifics.resume_result.uri = sr->uri; + pi.value.search.specifics.resume_result.result = sr; + pi.value.search.specifics.resume_result.availability_rank = + 2 * sr->availability_success - sr->availability_trials; + pi.value.search.specifics.resume_result.availability_certainty = + sr->availability_trials; + pi.value.search.specifics.resume_result.applicability_rank = + sr->optional_support; + sr->client_info = GNUNET_FS_search_make_status_ (&pi, sc); + } + if (sr->download != NULL) + { + signal_download_resume (sr->download); + } + else + { + GNUNET_FS_search_start_probe_ (sr); + } + if (sr->update_search != NULL) + signal_search_resume (sr->update_search); + return GNUNET_YES; +} + + +/** + * Free memory allocated by the search context and its children + * + * @param sc search context to free + */ +static void +free_search_context (struct GNUNET_FS_SearchContext *sc); + + +/** + * Iterator over search results freeing each. + * + * @param cls closure, the 'struct GNUNET_FS_SearchContext' + * @param key current key code + * @param value value in the hash map, the 'struct GNUNET_FS_SearchResult' + * @return GNUNET_YES (we should continue to iterate) + */ +static int +free_result (void *cls, const GNUNET_HashCode * key, void *value) +{ + struct GNUNET_FS_SearchResult *sr = value; + + if (sr->update_search != NULL) + { + free_search_context (sr->update_search); + GNUNET_assert (NULL == sr->update_search); + } + GNUNET_CONTAINER_meta_data_destroy (sr->meta); + GNUNET_FS_uri_destroy (sr->uri); + GNUNET_free (sr); + return GNUNET_YES; +} + + +/** + * Free memory allocated by the search context and its children + * + * @param sc search context to free + */ +static void +free_search_context (struct GNUNET_FS_SearchContext *sc) +{ + if (sc->serialization != NULL) + { + GNUNET_FS_remove_sync_file_ (sc->h, + (sc->psearch_result == + NULL) ? GNUNET_FS_SYNC_PATH_MASTER_SEARCH : + GNUNET_FS_SYNC_PATH_CHILD_SEARCH, + sc->serialization); + GNUNET_FS_remove_sync_dir_ (sc->h, + (sc->psearch_result == + NULL) ? GNUNET_FS_SYNC_PATH_MASTER_SEARCH : + GNUNET_FS_SYNC_PATH_CHILD_SEARCH, + sc->serialization); + } + GNUNET_free_non_null (sc->serialization); + GNUNET_free_non_null (sc->emsg); + if (sc->uri != NULL) + GNUNET_FS_uri_destroy (sc->uri); + if (sc->master_result_map != NULL) + { + GNUNET_CONTAINER_multihashmap_iterate (sc->master_result_map, &free_result, + sc); + GNUNET_CONTAINER_multihashmap_destroy (sc->master_result_map); + } + GNUNET_free (sc); +} + + +/** + * Function called with a filename of serialized sub-download + * to deserialize. + * + * @param cls the 'struct GNUNET_FS_DownloadContext*' (parent) + * @param filename complete filename (absolute path) + * @return GNUNET_OK (continue to iterate) + */ +static int +deserialize_subdownload (void *cls, const char *filename) +{ + struct GNUNET_FS_DownloadContext *parent = cls; + char *ser; + char *emsg; + struct GNUNET_BIO_ReadHandle *rh; + + ser = get_serialization_short_name (filename); + rh = GNUNET_BIO_read_open (filename); + if (rh == NULL) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _ + ("Failed to resume sub-download `%s': could not open file `%s'\n"), + ser, filename); + GNUNET_free (ser); + return GNUNET_OK; + } + deserialize_download (parent->h, rh, parent, NULL, ser); + if (GNUNET_OK != GNUNET_BIO_read_close (rh, &emsg)) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _("Failed to resume sub-download `%s': %s\n"), ser, emsg); + GNUNET_free (emsg); + } + GNUNET_free (ser); + return GNUNET_OK; +} + + +/** + * Free this download context and all of its descendants. + * (only works during deserialization since not all possible + * state it taken care of). + * + * @param dc context to free + */ +static void +free_download_context (struct GNUNET_FS_DownloadContext *dc) +{ + struct GNUNET_FS_DownloadContext *dcc; + + if (dc->meta != NULL) + GNUNET_CONTAINER_meta_data_destroy (dc->meta); + if (dc->uri != NULL) + GNUNET_FS_uri_destroy (dc->uri); + GNUNET_free_non_null (dc->temp_filename); + GNUNET_free_non_null (dc->emsg); + GNUNET_free_non_null (dc->filename); + GNUNET_free_non_null (dc->serialization); + while (NULL != (dcc = dc->child_head)) + { + GNUNET_CONTAINER_DLL_remove (dc->child_head, dc->child_tail, dcc); + free_download_context (dcc); + } + GNUNET_FS_free_download_request_ (dc->top_request); + if (NULL != dc->active) + GNUNET_CONTAINER_multihashmap_destroy (dc->active); + GNUNET_free (dc); +} + + +/** + * Deserialize a download. + * + * @param h overall context + * @param rh file to deserialize from + * @param parent parent download + * @param search associated search + * @param serialization name under which the search was serialized + */ +static void +deserialize_download (struct GNUNET_FS_Handle *h, + struct GNUNET_BIO_ReadHandle *rh, + struct GNUNET_FS_DownloadContext *parent, + struct GNUNET_FS_SearchResult *search, + const char *serialization) +{ + struct GNUNET_FS_DownloadContext *dc; + char *emsg; + char *uris; + char *dn; + uint32_t options; + uint32_t status; + + uris = NULL; + emsg = NULL; + dc = GNUNET_malloc (sizeof (struct GNUNET_FS_DownloadContext)); + dc->parent = parent; + dc->h = h; + dc->serialization = GNUNET_strdup (serialization); + if ((GNUNET_OK != + GNUNET_BIO_read_string (rh, "download-uri", &uris, 10 * 1024)) || + (NULL == (dc->uri = GNUNET_FS_uri_parse (uris, &emsg))) || + ((GNUNET_YES != GNUNET_FS_uri_test_chk (dc->uri)) && + (GNUNET_YES != GNUNET_FS_uri_test_loc (dc->uri))) || + (GNUNET_OK != GNUNET_BIO_read_meta_data (rh, "download-meta", &dc->meta)) + || (GNUNET_OK != + GNUNET_BIO_read_string (rh, "download-emsg", &dc->emsg, 10 * 1024)) || + (GNUNET_OK != + GNUNET_BIO_read_string (rh, "download-fn", &dc->filename, 10 * 1024)) || + (GNUNET_OK != + GNUNET_BIO_read_string (rh, "download-tfn", &dc->temp_filename, + 10 * 1024)) || + (GNUNET_OK != GNUNET_BIO_read_int64 (rh, &dc->old_file_size)) || + (GNUNET_OK != GNUNET_BIO_read_int64 (rh, &dc->offset)) || + (GNUNET_OK != GNUNET_BIO_read_int64 (rh, &dc->length)) || + (GNUNET_OK != GNUNET_BIO_read_int64 (rh, &dc->completed)) || + (GNUNET_OK != read_start_time (rh, &dc->start_time)) || + (GNUNET_OK != GNUNET_BIO_read_int32 (rh, &dc->anonymity)) || + (GNUNET_OK != GNUNET_BIO_read_int32 (rh, &options)) || + (GNUNET_OK != GNUNET_BIO_read_int32 (rh, &status))) + { + GNUNET_break (0); + goto cleanup; + } + dc->options = (enum GNUNET_FS_DownloadOptions) options; + dc->active = + GNUNET_CONTAINER_multihashmap_create (1 + 2 * (dc->length / DBLOCK_SIZE)); + dc->has_finished = (int) status; + dc->treedepth = + GNUNET_FS_compute_depth (GNUNET_FS_uri_chk_get_file_size (dc->uri)); + if (GNUNET_FS_uri_test_loc (dc->uri)) + GNUNET_assert (GNUNET_OK == + GNUNET_FS_uri_loc_get_peer_identity (dc->uri, &dc->target)); + if (dc->emsg == NULL) + { + dc->top_request = read_download_request (rh); + if (dc->top_request == NULL) + { + GNUNET_break (0); + goto cleanup; + } + } + dn = get_download_sync_filename (dc, dc->serialization, ".dir"); + if (dn != NULL) + { + if (GNUNET_YES == GNUNET_DISK_directory_test (dn)) + GNUNET_DISK_directory_scan (dn, &deserialize_subdownload, dc); + GNUNET_free (dn); + } + if (parent != NULL) + { + GNUNET_abort (); // for debugging for now - FIXME + GNUNET_CONTAINER_DLL_insert (parent->child_head, parent->child_tail, dc); + } + if (search != NULL) + { + dc->search = search; + search->download = dc; + } + if ((parent == NULL) && (search == NULL)) + { + dc->top = + GNUNET_FS_make_top (dc->h, &GNUNET_FS_download_signal_suspend_, dc); + signal_download_resume (dc); + } + GNUNET_free (uris); + dc->task = GNUNET_SCHEDULER_add_now (&GNUNET_FS_download_start_task_, dc); + return; +cleanup: + GNUNET_free_non_null (uris); + GNUNET_free_non_null (emsg); + free_download_context (dc); +} + + +/** + * Signal resuming of a search to our clients (for the + * top level search and all sub-searches). + * + * @param sc search being resumed + */ +static void +signal_search_resume (struct GNUNET_FS_SearchContext *sc) +{ + struct GNUNET_FS_ProgressInfo pi; + + pi.status = GNUNET_FS_STATUS_SEARCH_RESUME; + pi.value.search.specifics.resume.message = sc->emsg; + pi.value.search.specifics.resume.is_paused = + (sc->client == NULL) ? GNUNET_YES : GNUNET_NO; + sc->client_info = GNUNET_FS_search_make_status_ (&pi, sc); + GNUNET_CONTAINER_multihashmap_iterate (sc->master_result_map, + &signal_result_resume, sc); + +} + + +/** + * Deserialize a search. + * + * @param h overall context + * @param rh file to deserialize from + * @param psearch_result parent search result + * @param serialization name under which the search was serialized + */ +static struct GNUNET_FS_SearchContext * +deserialize_search (struct GNUNET_FS_Handle *h, + struct GNUNET_BIO_ReadHandle *rh, + struct GNUNET_FS_SearchResult *psearch_result, + const char *serialization) +{ + struct GNUNET_FS_SearchContext *sc; + char *emsg; + char *uris; + char *dn; + uint32_t options; + char in_pause; + + if ((psearch_result != NULL) && (psearch_result->update_search != NULL)) + { + GNUNET_break (0); + return NULL; + } + uris = NULL; + emsg = NULL; + sc = GNUNET_malloc (sizeof (struct GNUNET_FS_SearchContext)); + if (psearch_result != NULL) + { + sc->psearch_result = psearch_result; + psearch_result->update_search = sc; + } + sc->h = h; + sc->serialization = GNUNET_strdup (serialization); + if ((GNUNET_OK != GNUNET_BIO_read_string (rh, "search-uri", &uris, 10 * 1024)) + || (NULL == (sc->uri = GNUNET_FS_uri_parse (uris, &emsg))) || + ((GNUNET_YES != GNUNET_FS_uri_test_ksk (sc->uri)) && + (GNUNET_YES != GNUNET_FS_uri_test_sks (sc->uri))) || + (GNUNET_OK != read_start_time (rh, &sc->start_time)) || + (GNUNET_OK != + GNUNET_BIO_read_string (rh, "search-emsg", &sc->emsg, 10 * 1024)) || + (GNUNET_OK != GNUNET_BIO_read_int32 (rh, &options)) || + (GNUNET_OK != + GNUNET_BIO_read (rh, "search-pause", &in_pause, sizeof (in_pause))) || + (GNUNET_OK != GNUNET_BIO_read_int32 (rh, &sc->anonymity))) + { + GNUNET_break (0); + goto cleanup; + } + sc->options = (enum GNUNET_FS_SearchOptions) options; + sc->master_result_map = GNUNET_CONTAINER_multihashmap_create (16); + dn = get_serialization_file_name_in_dir (h, + (sc->psearch_result == + NULL) ? + GNUNET_FS_SYNC_PATH_MASTER_SEARCH : + GNUNET_FS_SYNC_PATH_CHILD_SEARCH, + sc->serialization, ""); + if (dn != NULL) + { + if (GNUNET_YES == GNUNET_DISK_directory_test (dn)) + GNUNET_DISK_directory_scan (dn, &deserialize_search_result, sc); + GNUNET_free (dn); + } + if (('\0' == in_pause) && + (GNUNET_OK != GNUNET_FS_search_start_searching_ (sc))) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _ + ("Could not resume running search, will resume as paused search\n")); + } + signal_search_resume (sc); + GNUNET_free (uris); + return sc; +cleanup: + GNUNET_free_non_null (emsg); + free_search_context (sc); + GNUNET_free_non_null (uris); + return NULL; +} + + +/** + * Function called with a filename of serialized search operation + * to deserialize. + * + * @param cls the 'struct GNUNET_FS_Handle*' + * @param filename complete filename (absolute path) + * @return GNUNET_OK (continue to iterate) + */ +static int +deserialize_search_file (void *cls, const char *filename) +{ + struct GNUNET_FS_Handle *h = cls; + char *ser; + char *emsg; + struct GNUNET_BIO_ReadHandle *rh; + struct GNUNET_FS_SearchContext *sc; + struct stat buf; + + if (0 != STAT (filename, &buf)) + { + GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "stat", filename); + return GNUNET_OK; + } + if (S_ISDIR (buf.st_mode)) + return GNUNET_OK; /* skip directories */ + ser = get_serialization_short_name (filename); + rh = GNUNET_BIO_read_open (filename); + if (rh == NULL) + { + if (ser != NULL) + { + GNUNET_FS_remove_sync_file_ (h, GNUNET_FS_SYNC_PATH_MASTER_SEARCH, ser); + GNUNET_free (ser); + } + return GNUNET_OK; + } + sc = deserialize_search (h, rh, NULL, ser); + if (sc != NULL) + sc->top = GNUNET_FS_make_top (h, &GNUNET_FS_search_signal_suspend_, sc); + GNUNET_free (ser); + if (GNUNET_OK != GNUNET_BIO_read_close (rh, &emsg)) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _("Failure while resuming search operation `%s': %s\n"), + filename, emsg); + GNUNET_free (emsg); + } + return GNUNET_OK; +} + + +/** + * Function called with a filename of serialized download operation + * to deserialize. + * + * @param cls the 'struct GNUNET_FS_Handle*' + * @param filename complete filename (absolute path) + * @return GNUNET_OK (continue to iterate) + */ +static int +deserialize_download_file (void *cls, const char *filename) +{ + struct GNUNET_FS_Handle *h = cls; + char *ser; + char *emsg; + struct GNUNET_BIO_ReadHandle *rh; + + ser = get_serialization_short_name (filename); + rh = GNUNET_BIO_read_open (filename); + if (rh == NULL) + { + if (0 != UNLINK (filename)) + GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", filename); + GNUNET_free (ser); + return GNUNET_OK; + } + deserialize_download (h, rh, NULL, NULL, ser); + GNUNET_free (ser); + if (GNUNET_OK != GNUNET_BIO_read_close (rh, &emsg)) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _("Failure while resuming download operation `%s': %s\n"), + filename, emsg); + GNUNET_free (emsg); + } + return GNUNET_OK; +} + + +/** + * Deserialize informatin about pending operations. + * + * @param master_path which master directory should be scanned + * @param proc function to call for each entry (will get 'h' for 'cls') + * @param h the 'struct GNUNET_FS_Handle*' + */ +static void +deserialization_master (const char *master_path, GNUNET_FileNameCallback proc, + struct GNUNET_FS_Handle *h) +{ + char *dn; + + dn = get_serialization_file_name (h, master_path, ""); + if (dn == NULL) + return; + if (GNUNET_YES == GNUNET_DISK_directory_test (dn)) + GNUNET_DISK_directory_scan (dn, proc, h); + GNUNET_free (dn); +} + + +/** + * Setup a connection to the file-sharing service. + * + * @param cfg configuration to use + * @param client_name unique identifier for this client + * @param upcb function to call to notify about FS actions + * @param upcb_cls closure for upcb + * @param flags specific attributes for fs-operations + * @param ... list of optional options, terminated with GNUNET_FS_OPTIONS_END + * @return NULL on error + */ +struct GNUNET_FS_Handle * +GNUNET_FS_start (const struct GNUNET_CONFIGURATION_Handle *cfg, + const char *client_name, GNUNET_FS_ProgressCallback upcb, + void *upcb_cls, enum GNUNET_FS_Flags flags, ...) +{ + struct GNUNET_FS_Handle *ret; + enum GNUNET_FS_OPTIONS opt; + va_list ap; + + ret = GNUNET_malloc (sizeof (struct GNUNET_FS_Handle)); + ret->cfg = cfg; + ret->client_name = GNUNET_strdup (client_name); + ret->upcb = upcb; + ret->upcb_cls = upcb_cls; + ret->flags = flags; + ret->max_parallel_downloads = 1; + ret->max_parallel_requests = 1; + ret->avg_block_latency = GNUNET_TIME_UNIT_MINUTES; /* conservative starting point */ + va_start (ap, flags); + while (GNUNET_FS_OPTIONS_END != (opt = va_arg (ap, enum GNUNET_FS_OPTIONS))) + { + switch (opt) + { + case GNUNET_FS_OPTIONS_DOWNLOAD_PARALLELISM: + ret->max_parallel_downloads = va_arg (ap, unsigned int); + + break; + case GNUNET_FS_OPTIONS_REQUEST_PARALLELISM: + ret->max_parallel_requests = va_arg (ap, unsigned int); + + break; + default: + GNUNET_break (0); + GNUNET_free (ret->client_name); + GNUNET_free (ret); + va_end (ap); + return NULL; + } + } + va_end (ap); + if (0 != (GNUNET_FS_FLAGS_PERSISTENCE & flags)) + { + deserialization_master (GNUNET_FS_SYNC_PATH_MASTER_PUBLISH, + &deserialize_publish_file, ret); + deserialization_master (GNUNET_FS_SYNC_PATH_MASTER_SEARCH, + &deserialize_search_file, ret); + deserialization_master (GNUNET_FS_SYNC_PATH_MASTER_DOWNLOAD, + &deserialize_download_file, ret); + deserialization_master (GNUNET_FS_SYNC_PATH_MASTER_UNINDEX, + &deserialize_unindex_file, ret); + } + return ret; +} + + +/** + * Close our connection with the file-sharing service. + * The callback given to GNUNET_FS_start will no longer be + * called after this function returns. + * + * @param h handle that was returned from GNUNET_FS_start + */ +void +GNUNET_FS_stop (struct GNUNET_FS_Handle *h) +{ + while (h->top_head != NULL) + h->top_head->ssf (h->top_head->ssf_cls); + if (h->queue_job != GNUNET_SCHEDULER_NO_TASK) + GNUNET_SCHEDULER_cancel (h->queue_job); + GNUNET_free (h->client_name); + GNUNET_free (h); +} + + +/* end of fs.c */ diff --git a/src/fs/fs_api.h b/src/fs/fs_api.h new file mode 100644 index 0000000..de66ac6 --- /dev/null +++ b/src/fs/fs_api.h @@ -0,0 +1,1919 @@ +/* + This file is part of GNUnet. + (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/fs_api.h + * @brief shared definitions for the FS library + * @author Igor Wronsky, Christian Grothoff + */ +#ifndef FS_API_H +#define FS_API_H + +#include "gnunet_constants.h" +#include "gnunet_datastore_service.h" +#include "gnunet_dht_service.h" +#include "gnunet_fs_service.h" +#include "gnunet_block_lib.h" +#include "block_fs.h" +#include "fs.h" + +/** + * Size of the individual blocks used for file-sharing. + */ +#define DBLOCK_SIZE (32*1024) + +/** + * Pick a multiple of 2 here to achive 8-byte alignment! We also + * probably want DBlocks to have (roughly) the same size as IBlocks. + * With SHA-512, the optimal value is 32768 byte / 128 byte = 256 (128 + * byte = 2 * 512 bits). DO NOT CHANGE! + */ +#define CHK_PER_INODE 256 + +/** + * Maximum size for a file to be considered for inlining in a + * directory. + */ +#define MAX_INLINE_SIZE 65536 + +/** + * Name of the directory with top-level searches. + */ +#define GNUNET_FS_SYNC_PATH_MASTER_SEARCH "search" + +/** + * Name of the directory with sub-searches (namespace-updates). + */ +#define GNUNET_FS_SYNC_PATH_CHILD_SEARCH "search-child" + +/** + * Name of the directory with master downloads (not associated + * with search or part of another download). + */ +#define GNUNET_FS_SYNC_PATH_MASTER_DOWNLOAD "download" + +/** + * Name of the directory with downloads that are part of another + * download or a search. + */ +#define GNUNET_FS_SYNC_PATH_CHILD_DOWNLOAD "download-child" + +/** + * Name of the directory with publishing operations. + */ +#define GNUNET_FS_SYNC_PATH_MASTER_PUBLISH "publish" + +/** + * Name of the directory with files that are being published + */ +#define GNUNET_FS_SYNC_PATH_FILE_INFO "publish-file" + +/** + * Name of the directory with unindex operations. + */ +#define GNUNET_FS_SYNC_PATH_MASTER_UNINDEX "unindex" + + +/** + * @brief complete information needed + * to download a file. + */ +struct FileIdentifier +{ + + /** + * Total size of the file in bytes. (network byte order (!)) + */ + uint64_t file_length; + + /** + * Query and key of the top GNUNET_EC_IBlock. + */ + struct ContentHashKey chk; + +}; + + +/** + * Information about a file and its location + * (peer claiming to share the file). + */ +struct Location +{ + /** + * Information about the shared file. + */ + struct FileIdentifier fi; + + /** + * Identity of the peer sharing the file. + */ + struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded peer; + + /** + * Time when this location URI expires. + */ + struct GNUNET_TIME_Absolute expirationTime; + + /** + * RSA signature over the GNUNET_EC_FileIdentifier, + * GNUNET_hash of the peer and expiration time. + */ + struct GNUNET_CRYPTO_RsaSignature contentSignature; + +}; + +/** + * Types of URIs. + */ +enum uri_types +{ + /** + * Content-hash-key (simple file). + */ + chk, + + /** + * Signed key space (file in namespace). + */ + sks, + + /** + * Keyword search key (query with keywords). + */ + ksk, + + /** + * Location (chk with identity of hosting peer). + */ + loc +}; + +/** + * A Universal Resource Identifier (URI), opaque. + */ +struct GNUNET_FS_Uri +{ + /** + * Type of the URI. + */ + enum uri_types type; + + union + { + struct + { + /** + * Keywords start with a '+' if they are + * mandatory (in which case the '+' is NOT + * part of the keyword) and with a + * simple space if they are optional + * (in which case the space is ALSO not + * part of the actual keyword). + * + * Double-quotes to protect spaces and + * %-encoding are NOT used internally + * (only in URI-strings). + */ + char **keywords; + + /** + * Size of the keywords array. + */ + unsigned int keywordCount; + } ksk; + + struct + { + /** + * Hash of the public key for the namespace. + */ + GNUNET_HashCode namespace; + + /** + * Human-readable identifier chosen for this + * entry in the namespace. + */ + char *identifier; + } sks; + + /** + * Information needed to retrieve a file (content-hash-key + * plus file size). + */ + struct FileIdentifier chk; + + /** + * Information needed to retrieve a file including signed + * location (identity of a peer) of the content. + */ + struct Location loc; + } data; + +}; + + +/** + * Information for a file or directory that is + * about to be published. + */ +struct GNUNET_FS_FileInformation +{ + + /** + * Files in a directory are kept as a linked list. + */ + struct GNUNET_FS_FileInformation *next; + + /** + * If this is a file in a directory, "dir" refers to + * the directory; otherwise NULL. + */ + struct GNUNET_FS_FileInformation *dir; + + /** + * Handle to the master context. + */ + struct GNUNET_FS_Handle *h; + + /** + * Pointer kept for the client. + */ + void *client_info; + + /** + * Metadata to use for the file. + */ + struct GNUNET_CONTAINER_MetaData *meta; + + /** + * Keywords to use for KBlocks. + */ + struct GNUNET_FS_Uri *keywords; + + /** + * CHK for this file or directory. NULL if + * we have not yet computed it. + */ + struct GNUNET_FS_Uri *chk_uri; + + /** + * Block options for the file. + */ + struct GNUNET_FS_BlockOptions bo; + + /** + * At what time did we start this upload? + */ + struct GNUNET_TIME_Absolute start_time; + + /** + * Under what filename is this struct serialized + * (for operational persistence). Should be determined + * using 'mktemp'. + */ + char *serialization; + + /** + * Encoder being used to publish this file. + */ + struct GNUNET_FS_TreeEncoder *te; + + /** + * Error message (non-NULL if this operation failed). + */ + char *emsg; + + /** + * Name of the file or directory (must be an absolute path). + */ + char *filename; + + /** + * Data describing either the file or the directory. + */ + union + { + + /** + * Data for a file. + */ + struct + { + + /** + * Function that can be used to read the data for the file. + */ + GNUNET_FS_DataReader reader; + + /** + * Closure for reader. + */ + void *reader_cls; + + /** + * If this file is being indexed, this value is set to the hash + * over the entire file (when the indexing process is started). + * Otherwise this field is not used. + */ + GNUNET_HashCode file_id; + + /** + * Size of the file (in bytes). + */ + uint64_t file_size; + + /** + * Should the file be indexed or inserted? + */ + int do_index; + + /** + * Is "file_id" already valid? Set to GNUNET_YES once the hash + * has been calculated. + */ + int have_hash; + + /** + * Has the service confirmed our INDEX_START request? + * GNUNET_YES if this step has been completed. + */ + int index_start_confirmed; + + } file; + + /** + * Data for a directory. + */ + struct + { + + /** + * Linked list of entries in the directory. + */ + struct GNUNET_FS_FileInformation *entries; + + /** + * Size of the directory itself (in bytes); 0 if the + * size has not yet been calculated. + */ + size_t dir_size; + + /** + * Pointer to the data for the directory (or NULL if not + * available). + */ + void *dir_data; + + } dir; + + } data; + + /** + * Is this struct for a file or directory? + */ + int is_directory; + + /** + * Are we done publishing this file? + */ + int is_published; + +}; + + +/** + * The job is now ready to run and should use the given client + * handle to communicate with the FS service. + * + * @param cls closure + * @param client handle to use for FS communication + */ +typedef void (*GNUNET_FS_QueueStart) (void *cls, + struct GNUNET_CLIENT_Connection * client); + + +/** + * The job must now stop to run and should destry the client handle as + * soon as possible (ideally prior to returning). + */ +typedef void (*GNUNET_FS_QueueStop) (void *cls); + + +/** + * Entry in the job queue. + */ +struct GNUNET_FS_QueueEntry +{ + /** + * This is a linked list. + */ + struct GNUNET_FS_QueueEntry *next; + + /** + * This is a linked list. + */ + struct GNUNET_FS_QueueEntry *prev; + + /** + * Function to call when the job is started. + */ + GNUNET_FS_QueueStart start; + + /** + * Function to call when the job needs to stop (or is done / dequeued). + */ + GNUNET_FS_QueueStop stop; + + /** + * Closure for start and stop. + */ + void *cls; + + /** + * Handle to FS primary context. + */ + struct GNUNET_FS_Handle *h; + + /** + * Client handle, or NULL if job is not running. + */ + struct GNUNET_CLIENT_Connection *client; + + /** + * Time the job was originally queued. + */ + struct GNUNET_TIME_Absolute queue_time; + + /** + * Time the job was started last. + */ + struct GNUNET_TIME_Absolute start_time; + + /** + * Total amount of time the job has been running (except for the + * current run). + */ + struct GNUNET_TIME_Relative run_time; + + /** + * How many blocks do the active downloads have? + */ + unsigned int blocks; + + /** + * How often have we (re)started this download? + */ + unsigned int start_times; + +}; + + + + +/** + * Information we store for each search result. + */ +struct GNUNET_FS_SearchResult +{ + + /** + * Search context this result belongs to. + */ + struct GNUNET_FS_SearchContext *sc; + + /** + * URI to which this search result refers to. + */ + struct GNUNET_FS_Uri *uri; + + /** + * Metadata for the search result. + */ + struct GNUNET_CONTAINER_MetaData *meta; + + /** + * Client info for this search result. + */ + void *client_info; + + /** + * ID of a job that is currently probing this results' availability + * (NULL if we are not currently probing). + */ + struct GNUNET_FS_DownloadContext *probe_ctx; + + /** + * ID of an associated download based on this search result (or + * NULL for none). + */ + struct GNUNET_FS_DownloadContext *download; + + /** + * If this search result triggered an update search, this field + * links to the update search. + */ + struct GNUNET_FS_SearchContext *update_search; + + /** + * Name under which this search result is stored on disk. + */ + char *serialization; + + /** + * Bitmap that specifies precisely which keywords have been matched already. + */ + uint8_t *keyword_bitmap; + + /** + * Key for the search result + */ + GNUNET_HashCode key; + + /** + * ID of the task that will clean up the probe_ctx should it not + * complete on time (and that will need to be cancelled if we clean + * up the search result before then). + */ + GNUNET_SCHEDULER_TaskIdentifier probe_cancel_task; + + /** + * When did the current probe become active? + */ + struct GNUNET_TIME_Absolute probe_active_time; + + /** + * How much longer should we run the current probe before giving up? + */ + struct GNUNET_TIME_Relative remaining_probe_time; + + /** + * Number of mandatory keywords for which we have NOT yet found the + * search result; when this value hits zero, the search result is + * given to the callback. + */ + uint32_t mandatory_missing; + + /** + * Number of optional keywords under which this result was also + * found. + */ + uint32_t optional_support; + + /** + * Number of availability tests that have succeeded for this result. + */ + uint32_t availability_success; + + /** + * Number of availability trials that we have performed for this + * search result. + */ + uint32_t availability_trials; + +}; + + +/** + * Add a job to the queue. + * + * @param h handle to the overall FS state + * @param start function to call to begin the job + * @param stop function to call to pause the job, or on dequeue (if the job was running) + * @param cls closure for start and stop + * @param blocks number of blocks this download has + * @return queue handle + */ +struct GNUNET_FS_QueueEntry * +GNUNET_FS_queue_ (struct GNUNET_FS_Handle *h, GNUNET_FS_QueueStart start, + GNUNET_FS_QueueStop stop, void *cls, unsigned int blocks); + + +/** + * Dequeue a job from the queue. + * @param qh handle for the job + */ +void +GNUNET_FS_dequeue_ (struct GNUNET_FS_QueueEntry *qh); + + +/** + * Function that provides data by reading from a file. + * + * @param cls closure (points to the file information) + * @param offset offset to read from; it is possible + * that the caller might need to go backwards + * a bit at times + * @param max maximum number of bytes that should be + * copied to buf; readers are not allowed + * to provide less data unless there is an error; + * a value of "0" will be used at the end to allow + * the reader to clean up its internal state + * @param buf where the reader should write the data + * @param emsg location for the reader to store an error message + * @return number of bytes written, usually "max", 0 on error + */ +size_t +GNUNET_FS_data_reader_file_ (void *cls, uint64_t offset, size_t max, void *buf, + char **emsg); + + +/** + * Create the closure for the 'GNUNET_FS_data_reader_file_' callback. + * + * @param filename file to read + * @return closure to use + */ +void * +GNUNET_FS_make_file_reader_context_ (const char *filename); + + + +/** + * Function that provides data by copying from a buffer. + * + * @param cls closure (points to the buffer) + * @param offset offset to read from; it is possible + * that the caller might need to go backwards + * a bit at times + * @param max maximum number of bytes that should be + * copied to buf; readers are not allowed + * to provide less data unless there is an error; + * a value of "0" will be used at the end to allow + * the reader to clean up its internal state + * @param buf where the reader should write the data + * @param emsg location for the reader to store an error message + * @return number of bytes written, usually "max", 0 on error + */ +size_t +GNUNET_FS_data_reader_copy_ (void *cls, uint64_t offset, size_t max, void *buf, + char **emsg); + +/** + * Notification of FS that a search probe has made progress. + * This function is used INSTEAD of the client's event handler + * for downloads where the GNUNET_FS_DOWNLOAD_IS_PROBE flag is set. + * + * @param cls closure, always NULL (!), actual closure + * is in the client-context of the info struct + * @param info details about the event, specifying the event type + * and various bits about the event + * @return client-context (for the next progress call + * for this operation; should be set to NULL for + * SUSPEND and STOPPED events). The value returned + * will be passed to future callbacks in the respective + * field in the GNUNET_FS_ProgressInfo struct. + */ +void * +GNUNET_FS_search_probe_progress_ (void *cls, + const struct GNUNET_FS_ProgressInfo *info); + + +/** + * Main function that performs the upload. + * + * @param cls "struct GNUNET_FS_PublishContext" identifies the upload + * @param tc task context + */ +void +GNUNET_FS_publish_main_ (void *cls, + const struct GNUNET_SCHEDULER_TaskContext *tc); + + +/** + * Function called once the hash of the file + * that is being unindexed has been computed. + * + * @param cls closure, unindex context + * @param file_id computed hash, NULL on error + */ +void +GNUNET_FS_unindex_process_hash_ (void *cls, const GNUNET_HashCode * file_id); + + +/** + * Fill in all of the generic fields for a publish event and call the + * callback. + * + * @param pi structure to fill in + * @param pc overall publishing context + * @param p file information for the file being published + * @param offset where in the file are we so far + * @return value returned from callback + */ +void * +GNUNET_FS_publish_make_status_ (struct GNUNET_FS_ProgressInfo *pi, + struct GNUNET_FS_PublishContext *pc, + const struct GNUNET_FS_FileInformation *p, + uint64_t offset); + + +/** + * Fill in all of the generic fields for a download event and call the + * callback. + * + * @param pi structure to fill in + * @param dc overall download context + */ +void +GNUNET_FS_download_make_status_ (struct GNUNET_FS_ProgressInfo *pi, + struct GNUNET_FS_DownloadContext *dc); + + +/** + * Task that creates the initial (top-level) download + * request for the file. + * + * @param cls the 'struct GNUNET_FS_DownloadContext' + * @param tc scheduler context + */ +void +GNUNET_FS_download_start_task_ (void *cls, + const struct GNUNET_SCHEDULER_TaskContext *tc); + + + +/** + * Fill in all of the generic fields for + * an unindex event and call the callback. + * + * @param pi structure to fill in + * @param uc overall unindex context + * @param offset where we are in the file (for progress) + */ +void +GNUNET_FS_unindex_make_status_ (struct GNUNET_FS_ProgressInfo *pi, + struct GNUNET_FS_UnindexContext *uc, + uint64_t offset); + +/** + * Fill in all of the generic fields for a search event and + * call the callback. + * + * @param pi structure to fill in + * @param sc overall search context + * @return value returned by the callback + */ +void * +GNUNET_FS_search_make_status_ (struct GNUNET_FS_ProgressInfo *pi, + struct GNUNET_FS_SearchContext *sc); + + +/** + * Connect to the datastore and remove the blocks. + * + * @param uc context for the unindex operation. + */ +void +GNUNET_FS_unindex_do_remove_ (struct GNUNET_FS_UnindexContext *uc); + +/** + * Build the request and actually initiate the search using the + * GNUnet FS service. + * + * @param sc search context + * @return GNUNET_OK on success, GNUNET_SYSERR on error + */ +int +GNUNET_FS_search_start_searching_ (struct GNUNET_FS_SearchContext *sc); + +/** + * Start the downloading process (by entering the queue). + * + * @param dc our download context + */ +void +GNUNET_FS_download_start_downloading_ (struct GNUNET_FS_DownloadContext *dc); + + +/** + * Start download probes for the given search result. + * + * @param sr the search result + */ +void +GNUNET_FS_search_start_probe_ (struct GNUNET_FS_SearchResult *sr); + +/** + * Remove serialization/deserialization file from disk. + * + * @param h master context + * @param ext component of the path + * @param ent entity identifier + */ +void +GNUNET_FS_remove_sync_file_ (struct GNUNET_FS_Handle *h, const char *ext, + const char *ent); + + +/** + * Remove serialization/deserialization directory from disk. + * + * @param h master context + * @param ext component of the path + * @param uni unique name of parent + */ +void +GNUNET_FS_remove_sync_dir_ (struct GNUNET_FS_Handle *h, const char *ext, + const char *uni); + + +/** + * Synchronize this file-information struct with its mirror + * on disk. Note that all internal FS-operations that change + * file information data should already call "sync" internally, + * so this function is likely not useful for clients. + * + * @param fi the struct to sync + */ +void +GNUNET_FS_file_information_sync_ (struct GNUNET_FS_FileInformation *f); + +/** + * Synchronize this publishing struct with its mirror + * on disk. Note that all internal FS-operations that change + * publishing structs should already call "sync" internally, + * so this function is likely not useful for clients. + * + * @param pc the struct to sync + */ +void +GNUNET_FS_publish_sync_ (struct GNUNET_FS_PublishContext *pc); + +/** + * Synchronize this unindex struct with its mirror + * on disk. Note that all internal FS-operations that change + * publishing structs should already call "sync" internally, + * so this function is likely not useful for clients. + * + * @param uc the struct to sync + */ +void +GNUNET_FS_unindex_sync_ (struct GNUNET_FS_UnindexContext *uc); + +/** + * Synchronize this search struct with its mirror + * on disk. Note that all internal FS-operations that change + * publishing structs should already call "sync" internally, + * so this function is likely not useful for clients. + * + * @param sc the struct to sync + */ +void +GNUNET_FS_search_sync_ (struct GNUNET_FS_SearchContext *sc); + +/** + * Synchronize this search result with its mirror + * on disk. Note that all internal FS-operations that change + * publishing structs should already call "sync" internally, + * so this function is likely not useful for clients. + * + * @param sr the struct to sync + */ +void +GNUNET_FS_search_result_sync_ (struct GNUNET_FS_SearchResult *sr); + +/** + * Synchronize this download struct with its mirror + * on disk. Note that all internal FS-operations that change + * publishing structs should already call "sync" internally, + * so this function is likely not useful for clients. + * + * @param dc the struct to sync + */ +void +GNUNET_FS_download_sync_ (struct GNUNET_FS_DownloadContext *dc); + +/** + * Create SUSPEND event for the given publish operation + * and then clean up our state (without stop signal). + * + * @param cls the 'struct GNUNET_FS_PublishContext' to signal for + */ +void +GNUNET_FS_publish_signal_suspend_ (void *cls); + +/** + * Create SUSPEND event for the given search operation + * and then clean up our state (without stop signal). + * + * @param cls the 'struct GNUNET_FS_SearchContext' to signal for + */ +void +GNUNET_FS_search_signal_suspend_ (void *cls); + +/** + * Create SUSPEND event for the given download operation + * and then clean up our state (without stop signal). + * + * @param cls the 'struct GNUNET_FS_DownloadContext' to signal for + */ +void +GNUNET_FS_download_signal_suspend_ (void *cls); + +/** + * Create SUSPEND event for the given unindex operation + * and then clean up our state (without stop signal). + * + * @param cls the 'struct GNUNET_FS_UnindexContext' to signal for + */ +void +GNUNET_FS_unindex_signal_suspend_ (void *cls); + +/** + * Function signature of the functions that can be called + * to trigger suspend signals and clean-up for top-level + * activities. + * + * @param cls closure + */ +typedef void (*SuspendSignalFunction) (void *cls); + +/** + * We track all of the top-level activities of FS + * so that we can signal 'suspend' on shutdown. + */ +struct TopLevelActivity +{ + /** + * This is a doubly-linked list. + */ + struct TopLevelActivity *next; + + /** + * This is a doubly-linked list. + */ + struct TopLevelActivity *prev; + + /** + * Function to call for suspend-signalling and clean up. + */ + SuspendSignalFunction ssf; + + /** + * Closure for 'ssf' (some struct GNUNET_FS_XXXHandle*) + */ + void *ssf_cls; +}; + + +/** + * Create a top-level activity entry. + * + * @param h global fs handle + * @param ssf suspend signal function to use + * @param ssf_cls closure for ssf + * @return fresh top-level activity handle + */ +struct TopLevelActivity * +GNUNET_FS_make_top (struct GNUNET_FS_Handle *h, SuspendSignalFunction ssf, + void *ssf_cls); + + +/** + * Destroy a top-level activity entry. + * + * @param h global fs handle + * @param top top level activity entry + */ +void +GNUNET_FS_end_top (struct GNUNET_FS_Handle *h, struct TopLevelActivity *top); + + + +/** + * Master context for most FS operations. + */ +struct GNUNET_FS_Handle +{ + /** + * Configuration to use. + */ + const struct GNUNET_CONFIGURATION_Handle *cfg; + + /** + * Name of our client. + */ + char *client_name; + + /** + * Function to call with updates on our progress. + */ + GNUNET_FS_ProgressCallback upcb; + + /** + * Closure for upcb. + */ + void *upcb_cls; + + /** + * Head of DLL of top-level activities. + */ + struct TopLevelActivity *top_head; + + /** + * Tail of DLL of top-level activities. + */ + struct TopLevelActivity *top_tail; + + /** + * Head of DLL of running jobs. + */ + struct GNUNET_FS_QueueEntry *running_head; + + /** + * Tail of DLL of running jobs. + */ + struct GNUNET_FS_QueueEntry *running_tail; + + /** + * Head of DLL of pending jobs. + */ + struct GNUNET_FS_QueueEntry *pending_head; + + /** + * Tail of DLL of pending jobs. + */ + struct GNUNET_FS_QueueEntry *pending_tail; + + /** + * Task that processes the jobs in the running and pending queues + * (and moves jobs around as needed). + */ + GNUNET_SCHEDULER_TaskIdentifier queue_job; + + /** + * Average time we take for a single request to be satisfied. + * FIXME: not yet calcualted properly... + */ + struct GNUNET_TIME_Relative avg_block_latency; + + /** + * How many actual downloads do we have running right now? + */ + unsigned int active_downloads; + + /** + * How many blocks do the active downloads have? + */ + unsigned int active_blocks; + + /** + * General flags. + */ + enum GNUNET_FS_Flags flags; + + /** + * Maximum number of parallel downloads. + */ + unsigned int max_parallel_downloads; + + /** + * Maximum number of parallel requests. + */ + unsigned int max_parallel_requests; + +}; + + +/** + * Handle for controlling a publication process. + */ +struct GNUNET_FS_PublishContext +{ + /** + * Handle to the global fs context. + */ + struct GNUNET_FS_Handle *h; + + /** + * Our top-level activity entry (if we are top-level, otherwise NULL). + */ + struct TopLevelActivity *top; + + /** + * File-structure that is being shared. + */ + struct GNUNET_FS_FileInformation *fi; + + /** + * Namespace that we are publishing in, NULL if we have no namespace. + */ + struct GNUNET_FS_Namespace *namespace; + + /** + * ID of the content in the namespace, NULL if we have no namespace. + */ + char *nid; + + /** + * ID for future updates, NULL if we have no namespace or no updates. + */ + char *nuid; + + /** + * Filename used for serializing information about this operation + * (should be determined using 'mktemp'). + */ + char *serialization; + + /** + * Our own client handle for the FS service; only briefly used when + * we start to index a file, otherwise NULL. + */ + struct GNUNET_CLIENT_Connection *client; + + /** + * Current position in the file-tree for the upload. + */ + struct GNUNET_FS_FileInformation *fi_pos; + + /** + * Non-null if we are currently hashing a file. + */ + struct GNUNET_CRYPTO_FileHashContext *fhc; + + /** + * Connection to the datastore service. + */ + struct GNUNET_DATASTORE_Handle *dsh; + + /** + * Queue entry for reservation/unreservation. + */ + struct GNUNET_DATASTORE_QueueEntry *qre; + + /** + * Context for SKS publishing operation that is part of this publishing operation + * (NULL if not active). + */ + struct GNUNET_FS_PublishSksContext *sks_pc; + + /** + * Context for KSK publishing operation that is part of this publishing operation + * (NULL if not active). + */ + struct GNUNET_FS_PublishKskContext *ksk_pc; + + /** + * ID of the task performing the upload. NO_TASK if the upload has + * completed. + */ + GNUNET_SCHEDULER_TaskIdentifier upload_task; + + /** + * Storage space to reserve for the operation. + */ + uint64_t reserve_space; + + /** + * Overall number of entries to reserve for the + * publish operation. + */ + uint32_t reserve_entries; + + /** + * Options for publishing. + */ + enum GNUNET_FS_PublishOptions options; + + /** + * Space reservation ID with datastore service + * for this upload. + */ + int rid; + + /** + * Set to GNUNET_YES if all processing has completed. + */ + int all_done; + + /** + * Flag set to GNUNET_YES if the next callback from + * GNUNET_FS_file_information_inspect should be skipped because it + * is for the directory which was already processed with the parent. + */ + int skip_next_fi_callback; +}; + + +/** + * Phases of unindex processing (state machine). + */ +enum UnindexState +{ + /** + * We're currently hashing the file. + */ + UNINDEX_STATE_HASHING = 0, + + /** + * We're telling the datastore to delete + * the respective entries. + */ + UNINDEX_STATE_DS_REMOVE = 1, + + /** + * We're notifying the FS service about + * the unindexing. + */ + UNINDEX_STATE_FS_NOTIFY = 2, + + /** + * We're done. + */ + UNINDEX_STATE_COMPLETE = 3, + + /** + * We've encountered a fatal error. + */ + UNINDEX_STATE_ERROR = 4 +}; + + +/** + * Handle for controlling an unindexing operation. + */ +struct GNUNET_FS_UnindexContext +{ + + /** + * Global FS context. + */ + struct GNUNET_FS_Handle *h; + + /** + * Our top-level activity entry. + */ + struct TopLevelActivity *top; + + /** + * Name of the file that we are unindexing. + */ + char *filename; + + /** + * Short name under which we are serializing the state of this operation. + */ + char *serialization; + + /** + * Connection to the FS service, only valid during the + * UNINDEX_STATE_FS_NOTIFY phase. + */ + struct GNUNET_CLIENT_Connection *client; + + /** + * Connection to the datastore service, only valid during the + * UNINDEX_STATE_DS_NOTIFY phase. + */ + struct GNUNET_DATASTORE_Handle *dsh; + + /** + * Pointer kept for the client. + */ + void *client_info; + + /** + * Merkle-ish tree encoder context. + */ + struct GNUNET_FS_TreeEncoder *tc; + + /** + * Handle used to read the file. + */ + struct GNUNET_DISK_FileHandle *fh; + + /** + * Error message, NULL on success. + */ + char *emsg; + + /** + * Context for hashing of the file. + */ + struct GNUNET_CRYPTO_FileHashContext *fhc; + + /** + * Overall size of the file. + */ + uint64_t file_size; + + /** + * When did we start? + */ + struct GNUNET_TIME_Absolute start_time; + + /** + * Hash of the file's contents (once computed). + */ + GNUNET_HashCode file_id; + + /** + * Current operatinonal phase. + */ + enum UnindexState state; + +}; + + +/** + * Information we keep for each keyword in + * a keyword search. + */ +struct SearchRequestEntry +{ + /** + * Hash of the original keyword, also known as the + * key (for decrypting the KBlock). + */ + GNUNET_HashCode key; + + /** + * Hash of the public key, also known as the query. + */ + GNUNET_HashCode query; + + /** + * Map that contains a "struct GNUNET_FS_SearchResult" for each result that + * was found under this keyword. Note that the entries will point + * to the same locations as those in the master result map (in + * "struct GNUNET_FS_SearchContext"), so they should not be freed. + * The key for each entry is the XOR of the key and query in the CHK + * URI (as a unique identifier for the search result). + */ + struct GNUNET_CONTAINER_MultiHashMap *results; + + /** + * Is this keyword a mandatory keyword + * (started with '+')? + */ + int mandatory; + +}; + + +/** + * Handle for controlling a search. + */ +struct GNUNET_FS_SearchContext +{ + /** + * Handle to the global FS context. + */ + struct GNUNET_FS_Handle *h; + + /** + * Our top-level activity entry (if we are top-level, otherwise NULL). + */ + struct TopLevelActivity *top; + + /** + * List of keywords that we're looking for. + */ + struct GNUNET_FS_Uri *uri; + + /** + * For update-searches, link to the search result that triggered + * the update search; otherwise NULL. + */ + struct GNUNET_FS_SearchResult *psearch_result; + + /** + * Connection to the FS service. + */ + struct GNUNET_CLIENT_Connection *client; + + /** + * Pointer we keep for the client. + */ + void *client_info; + + /** + * Name of the file on disk we use for persistence. + */ + char *serialization; + + /** + * Error message (non-NULL if this operation failed). + */ + char *emsg; + + /** + * Map that contains a "struct GNUNET_FS_SearchResult" for each result that + * was found in the search. The key for each entry is the XOR of + * the key and query in the CHK URI (as a unique identifier for the + * search result). + */ + struct GNUNET_CONTAINER_MultiHashMap *master_result_map; + + /** + * Per-keyword information for a keyword search. This array will + * have exactly as many entries as there were keywords. + */ + struct SearchRequestEntry *requests; + + /** + * When did we start? + */ + struct GNUNET_TIME_Absolute start_time; + + /** + * ID of a task that is using this struct and that must be cancelled + * when the search is being stopped (if not + * GNUNET_SCHEDULER_NO_TASK). Used for the task that adds some + * artificial delay when trying to reconnect to the FS service. + */ + GNUNET_SCHEDULER_TaskIdentifier task; + + /** + * How many of the entries in the search request + * map have been passed to the service so far? + */ + unsigned int search_request_map_offset; + + /** + * How many of the keywords in the KSK + * map have been passed to the service so far? + */ + unsigned int keyword_offset; + + /** + * Anonymity level for the search. + */ + uint32_t anonymity; + + /** + * Number of mandatory keywords in this query. + */ + uint32_t mandatory_count; + + /** + * Options for the search. + */ + enum GNUNET_FS_SearchOptions options; +}; + + +/** + * FSM for possible states a block can go through. The typical + * order of progression is linear through the states, alternatives + * are documented in the comments. + */ +enum BlockRequestState +{ + /** + * Initial state, block has only been allocated (since it is + * relevant to the overall download request). + */ + BRS_INIT = 0, + + /** + * We've checked the block on the path down the tree, and the + * content on disk did match the desired CHK, but not all + * the way down, so at the bottom some blocks will still + * need to be reconstructed). + */ + BRS_RECONSTRUCT_DOWN = 1, + + /** + * We've calculated the CHK bottom-up based on the meta data. + * This may work, but if it did we have to write the meta data to + * disk at the end (and we still need to check against the + * CHK set on top). + */ + BRS_RECONSTRUCT_META_UP = 2, + + /** + * We've calculated the CHK bottom-up based on what we have on + * disk, which may not be what the desired CHK is. If the + * reconstructed CHKs match whatever comes from above, we're + * done with the respective subtree. + */ + BRS_RECONSTRUCT_UP = 3, + + /** + * We've determined the real, desired CHK for this block + * (full tree reconstruction failed), request is now pending. + * If the CHK that bubbled up through reconstruction did match + * the top-level request, the state machine for the subtree + * would have moved to BRS_DOWNLOAD_UP. + */ + BRS_CHK_SET = 4, + + /** + * We've successfully downloaded this block, but the children + * still need to be either downloaded or verified (download + * request propagates down). If the download fails, the + * state machine for this block may move to + * BRS_DOWNLOAD_ERROR instead. + */ + BRS_DOWNLOAD_DOWN = 5, + + /** + * This block and all of its children have been downloaded + * successfully (full completion propagates up). + */ + BRS_DOWNLOAD_UP = 6, + + /** + * We got a block back that matched the query but did not hash to + * the key (malicious publisher or hash collision); this block + * can never be downloaded (error propagates up). + */ + BRS_ERROR = 7 +}; + + +/** + * Information about an active download request. + */ +struct DownloadRequest +{ + /** + * While pending, we keep all download requests in a doubly-linked list. + */ + struct DownloadRequest *next; + + /** + * While pending, we keep all download requests in a doubly-linked list. + */ + struct DownloadRequest *prev; + + /** + * Parent in the CHK-tree. + */ + struct DownloadRequest *parent; + + /** + * Array (!) of child-requests, or NULL for the bottom of the tree. + */ + struct DownloadRequest **children; + + /** + * CHK for the request for this block (set during reconstruction + * to what we have on disk, later to what we want to have). + */ + struct ContentHashKey chk; + + /** + * Offset of the corresponding block. Specifically, first (!) byte of + * the first DBLOCK in the subtree induced by block represented by + * this request. + */ + uint64_t offset; + + /** + * Number of entries in 'children' array. + */ + unsigned int num_children; + + /** + * Depth of the corresponding block in the tree. 0==DBLOCKs. + */ + unsigned int depth; + + /** + * State in the FSM. + */ + enum BlockRequestState state; + + /** + * GNUNET_YES if this entry is in the pending list. + */ + int is_pending; + +}; + + +/** + * (recursively) free download request structure + * + * @param dr request to free + */ +void +GNUNET_FS_free_download_request_ (struct DownloadRequest *dr); + + +/** + * Context for controlling a download. + */ +struct GNUNET_FS_DownloadContext +{ + + /** + * Global FS context. + */ + struct GNUNET_FS_Handle *h; + + /** + * Our top-level activity entry (if we are top-level, otherwise NULL). + */ + struct TopLevelActivity *top; + + /** + * Connection to the FS service. + */ + struct GNUNET_CLIENT_Connection *client; + + /** + * Parent download (used when downloading files + * in directories). + */ + struct GNUNET_FS_DownloadContext *parent; + + /** + * Associated search (used when downloading files + * based on search results), or NULL for none. + */ + struct GNUNET_FS_SearchResult *search; + + /** + * Head of list of child downloads. + */ + struct GNUNET_FS_DownloadContext *child_head; + + /** + * Tail of list of child downloads. + */ + struct GNUNET_FS_DownloadContext *child_tail; + + /** + * Previous download belonging to the same parent. + */ + struct GNUNET_FS_DownloadContext *prev; + + /** + * Next download belonging to the same parent. + */ + struct GNUNET_FS_DownloadContext *next; + + /** + * Context kept for the client. + */ + void *client_info; + + /** + * URI that identifies the file that we are downloading. + */ + struct GNUNET_FS_Uri *uri; + + /** + * Known meta-data for the file (can be NULL). + */ + struct GNUNET_CONTAINER_MetaData *meta; + + /** + * Error message, NULL if we're doing OK. + */ + char *emsg; + + /** + * Random portion of filename we use for syncing state of this + * download. + */ + char *serialization; + + /** + * Where are we writing the data (name of the + * file, can be NULL!). + */ + char *filename; + + /** + * Where are we writing the data temporarily (name of the + * file, can be NULL!); used if we do not have a permanent + * name and we are a directory and we do a recursive download. + */ + char *temp_filename; + + /** + * Our entry in the job queue. + */ + struct GNUNET_FS_QueueEntry *job_queue; + + /** + * Non-NULL if we are currently having a request for + * transmission pending with the client handle. + */ + struct GNUNET_CLIENT_TransmitHandle *th; + + /** + * Tree encoder used for the reconstruction. + */ + struct GNUNET_FS_TreeEncoder *te; + + /** + * File handle for reading data from an existing file + * (to pass to tree encoder). + */ + struct GNUNET_DISK_FileHandle *rfh; + + /** + * Map of active requests (those waiting for a response). The key + * is the hash of the encryped block (aka query). + */ + struct GNUNET_CONTAINER_MultiHashMap *active; + + /** + * Head of linked list of pending requests. + */ + struct DownloadRequest *pending_head; + + /** + * Head of linked list of pending requests. + */ + struct DownloadRequest *pending_tail; + + /** + * Top-level download request. + */ + struct DownloadRequest *top_request; + + /** + * Identity of the peer having the content, or all-zeros + * if we don't know of such a peer. + */ + struct GNUNET_PeerIdentity target; + + /** + * ID of a task that is using this struct and that must be cancelled + * when the download is being stopped (if not + * GNUNET_SCHEDULER_NO_TASK). Used for the task that adds some + * artificial delay when trying to reconnect to the FS service or + * the task processing incrementally the data on disk, or the + * task requesting blocks, etc. + */ + GNUNET_SCHEDULER_TaskIdentifier task; + + /** + * What is the first offset that we're interested + * in? + */ + uint64_t offset; + + /** + * How many bytes starting from offset are desired? + * This is NOT the overall length of the file! + */ + uint64_t length; + + /** + * How many bytes have we already received within + * the specified range (DBlocks only). + */ + uint64_t completed; + + /** + * What was the size of the file on disk that we're downloading + * before we started? Used to detect if there is a point in + * checking an existing block on disk for matching the desired + * content. 0 if the file did not exist already. + */ + uint64_t old_file_size; + + /** + * Time download was started. + */ + struct GNUNET_TIME_Absolute start_time; + + /** + * Desired level of anonymity. + */ + uint32_t anonymity; + + /** + * The depth of the file-tree. + */ + unsigned int treedepth; + + /** + * Options for the download. + */ + enum GNUNET_FS_DownloadOptions options; + + /** + * Flag set upon transitive completion (includes child downloads). + * This flag is only set to GNUNET_YES for directories where all + * child-downloads have also completed (and signalled completion). + */ + int has_finished; + + /** + * Have we started the receive continuation yet? + */ + int in_receive; + +}; + + +/** + * Information about an (updateable) node in the + * namespace. + */ +struct NamespaceUpdateNode +{ + /** + * Identifier for this node. + */ + char *id; + + /** + * Identifier of children of this node. + */ + char *update; + + /** + * Metadata for this entry. + */ + struct GNUNET_CONTAINER_MetaData *md; + + /** + * URI of this entry in the namespace. + */ + struct GNUNET_FS_Uri *uri; + + /** + * Namespace update generation ID. Used to ensure + * freshness of the tree_id. + */ + unsigned int nug; + + /** + * TREE this entry belongs to (if nug is current). + */ + unsigned int tree_id; + +}; + + +struct GNUNET_FS_Namespace +{ + + /** + * Handle to the FS service context. + */ + struct GNUNET_FS_Handle *h; + + /** + * Array with information about nodes in the namespace. + */ + struct NamespaceUpdateNode **update_nodes; + + /** + * Private key for the namespace. + */ + struct GNUNET_CRYPTO_RsaPrivateKey *key; + + /** + * Hash map mapping identifiers of update nodes + * to the update nodes (initialized on-demand). + */ + struct GNUNET_CONTAINER_MultiHashMap *update_map; + + /** + * Name of the file with the private key. + */ + char *filename; + + /** + * Name of the namespace. + */ + char *name; + + /** + * Size of the update nodes array. + */ + unsigned int update_node_count; + + /** + * Reference counter. + */ + unsigned int rc; + + /** + * Generator for unique nug numbers. + */ + unsigned int nug_gen; +}; + +#endif + +/* end of fs_api.h */ diff --git a/src/fs/fs_directory.c b/src/fs/fs_directory.c new file mode 100644 index 0000000..b26ec12 --- /dev/null +++ b/src/fs/fs_directory.c @@ -0,0 +1,644 @@ +/* + This file is part of GNUnet. + (C) 2003, 2004, 2006, 2009 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/fs_directory.c + * @brief Helper functions for building directories. + * @author Christian Grothoff + * + * TODO: + * - modify directory builder API to support incremental + * generation of directories (to allow directories that + * would not fit into memory to be created) + * - modify directory processor API to support incremental + * iteration over FULL directories (without missing entries) + * to allow access to directories that do not fit entirely + * into memory + */ +#include "platform.h" +#include "gnunet_fs_service.h" +#include "fs_api.h" + +/** + * String that is used to indicate that a file + * is a GNUnet directory. + */ +#define GNUNET_DIRECTORY_MAGIC "\211GND\r\n\032\n" + + +/** + * Does the meta-data claim that this is a directory? + * Checks if the mime-type is that of a GNUnet directory. + * + * @return GNUNET_YES if it is, GNUNET_NO if it is not, GNUNET_SYSERR if + * we have no mime-type information (treat as 'GNUNET_NO') + */ +int +GNUNET_FS_meta_data_test_for_directory (const struct GNUNET_CONTAINER_MetaData + *md) +{ + char *mime; + int ret; + + if (NULL == md) + return GNUNET_SYSERR; + mime = + GNUNET_CONTAINER_meta_data_get_by_type (md, EXTRACTOR_METATYPE_MIMETYPE); + if (mime == NULL) + return GNUNET_SYSERR; + ret = (0 == strcmp (mime, GNUNET_FS_DIRECTORY_MIME)) ? GNUNET_YES : GNUNET_NO; + GNUNET_free (mime); + return ret; +} + + +/** + * Set the MIMETYPE information for the given + * metadata to "application/gnunet-directory". + * + * @param md metadata to add mimetype to + */ +void +GNUNET_FS_meta_data_make_directory (struct GNUNET_CONTAINER_MetaData *md) +{ + char *mime; + + mime = + GNUNET_CONTAINER_meta_data_get_by_type (md, EXTRACTOR_METATYPE_MIMETYPE); + if (mime != NULL) + { + GNUNET_break (0 == strcmp (mime, GNUNET_FS_DIRECTORY_MIME)); + GNUNET_free (mime); + return; + } + GNUNET_CONTAINER_meta_data_insert (md, "<gnunet>", + EXTRACTOR_METATYPE_MIMETYPE, + EXTRACTOR_METAFORMAT_UTF8, "text/plain", + GNUNET_FS_DIRECTORY_MIME, + strlen (GNUNET_FS_DIRECTORY_MIME) + 1); +} + + +/** + * Closure for 'find_full_data'. + */ +struct GetFullDataClosure +{ + + /** + * Extracted binary meta data. + */ + void *data; + + /** + * Number of bytes stored in data. + */ + size_t size; +}; + + +/** + * Type of a function that libextractor calls for each + * meta data item found. + * + * @param cls closure (user-defined) + * @param plugin_name name of the plugin that produced this value; + * special values can be used (i.e. '<zlib>' for zlib being + * used in the main libextractor library and yielding + * meta data). + * @param type libextractor-type describing the meta data + * @param format basic format information about data + * @param data_mime_type mime-type of data (not of the original file); + * can be NULL (if mime-type is not known) + * @param data actual meta-data found + * @param data_len number of bytes in data + * @return 0 to continue extracting, 1 to abort + */ +static int +find_full_data (void *cls, const char *plugin_name, + enum EXTRACTOR_MetaType type, enum EXTRACTOR_MetaFormat format, + const char *data_mime_type, const char *data, size_t data_len) +{ + struct GetFullDataClosure *gfdc = cls; + + if (type == EXTRACTOR_METATYPE_GNUNET_FULL_DATA) + { + gfdc->size = data_len; + if (data_len > 0) + { + gfdc->data = GNUNET_malloc (data_len); + memcpy (gfdc->data, data, data_len); + } + return 1; + } + return 0; +} + + +/** + * Iterate over all entries in a directory. Note that directories + * are structured such that it is possible to iterate over the + * individual blocks as well as over the entire directory. Thus + * a client can call this function on the buffer in the + * GNUNET_FS_ProgressCallback. Also, directories can optionally + * include the contents of (small) files embedded in the directory + * itself; for those files, the processor may be given the + * contents of the file directly by this function. + * <p> + * + * Note that this function maybe called on parts of directories. Thus + * parser errors should not be reported _at all_ (with GNUNET_break). + * Still, if some entries can be recovered despite these parsing + * errors, the function should try to do this. + * + * @param size number of bytes in data + * @param data pointer to the beginning of the directory + * @param offset offset of data in the directory + * @param dep function to call on each entry + * @param dep_cls closure for dep + * @return GNUNET_OK if this could be a block in a directory, + * GNUNET_NO if this could be part of a directory (but not 100% OK) + * GNUNET_SYSERR if 'data' does not represent a directory + */ +int +GNUNET_FS_directory_list_contents (size_t size, const void *data, + uint64_t offset, + GNUNET_FS_DirectoryEntryProcessor dep, + void *dep_cls) +{ + struct GetFullDataClosure full_data; + const char *cdata = data; + char *emsg; + uint64_t pos; + uint64_t align; + uint32_t mdSize; + uint64_t epos; + struct GNUNET_FS_Uri *uri; + struct GNUNET_CONTAINER_MetaData *md; + char *filename; + + if ((offset == 0) && + ((size < 8 + sizeof (uint32_t)) || + (0 != memcmp (cdata, GNUNET_FS_DIRECTORY_MAGIC, 8)))) + return GNUNET_SYSERR; + pos = offset; + if (offset == 0) + { + memcpy (&mdSize, &cdata[8], sizeof (uint32_t)); + mdSize = ntohl (mdSize); + if (mdSize > size - 8 - sizeof (uint32_t)) + { + /* invalid size */ + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _("MAGIC mismatch. This is not a GNUnet directory.\n")); + return GNUNET_SYSERR; + } + md = GNUNET_CONTAINER_meta_data_deserialize (&cdata[8 + sizeof (uint32_t)], + mdSize); + if (md == NULL) + { + GNUNET_break (0); + return GNUNET_SYSERR; /* malformed ! */ + } + dep (dep_cls, NULL, NULL, md, 0, NULL); + GNUNET_CONTAINER_meta_data_destroy (md); + pos = 8 + sizeof (uint32_t) + mdSize; + } + while (pos < size) + { + /* find end of URI */ + if (cdata[pos] == '\0') + { + /* URI is never empty, must be end of block, + * skip to next alignment */ + align = ((pos / DBLOCK_SIZE) + 1) * DBLOCK_SIZE; + if (align == pos) + { + /* if we were already aligned, still skip a block! */ + align += DBLOCK_SIZE; + } + pos = align; + if (pos >= size) + { + /* malformed - or partial download... */ + break; + } + } + epos = pos; + while ((epos < size) && (cdata[epos] != '\0')) + epos++; + if (epos >= size) + return GNUNET_NO; /* malformed - or partial download */ + + uri = GNUNET_FS_uri_parse (&cdata[pos], &emsg); + pos = epos + 1; + if (uri == NULL) + { + GNUNET_free (emsg); + pos--; /* go back to '\0' to force going to next alignment */ + continue; + } + if (GNUNET_FS_uri_test_ksk (uri)) + { + GNUNET_FS_uri_destroy (uri); + GNUNET_break (0); + return GNUNET_NO; /* illegal in directory! */ + } + + memcpy (&mdSize, &cdata[pos], sizeof (uint32_t)); + mdSize = ntohl (mdSize); + pos += sizeof (uint32_t); + if (pos + mdSize > size) + { + GNUNET_FS_uri_destroy (uri); + return GNUNET_NO; /* malformed - or partial download */ + } + + md = GNUNET_CONTAINER_meta_data_deserialize (&cdata[pos], mdSize); + if (md == NULL) + { + GNUNET_FS_uri_destroy (uri); + GNUNET_break (0); + return GNUNET_NO; /* malformed ! */ + } + pos += mdSize; + filename = + GNUNET_CONTAINER_meta_data_get_by_type (md, + EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME); + full_data.size = 0; + full_data.data = NULL; + GNUNET_CONTAINER_meta_data_iterate (md, &find_full_data, &full_data); + if (dep != NULL) + { + dep (dep_cls, filename, uri, md, full_data.size, full_data.data); + } + GNUNET_free_non_null (full_data.data); + GNUNET_free_non_null (filename); + GNUNET_CONTAINER_meta_data_destroy (md); + GNUNET_FS_uri_destroy (uri); + } + return GNUNET_OK; +} + +/** + * Entries in the directory (builder). + */ +struct BuilderEntry +{ + /** + * This is a linked list. + */ + struct BuilderEntry *next; + + /** + * Length of this entry. + */ + size_t len; +}; + +/** + * Internal state of a directory builder. + */ +struct GNUNET_FS_DirectoryBuilder +{ + /** + * Meta-data for the directory itself. + */ + struct GNUNET_CONTAINER_MetaData *meta; + + /** + * Head of linked list of entries. + */ + struct BuilderEntry *head; + + /** + * Number of entires in the directory. + */ + unsigned int count; +}; + + +/** + * Create a directory builder. + * + * @param mdir metadata for the directory + */ +struct GNUNET_FS_DirectoryBuilder * +GNUNET_FS_directory_builder_create (const struct GNUNET_CONTAINER_MetaData + *mdir) +{ + struct GNUNET_FS_DirectoryBuilder *ret; + + ret = GNUNET_malloc (sizeof (struct GNUNET_FS_DirectoryBuilder)); + if (mdir != NULL) + ret->meta = GNUNET_CONTAINER_meta_data_duplicate (mdir); + else + ret->meta = GNUNET_CONTAINER_meta_data_create (); + GNUNET_FS_meta_data_make_directory (ret->meta); + return ret; +} + + +/** + * Add an entry to a directory. + * + * @param bld directory to extend + * @param uri uri of the entry (must not be a KSK) + * @param md metadata of the entry + * @param data raw data of the entry, can be NULL, otherwise + * data must point to exactly the number of bytes specified + * by the uri which must be of type LOC or CHK + */ +void +GNUNET_FS_directory_builder_add (struct GNUNET_FS_DirectoryBuilder *bld, + const struct GNUNET_FS_Uri *uri, + const struct GNUNET_CONTAINER_MetaData *md, + const void *data) +{ + struct GNUNET_FS_Uri *curi; + struct BuilderEntry *e; + uint64_t fsize; + uint32_t big; + ssize_t ret; + size_t mds; + size_t mdxs; + char *uris; + char *ser; + char *sptr; + size_t slen; + struct GNUNET_CONTAINER_MetaData *meta; + const struct GNUNET_CONTAINER_MetaData *meta_use; + + GNUNET_assert (!GNUNET_FS_uri_test_ksk (uri)); + if (NULL != data) + { + GNUNET_assert (!GNUNET_FS_uri_test_sks (uri)); + if (GNUNET_FS_uri_test_chk (uri)) + { + fsize = GNUNET_FS_uri_chk_get_file_size (uri); + } + else + { + curi = GNUNET_FS_uri_loc_get_uri (uri); + GNUNET_assert (NULL != curi); + fsize = GNUNET_FS_uri_chk_get_file_size (curi); + GNUNET_FS_uri_destroy (curi); + } + } + else + { + fsize = 0; /* not given */ + } + if (fsize > MAX_INLINE_SIZE) + fsize = 0; /* too large */ + uris = GNUNET_FS_uri_to_string (uri); + slen = strlen (uris) + 1; + mds = GNUNET_CONTAINER_meta_data_get_serialized_size (md); + meta_use = md; + meta = NULL; + if (fsize > 0) + { + meta = GNUNET_CONTAINER_meta_data_duplicate (md); + GNUNET_CONTAINER_meta_data_insert (meta, "<gnunet>", + EXTRACTOR_METATYPE_GNUNET_FULL_DATA, + EXTRACTOR_METAFORMAT_BINARY, NULL, data, + fsize); + mdxs = GNUNET_CONTAINER_meta_data_get_serialized_size (meta); + if ((slen + sizeof (uint32_t) + mdxs - 1) / DBLOCK_SIZE == + (slen + sizeof (uint32_t) + mds - 1) / DBLOCK_SIZE) + { + /* adding full data would not cause us to cross + * additional blocks, so add it! */ + meta_use = meta; + mds = mdxs; + } + } + + if (mds > GNUNET_MAX_MALLOC_CHECKED / 2) + mds = GNUNET_MAX_MALLOC_CHECKED / 2; + e = GNUNET_malloc (sizeof (struct BuilderEntry) + slen + mds + + sizeof (uint32_t)); + ser = (char *) &e[1]; + memcpy (ser, uris, slen); + GNUNET_free (uris); + sptr = &ser[slen + sizeof (uint32_t)]; + ret = + GNUNET_CONTAINER_meta_data_serialize (meta_use, &sptr, mds, + GNUNET_CONTAINER_META_DATA_SERIALIZE_PART); + if (NULL != meta) + GNUNET_CONTAINER_meta_data_destroy (meta); + if (ret == -1) + mds = 0; + else + mds = ret; + big = htonl (mds); + memcpy (&ser[slen], &big, sizeof (uint32_t)); + e->len = slen + sizeof (uint32_t) + mds; + e->next = bld->head; + bld->head = e; + bld->count++; +} + + +/** + * Given the start and end position of a block of + * data, return the end position of that data + * after alignment to the DBLOCK_SIZE. + */ +static size_t +do_align (size_t start_position, size_t end_position) +{ + size_t align; + + align = (end_position / DBLOCK_SIZE) * DBLOCK_SIZE; + if ((start_position < align) && (end_position > align)) + return align + end_position - start_position; + return end_position; +} + + +/** + * Compute a permuation of the blocks to + * minimize the cost of alignment. Greedy packer. + * + * @param start starting position for the first block + * @param count size of the two arrays + * @param sizes the sizes of the individual blocks + * @param perm the permutation of the blocks (updated) + */ +static void +block_align (size_t start, unsigned int count, const size_t * sizes, + unsigned int *perm) +{ + unsigned int i; + unsigned int j; + unsigned int tmp; + unsigned int best; + ssize_t badness; + size_t cpos; + size_t cend; + ssize_t cbad; + unsigned int cval; + + cpos = start; + for (i = 0; i < count; i++) + { + start = cpos; + badness = 0x7FFFFFFF; + best = -1; + for (j = i; j < count; j++) + { + cval = perm[j]; + cend = cpos + sizes[cval]; + if (cpos % DBLOCK_SIZE == 0) + { + /* prefer placing the largest blocks first */ + cbad = -(cend % DBLOCK_SIZE); + } + else + { + if (cpos / DBLOCK_SIZE == cend / DBLOCK_SIZE) + { + /* Data fits into the same block! Prefer small left-overs! */ + cbad = DBLOCK_SIZE - cend % DBLOCK_SIZE; + } + else + { + /* Would have to waste space to re-align, add big factor, this + * case is a real loss (proportional to space wasted)! */ + cbad = DBLOCK_SIZE * (DBLOCK_SIZE - cpos % DBLOCK_SIZE); + } + } + if (cbad < badness) + { + best = j; + badness = cbad; + } + } + GNUNET_assert (best != -1); + tmp = perm[i]; + perm[i] = perm[best]; + perm[best] = tmp; + cpos += sizes[perm[i]]; + cpos = do_align (start, cpos); + } +} + + +/** + * Finish building the directory. Frees the + * builder context and returns the directory + * in-memory. + * + * @param bld directory to finish + * @param rsize set to the number of bytes needed + * @param rdata set to the encoded directory + * @return GNUNET_OK on success + */ +int +GNUNET_FS_directory_builder_finish (struct GNUNET_FS_DirectoryBuilder *bld, + size_t * rsize, void **rdata) +{ + char *data; + char *sptr; + size_t *sizes; + unsigned int *perm; + unsigned int i; + unsigned int j; + struct BuilderEntry *pos; + struct BuilderEntry **bes; + size_t size; + size_t psize; + size_t off; + ssize_t ret; + uint32_t big; + + size = strlen (GNUNET_DIRECTORY_MAGIC) + sizeof (uint32_t); + size += GNUNET_CONTAINER_meta_data_get_serialized_size (bld->meta); + sizes = NULL; + perm = NULL; + bes = NULL; + if (0 < bld->count) + { + sizes = GNUNET_malloc (bld->count * sizeof (size_t)); + perm = GNUNET_malloc (bld->count * sizeof (unsigned int)); + bes = GNUNET_malloc (bld->count * sizeof (struct BuilderEntry *)); + pos = bld->head; + for (i = 0; i < bld->count; i++) + { + perm[i] = i; + bes[i] = pos; + sizes[i] = pos->len; + pos = pos->next; + } + block_align (size, bld->count, sizes, perm); + /* compute final size with alignment */ + for (i = 0; i < bld->count; i++) + { + psize = size; + size += sizes[perm[i]]; + size = do_align (psize, size); + } + } + *rsize = size; + data = GNUNET_malloc_large (size); + if (data == NULL) + { + GNUNET_log_strerror (GNUNET_ERROR_TYPE_ERROR, "malloc"); + *rsize = 0; + *rdata = NULL; + GNUNET_free_non_null (sizes); + GNUNET_free_non_null (perm); + GNUNET_free_non_null (bes); + return GNUNET_SYSERR; + } + *rdata = data; + memcpy (data, GNUNET_DIRECTORY_MAGIC, strlen (GNUNET_DIRECTORY_MAGIC)); + off = strlen (GNUNET_DIRECTORY_MAGIC); + + sptr = &data[off + sizeof (uint32_t)]; + ret = + GNUNET_CONTAINER_meta_data_serialize (bld->meta, &sptr, + size - off - sizeof (uint32_t), + GNUNET_CONTAINER_META_DATA_SERIALIZE_FULL); + GNUNET_assert (ret != -1); + big = htonl (ret); + memcpy (&data[off], &big, sizeof (uint32_t)); + off += sizeof (uint32_t) + ret; + for (j = 0; j < bld->count; j++) + { + i = perm[j]; + psize = off; + off += sizes[i]; + off = do_align (psize, off); + memcpy (&data[off - sizes[i]], &(bes[i])[1], sizes[i]); + GNUNET_free (bes[i]); + } + GNUNET_free_non_null (sizes); + GNUNET_free_non_null (perm); + GNUNET_free_non_null (bes); + GNUNET_assert (off == size); + GNUNET_CONTAINER_meta_data_destroy (bld->meta); + GNUNET_free (bld); + return GNUNET_OK; +} + + +/* end of fs_directory.c */ diff --git a/src/fs/fs_dirmetascan.c b/src/fs/fs_dirmetascan.c new file mode 100644 index 0000000..4e5354e --- /dev/null +++ b/src/fs/fs_dirmetascan.c @@ -0,0 +1,465 @@ +/* + This file is part of GNUnet + (C) 2005-2012 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 2, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/fs_dirmetascan.c + * @brief code to asynchronously build a 'struct GNUNET_FS_ShareTreeItem' + * from an on-disk directory for publishing; use the 'gnunet-helper-fs-publish'. + * @author LRN + * @author Christian Grothoff + */ +#include "platform.h" +#include "gnunet_fs_service.h" +#include "gnunet_scheduler_lib.h" +#include <pthread.h> + + +/** + * An opaque structure a pointer to which is returned to the + * caller to be used to control the scanner. + */ +struct GNUNET_FS_DirScanner +{ + + /** + * Helper process. + */ + struct GNUNET_HELPER_Handle *helper; + + /** + * Expanded filename (as given by the scan initiator). + * The scanner thread stores a copy here, and frees it when it finishes. + */ + char *filename_expanded; + + /** + * Second argument to helper process. + */ + char *ex_arg; + + /** + * The function that will be called every time there's a progress + * message. + */ + GNUNET_FS_DirScannerProgressCallback progress_callback; + + /** + * A closure for progress_callback. + */ + void *progress_callback_cls; + + /** + * After the scan is finished, it will contain a pointer to the + * top-level directory entry in the directory tree built by the + * scanner. + */ + struct GNUNET_FS_ShareTreeItem *toplevel; + + /** + * Current position during processing. + */ + struct GNUNET_FS_ShareTreeItem *pos; + + /** + * Task scheduled when we are done. + */ + GNUNET_SCHEDULER_TaskIdentifier stop_task; + + /** + * Arguments for helper. + */ + char *args[4]; + +}; + + + +/** + * Abort the scan. Must not be called from within the progress_callback + * function. + * + * @param ds directory scanner structure + */ +void +GNUNET_FS_directory_scan_abort (struct GNUNET_FS_DirScanner *ds) +{ + /* terminate helper */ + if (NULL != ds->helper) + GNUNET_HELPER_stop (ds->helper); + + /* free resources */ + if (NULL != ds->toplevel) + GNUNET_FS_share_tree_free (ds->toplevel); + if (GNUNET_SCHEDULER_NO_TASK != ds->stop_task) + GNUNET_SCHEDULER_cancel (ds->stop_task); + GNUNET_free_non_null (ds->ex_arg); + GNUNET_free (ds->filename_expanded); + GNUNET_free (ds); +} + + +/** + * Obtain the result of the scan after the scan has signalled + * completion. Must not be called prior to completion. The 'ds' is + * freed as part of this call. + * + * @param ds directory scanner structure + * @return the results of the scan (a directory tree) + */ +struct GNUNET_FS_ShareTreeItem * +GNUNET_FS_directory_scan_get_result (struct GNUNET_FS_DirScanner *ds) +{ + struct GNUNET_FS_ShareTreeItem *result; + + /* check that we're actually done */ + GNUNET_assert (NULL == ds->helper); + /* preserve result */ + result = ds->toplevel; + ds->toplevel = NULL; + GNUNET_FS_directory_scan_abort (ds); + return result; +} + + +/** + * Move in the directory from the given position to the next file + * in DFS traversal. + * + * @param pos current position + * @return next file, NULL for none + */ +static struct GNUNET_FS_ShareTreeItem * +advance (struct GNUNET_FS_ShareTreeItem *pos) +{ + int moved; + + GNUNET_assert (NULL != pos); + moved = 0; /* must not terminate, even on file, otherwise "normal" */ + while ( (pos->is_directory == GNUNET_YES) || + (0 == moved) ) + { + if ( (moved != -1) && + (NULL != pos->children_head) ) + { + pos = pos->children_head; + moved = 1; /* can terminate if file */ + continue; + } + if (NULL != pos->next) + { + pos = pos->next; + moved = 1; /* can terminate if file */ + continue; + } + if (NULL != pos->parent) + { + pos = pos->parent; + moved = -1; /* force move to 'next' or 'parent' */ + continue; + } + /* no more options, end of traversal */ + return NULL; + } + return pos; +} + + +/** + * Add another child node to the tree. + * + * @param parent parent of the child, NULL for top level + * @param filename name of the file or directory + * @param is_directory GNUNET_YES for directories + * @return new entry that was just created + */ +static struct GNUNET_FS_ShareTreeItem * +expand_tree (struct GNUNET_FS_ShareTreeItem *parent, + const char *filename, + int is_directory) +{ + struct GNUNET_FS_ShareTreeItem *chld; + size_t slen; + + chld = GNUNET_malloc (sizeof (struct GNUNET_FS_ShareTreeItem)); + chld->parent = parent; + chld->filename = GNUNET_strdup (filename); + GNUNET_asprintf (&chld->short_filename, + "%s%s", + GNUNET_STRINGS_get_short_name (filename), + is_directory == GNUNET_YES ? "/" : ""); + /* make sure we do not end with '//' */ + slen = strlen (chld->short_filename); + if ( (slen >= 2) && + (chld->short_filename[slen-1] == '/') && + (chld->short_filename[slen-2] == '/') ) + chld->short_filename[slen-1] = '\0'; + chld->is_directory = is_directory; + if (NULL != parent) + GNUNET_CONTAINER_DLL_insert (parent->children_head, + parent->children_tail, + chld); + return chld; +} + + +/** + * Task run last to shut everything down. + * + * @param cls the 'struct GNUNET_FS_DirScanner' + * @param tc unused + */ +static void +finish_scan (void *cls, + const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct GNUNET_FS_DirScanner *ds = cls; + + ds->stop_task = GNUNET_SCHEDULER_NO_TASK; + GNUNET_HELPER_stop (ds->helper); + ds->helper = NULL; + ds->progress_callback (ds->progress_callback_cls, + NULL, GNUNET_SYSERR, + GNUNET_FS_DIRSCANNER_FINISHED); +} + + +/** + * Called every time there is data to read from the scanner. + * Calls the scanner progress handler. + * + * @param cls the closure (directory scanner object) + * @param client always NULL + * @param msg message from the helper process + */ +static void +process_helper_msgs (void *cls, + void *client, + const struct GNUNET_MessageHeader *msg) +{ + struct GNUNET_FS_DirScanner *ds = cls; + const char *filename; + size_t left; + + left = ntohs (msg->size) - sizeof (struct GNUNET_MessageHeader); + filename = (const char*) &msg[1]; + switch (ntohs (msg->type)) + { + case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_FILE: + if (filename[left-1] != '\0') + { + GNUNET_break (0); + break; + } + ds->progress_callback (ds->progress_callback_cls, + filename, GNUNET_NO, + GNUNET_FS_DIRSCANNER_FILE_START); + if (NULL == ds->toplevel) + ds->toplevel = expand_tree (ds->pos, + filename, GNUNET_NO); + else + (void) expand_tree (ds->pos, + filename, GNUNET_NO); + return; + case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_DIRECTORY: + if (filename[left-1] != '\0') + { + GNUNET_break (0); + break; + } + if (0 == strcmp ("..", filename)) + { + if (NULL == ds->pos) + { + GNUNET_break (0); + break; + } + ds->pos = ds->pos->parent; + return; + } + ds->progress_callback (ds->progress_callback_cls, + filename, GNUNET_YES, + GNUNET_FS_DIRSCANNER_FILE_START); + ds->pos = expand_tree (ds->pos, + filename, GNUNET_YES); + if (NULL == ds->toplevel) + ds->toplevel = ds->pos; + return; + case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_ERROR: + break; + case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_SKIP_FILE: + if (filename[left-1] != '\0') + break; + ds->progress_callback (ds->progress_callback_cls, + filename, GNUNET_SYSERR, + GNUNET_FS_DIRSCANNER_FILE_IGNORED); + return; + case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_COUNTING_DONE: + if (0 != left) + { + GNUNET_break (0); + break; + } + if (NULL == ds->toplevel) + { + GNUNET_break (0); + break; + } + ds->progress_callback (ds->progress_callback_cls, + NULL, GNUNET_SYSERR, + GNUNET_FS_DIRSCANNER_ALL_COUNTED); + ds->pos = ds->toplevel; + if (ds->pos->is_directory == GNUNET_YES) + ds->pos = advance (ds->pos); + return; + case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_META_DATA: + { + size_t nlen; + const char *end; + + if (NULL == ds->pos) + { + GNUNET_break (0); + break; + } + end = memchr (filename, 0, left); + if (NULL == end) + { + GNUNET_break (0); + break; + } + end++; + nlen = end - filename; + left -= nlen; + if (0 != strcmp (filename, + ds->pos->filename)) + { + GNUNET_break (0); + break; + } + ds->progress_callback (ds->progress_callback_cls, + filename, GNUNET_YES, + GNUNET_FS_DIRSCANNER_EXTRACT_FINISHED); + if (0 < left) + { + ds->pos->meta = GNUNET_CONTAINER_meta_data_deserialize (end, left); + if (NULL == ds->pos->meta) + { + GNUNET_break (0); + break; + } + /* having full filenames is too dangerous; always make sure we clean them up */ + GNUNET_CONTAINER_meta_data_delete (ds->pos->meta, + EXTRACTOR_METATYPE_FILENAME, + NULL, 0); + /* instead, put in our 'safer' original filename */ + GNUNET_CONTAINER_meta_data_insert (ds->pos->meta, "<libgnunetfs>", + EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME, + EXTRACTOR_METAFORMAT_UTF8, "text/plain", + ds->pos->short_filename, + strlen (ds->pos->short_filename) + 1); + } + ds->pos->ksk_uri = GNUNET_FS_uri_ksk_create_from_meta_data (ds->pos->meta); + ds->pos = advance (ds->pos); + return; + } + case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_FINISHED: + if (NULL != ds->pos) + { + GNUNET_break (0); + break; + } + if (0 != left) + { + GNUNET_break (0); + break; + } + if (NULL == ds->toplevel) + { + GNUNET_break (0); + break; + } + ds->stop_task = GNUNET_SCHEDULER_add_now (&finish_scan, + ds); + return; + default: + GNUNET_break (0); + break; + } + ds->progress_callback (ds->progress_callback_cls, + NULL, GNUNET_SYSERR, + GNUNET_FS_DIRSCANNER_INTERNAL_ERROR); +} + + +/** + * Start a directory scanner thread. + * + * @param filename name of the directory to scan + * @param disable_extractor GNUNET_YES to not to run libextractor on files (only build a tree) + * @param ex if not NULL, must be a list of extra plugins for extractor + * @param cb the callback to call when there are scanning progress messages + * @param cb_cls closure for 'cb' + * @return directory scanner object to be used for controlling the scanner + */ +struct GNUNET_FS_DirScanner * +GNUNET_FS_directory_scan_start (const char *filename, + int disable_extractor, const char *ex, + GNUNET_FS_DirScannerProgressCallback cb, + void *cb_cls) +{ + struct stat sbuf; + char *filename_expanded; + struct GNUNET_FS_DirScanner *ds; + + if (0 != STAT (filename, &sbuf)) + return NULL; + filename_expanded = GNUNET_STRINGS_filename_expand (filename); + if (NULL == filename_expanded) + return NULL; + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Starting to scan directory `%s'\n", + filename_expanded); + ds = GNUNET_malloc (sizeof (struct GNUNET_FS_DirScanner)); + ds->progress_callback = cb; + ds->progress_callback_cls = cb_cls; + ds->filename_expanded = filename_expanded; + if (disable_extractor) + ds->ex_arg = GNUNET_strdup ("-"); + else + ds->ex_arg = (NULL != ex) ? GNUNET_strdup (ex) : NULL; + ds->args[0] = "gnunet-helper-fs-publish"; + ds->args[1] = ds->filename_expanded; + ds->args[2] = ds->ex_arg; + ds->args[3] = NULL; + ds->helper = GNUNET_HELPER_start ("gnunet-helper-fs-publish", + ds->args, + &process_helper_msgs, + ds); + if (NULL == ds->helper) + { + GNUNET_free (filename_expanded); + GNUNET_free (ds); + return NULL; + } + return ds; +} + + +/* end of fs_dirmetascan.c */ diff --git a/src/fs/fs_download.c b/src/fs/fs_download.c new file mode 100644 index 0000000..b23d14b --- /dev/null +++ b/src/fs/fs_download.c @@ -0,0 +1,2260 @@ +/* + This file is part of GNUnet. + (C) 2001-2011 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ +/** + * @file fs/fs_download.c + * @brief download methods + * @author Christian Grothoff + * + * TODO: + * - different priority for scheduling probe downloads? + */ +#include "platform.h" +#include "gnunet_constants.h" +#include "gnunet_fs_service.h" +#include "fs_api.h" +#include "fs_tree.h" + + +/** + * Determine if the given download (options and meta data) should cause + * use to try to do a recursive download. + */ +static int +is_recursive_download (struct GNUNET_FS_DownloadContext *dc) +{ + return (0 != (dc->options & GNUNET_FS_DOWNLOAD_OPTION_RECURSIVE)) && + ((GNUNET_YES == GNUNET_FS_meta_data_test_for_directory (dc->meta)) || + ((dc->meta == NULL) && + ((NULL == dc->filename) || + ((strlen (dc->filename) >= strlen (GNUNET_FS_DIRECTORY_EXT)) && + (NULL != + strstr (dc->filename + strlen (dc->filename) - + strlen (GNUNET_FS_DIRECTORY_EXT), + GNUNET_FS_DIRECTORY_EXT)))))); +} + + +/** + * We're storing the IBLOCKS after the DBLOCKS on disk (so that we + * only have to truncate the file once we're done). + * + * Given the offset of a block (with respect to the DBLOCKS) and its + * depth, return the offset where we would store this block in the + * file. + * + * @param fsize overall file size + * @param off offset of the block in the file + * @param depth depth of the block in the tree, 0 for DBLOCK + * @return off for DBLOCKS (depth == treedepth), + * otherwise an offset past the end + * of the file that does not overlap + * with the range for any other block + */ +static uint64_t +compute_disk_offset (uint64_t fsize, uint64_t off, unsigned int depth) +{ + unsigned int i; + uint64_t lsize; /* what is the size of all IBlocks for depth "i"? */ + uint64_t loff; /* where do IBlocks for depth "i" start? */ + unsigned int ioff; /* which IBlock corresponds to "off" at depth "i"? */ + + if (depth == 0) + return off; + /* first IBlocks start at the end of file, rounded up + * to full DBLOCK_SIZE */ + loff = ((fsize + DBLOCK_SIZE - 1) / DBLOCK_SIZE) * DBLOCK_SIZE; + lsize = + ((fsize + DBLOCK_SIZE - + 1) / DBLOCK_SIZE) * sizeof (struct ContentHashKey); + GNUNET_assert (0 == (off % DBLOCK_SIZE)); + ioff = (off / DBLOCK_SIZE); + for (i = 1; i < depth; i++) + { + loff += lsize; + lsize = (lsize + CHK_PER_INODE - 1) / CHK_PER_INODE; + GNUNET_assert (lsize > 0); + GNUNET_assert (0 == (ioff % CHK_PER_INODE)); + ioff /= CHK_PER_INODE; + } + return loff + ioff * sizeof (struct ContentHashKey); +} + + +/** + * Fill in all of the generic fields for a download event and call the + * callback. + * + * @param pi structure to fill in + * @param dc overall download context + */ +void +GNUNET_FS_download_make_status_ (struct GNUNET_FS_ProgressInfo *pi, + struct GNUNET_FS_DownloadContext *dc) +{ + pi->value.download.dc = dc; + pi->value.download.cctx = dc->client_info; + pi->value.download.pctx = + (dc->parent == NULL) ? NULL : dc->parent->client_info; + pi->value.download.sctx = + (dc->search == NULL) ? NULL : dc->search->client_info; + pi->value.download.uri = dc->uri; + pi->value.download.filename = dc->filename; + pi->value.download.size = dc->length; + /* FIXME: Fix duration calculation to account for pauses */ + pi->value.download.duration = + GNUNET_TIME_absolute_get_duration (dc->start_time); + pi->value.download.completed = dc->completed; + pi->value.download.anonymity = dc->anonymity; + pi->value.download.eta = + GNUNET_TIME_calculate_eta (dc->start_time, dc->completed, dc->length); + pi->value.download.is_active = (dc->client == NULL) ? GNUNET_NO : GNUNET_YES; + if (0 == (dc->options & GNUNET_FS_DOWNLOAD_IS_PROBE)) + dc->client_info = dc->h->upcb (dc->h->upcb_cls, pi); + else + dc->client_info = GNUNET_FS_search_probe_progress_ (NULL, pi); +} + + +/** + * We're ready to transmit a search request to the + * file-sharing service. Do it. If there is + * more than one request pending, try to send + * multiple or request another transmission. + * + * @param cls closure + * @param size number of bytes available in buf + * @param buf where the callee should write the message + * @return number of bytes written to buf + */ +static size_t +transmit_download_request (void *cls, size_t size, void *buf); + + +/** + * Closure for iterator processing results. + */ +struct ProcessResultClosure +{ + + /** + * Hash of data. + */ + GNUNET_HashCode query; + + /** + * Data found in P2P network. + */ + const void *data; + + /** + * Our download context. + */ + struct GNUNET_FS_DownloadContext *dc; + + /** + * Number of bytes in data. + */ + size_t size; + + /** + * Type of data. + */ + enum GNUNET_BLOCK_Type type; + + /** + * Flag to indicate if this block should be stored on disk. + */ + int do_store; + + struct GNUNET_TIME_Absolute last_transmission; + +}; + + +/** + * Iterator over entries in the pending requests in the 'active' map for the + * reply that we just got. + * + * @param cls closure (our 'struct ProcessResultClosure') + * @param key query for the given value / request + * @param value value in the hash map (a 'struct DownloadRequest') + * @return GNUNET_YES (we should continue to iterate); unless serious error + */ +static int +process_result_with_request (void *cls, const GNUNET_HashCode * key, + void *value); + + +/** + * We've found a matching block without downloading it. + * Encrypt it and pass it to our "receive" function as + * if we had received it from the network. + * + * @param dc download in question + * @param chk request this relates to + * @param dr request details + * @param block plaintext data matching request + * @param len number of bytes in block + * @param do_store should we still store the block on disk? + * @return GNUNET_OK on success + */ +static int +encrypt_existing_match (struct GNUNET_FS_DownloadContext *dc, + const struct ContentHashKey *chk, + struct DownloadRequest *dr, const char *block, + size_t len, int do_store) +{ + struct ProcessResultClosure prc; + char enc[len]; + struct GNUNET_CRYPTO_AesSessionKey sk; + struct GNUNET_CRYPTO_AesInitializationVector iv; + GNUNET_HashCode query; + + GNUNET_CRYPTO_hash_to_aes_key (&chk->key, &sk, &iv); + if (-1 == GNUNET_CRYPTO_aes_encrypt (block, len, &sk, &iv, enc)) + { + GNUNET_break (0); + return GNUNET_SYSERR; + } + GNUNET_CRYPTO_hash (enc, len, &query); + if (0 != memcmp (&query, &chk->query, sizeof (GNUNET_HashCode))) + { + GNUNET_break_op (0); + return GNUNET_SYSERR; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Matching block for `%s' at offset %llu already present, no need for download!\n", + dc->filename, (unsigned long long) dr->offset); + /* already got it! */ + prc.dc = dc; + prc.data = enc; + prc.size = len; + prc.type = + (0 == + dr->depth) ? GNUNET_BLOCK_TYPE_FS_DBLOCK : GNUNET_BLOCK_TYPE_FS_IBLOCK; + prc.query = chk->query; + prc.do_store = do_store; + prc.last_transmission = GNUNET_TIME_UNIT_FOREVER_ABS; + process_result_with_request (&prc, &chk->key, dr); + return GNUNET_OK; +} + + +/** + * We've lost our connection with the FS service. + * Re-establish it and re-transmit all of our + * pending requests. + * + * @param dc download context that is having trouble + */ +static void +try_reconnect (struct GNUNET_FS_DownloadContext *dc); + + +/** + * We found an entry in a directory. Check if the respective child + * already exists and if not create the respective child download. + * + * @param cls the parent download + * @param filename name of the file in the directory + * @param uri URI of the file (CHK or LOC) + * @param meta meta data of the file + * @param length number of bytes in data + * @param data contents of the file (or NULL if they were not inlined) + */ +static void +trigger_recursive_download (void *cls, const char *filename, + const struct GNUNET_FS_Uri *uri, + const struct GNUNET_CONTAINER_MetaData *meta, + size_t length, const void *data); + + +/** + * We're done downloading a directory. Open the file and + * trigger all of the (remaining) child downloads. + * + * @param dc context of download that just completed + */ +static void +full_recursive_download (struct GNUNET_FS_DownloadContext *dc) +{ + size_t size; + uint64_t size64; + void *data; + struct GNUNET_DISK_FileHandle *h; + struct GNUNET_DISK_MapHandle *m; + + size64 = GNUNET_FS_uri_chk_get_file_size (dc->uri); + size = (size_t) size64; + if (size64 != (uint64_t) size) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + _ + ("Recursive downloads of directories larger than 4 GB are not supported on 32-bit systems\n")); + return; + } + if (dc->filename != NULL) + { + h = GNUNET_DISK_file_open (dc->filename, GNUNET_DISK_OPEN_READ, + GNUNET_DISK_PERM_NONE); + } + else + { + GNUNET_assert (dc->temp_filename != NULL); + h = GNUNET_DISK_file_open (dc->temp_filename, GNUNET_DISK_OPEN_READ, + GNUNET_DISK_PERM_NONE); + } + if (h == NULL) + return; /* oops */ + data = GNUNET_DISK_file_map (h, &m, GNUNET_DISK_MAP_TYPE_READ, size); + if (data == NULL) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + _("Directory too large for system address space\n")); + } + else + { + GNUNET_FS_directory_list_contents (size, data, 0, + &trigger_recursive_download, dc); + GNUNET_DISK_file_unmap (m); + } + GNUNET_DISK_file_close (h); + if (dc->filename == NULL) + { + if (0 != UNLINK (dc->temp_filename)) + GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", + dc->temp_filename); + GNUNET_free (dc->temp_filename); + dc->temp_filename = NULL; + } +} + + +/** + * Check if all child-downloads have completed (or trigger them if + * necessary) and once we're completely done, signal completion (and + * possibly recurse to parent). This function MUST be called when the + * download of a file itself is done or when the download of a file is + * done and then later a direct child download has completed (and + * hence this download may complete itself). + * + * @param dc download to check for completion of children + */ +static void +check_completed (struct GNUNET_FS_DownloadContext *dc) +{ + struct GNUNET_FS_ProgressInfo pi; + struct GNUNET_FS_DownloadContext *pos; + + /* first, check if we need to download children */ + if ((dc->child_head == NULL) && (is_recursive_download (dc))) + full_recursive_download (dc); + /* then, check if children are done already */ + pos = dc->child_head; + while (pos != NULL) + { + if ((pos->emsg == NULL) && (pos->completed < pos->length)) + return; /* not done yet */ + if ((pos->child_head != NULL) && (pos->has_finished != GNUNET_YES)) + return; /* not transitively done yet */ + pos = pos->next; + } + /* All of our children are done, so mark this download done */ + dc->has_finished = GNUNET_YES; + if (dc->job_queue != NULL) + { + GNUNET_FS_dequeue_ (dc->job_queue); + dc->job_queue = NULL; + } + GNUNET_FS_download_sync_ (dc); + + /* signal completion */ + pi.status = GNUNET_FS_STATUS_DOWNLOAD_COMPLETED; + GNUNET_FS_download_make_status_ (&pi, dc); + + /* let parent know */ + if (dc->parent != NULL) + check_completed (dc->parent); +} + + +/** + * We got a block of plaintext data (from the meta data). + * Try it for upward reconstruction of the data. On success, + * the top-level block will move to state BRS_DOWNLOAD_UP. + * + * @param dc context for the download + * @param dr download request to match against + * @param data plaintext data, starting from the beginning of the file + * @param data_len number of bytes in data + */ +static void +try_match_block (struct GNUNET_FS_DownloadContext *dc, + struct DownloadRequest *dr, const char *data, size_t data_len) +{ + struct GNUNET_FS_ProgressInfo pi; + unsigned int i; + char enc[DBLOCK_SIZE]; + struct ContentHashKey chks[CHK_PER_INODE]; + struct ContentHashKey in_chk; + struct GNUNET_CRYPTO_AesSessionKey sk; + struct GNUNET_CRYPTO_AesInitializationVector iv; + size_t dlen; + struct DownloadRequest *drc; + struct GNUNET_DISK_FileHandle *fh; + int complete; + const char *fn; + const char *odata; + size_t odata_len; + + odata = data; + odata_len = data_len; + if (BRS_DOWNLOAD_UP == dr->state) + return; + if (dr->depth > 0) + { + complete = GNUNET_YES; + for (i = 0; i < dr->num_children; i++) + { + drc = dr->children[i]; + try_match_block (dc, drc, data, data_len); + if (drc->state != BRS_RECONSTRUCT_META_UP) + complete = GNUNET_NO; + else + chks[i] = drc->chk; + } + if (GNUNET_YES != complete) + return; + data = (const char *) chks; + dlen = dr->num_children * sizeof (struct ContentHashKey); + } + else + { + if (dr->offset > data_len) + return; /* oops */ + dlen = GNUNET_MIN (data_len - dr->offset, DBLOCK_SIZE); + } + GNUNET_CRYPTO_hash (&data[dr->offset], dlen, &in_chk.key); + GNUNET_CRYPTO_hash_to_aes_key (&in_chk.key, &sk, &iv); + if (-1 == GNUNET_CRYPTO_aes_encrypt (&data[dr->offset], dlen, &sk, &iv, enc)) + { + GNUNET_break (0); + return; + } + GNUNET_CRYPTO_hash (enc, dlen, &in_chk.query); + switch (dr->state) + { + case BRS_INIT: + dr->chk = in_chk; + dr->state = BRS_RECONSTRUCT_META_UP; + break; + case BRS_CHK_SET: + if (0 != memcmp (&in_chk, &dr->chk, sizeof (struct ContentHashKey))) + { + /* other peer provided bogus meta data */ + GNUNET_break_op (0); + break; + } + /* write block to disk */ + fn = dc->filename != NULL ? dc->filename : dc->temp_filename; + fh = GNUNET_DISK_file_open (fn, + GNUNET_DISK_OPEN_READWRITE | + GNUNET_DISK_OPEN_CREATE | + GNUNET_DISK_OPEN_TRUNCATE, + GNUNET_DISK_PERM_USER_READ | + GNUNET_DISK_PERM_USER_WRITE | + GNUNET_DISK_PERM_GROUP_READ | + GNUNET_DISK_PERM_OTHER_READ); + if (fh == NULL) + { + GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR, "open", fn); + GNUNET_asprintf (&dc->emsg, _("Failed to open file `%s' for writing"), + fn); + GNUNET_DISK_file_close (fh); + dr->state = BRS_ERROR; + pi.status = GNUNET_FS_STATUS_DOWNLOAD_ERROR; + pi.value.download.specifics.error.message = dc->emsg; + GNUNET_FS_download_make_status_ (&pi, dc); + return; + } + if (data_len != GNUNET_DISK_file_write (fh, odata, odata_len)) + { + GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR, "write", fn); + GNUNET_asprintf (&dc->emsg, _("Failed to open file `%s' for writing"), + fn); + GNUNET_DISK_file_close (fh); + dr->state = BRS_ERROR; + pi.status = GNUNET_FS_STATUS_DOWNLOAD_ERROR; + pi.value.download.specifics.error.message = dc->emsg; + GNUNET_FS_download_make_status_ (&pi, dc); + return; + } + GNUNET_DISK_file_close (fh); + /* signal success */ + dr->state = BRS_DOWNLOAD_UP; + dc->completed = dc->length; + GNUNET_FS_download_sync_ (dc); + pi.status = GNUNET_FS_STATUS_DOWNLOAD_PROGRESS; + pi.value.download.specifics.progress.data = data; + pi.value.download.specifics.progress.offset = 0; + pi.value.download.specifics.progress.data_len = dlen; + pi.value.download.specifics.progress.depth = 0; + pi.value.download.specifics.progress.trust_offered = 0; + GNUNET_FS_download_make_status_ (&pi, dc); + if ((NULL != dc->filename) && + (0 != + truncate (dc->filename, + GNUNET_ntohll (dc->uri->data.chk.file_length)))) + GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "truncate", + dc->filename); + check_completed (dc); + break; + default: + /* how did we get here? */ + GNUNET_break (0); + break; + } +} + + +/** + * Type of a function that libextractor calls for each + * meta data item found. If we find full data meta data, + * call 'try_match_block' on it. + * + * @param cls our 'struct GNUNET_FS_DownloadContext*' + * @param plugin_name name of the plugin that produced this value; + * special values can be used (i.e. '<zlib>' for zlib being + * used in the main libextractor library and yielding + * meta data). + * @param type libextractor-type describing the meta data + * @param format basic format information about data + * @param data_mime_type mime-type of data (not of the original file); + * can be NULL (if mime-type is not known) + * @param data actual meta-data found + * @param data_len number of bytes in data + * @return 0 to continue extracting, 1 to abort + */ +static int +match_full_data (void *cls, const char *plugin_name, + enum EXTRACTOR_MetaType type, enum EXTRACTOR_MetaFormat format, + const char *data_mime_type, const char *data, size_t data_len) +{ + struct GNUNET_FS_DownloadContext *dc = cls; + + if (type != EXTRACTOR_METATYPE_GNUNET_FULL_DATA) + return 0; + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Found %u bytes of FD!\n", + (unsigned int) data_len); + if (GNUNET_FS_uri_chk_get_file_size (dc->uri) != data_len) + { + GNUNET_break_op (0); + return 1; /* bogus meta data */ + } + try_match_block (dc, dc->top_request, data, data_len); + return 1; +} + + +/** + * Set the state of the given download request to + * BRS_DOWNLOAD_UP and propagate it up the tree. + * + * @param dr download request that is done + */ +static void +propagate_up (struct DownloadRequest *dr) +{ + unsigned int i; + + do + { + dr->state = BRS_DOWNLOAD_UP; + dr = dr->parent; + if (dr == NULL) + break; + for (i = 0; i < dr->num_children; i++) + if (dr->children[i]->state != BRS_DOWNLOAD_UP) + break; + } + while (i == dr->num_children); +} + + +/** + * Try top-down reconstruction. Before, the given request node + * must have the state BRS_CHK_SET. Afterwards, more nodes may + * have that state or advanced to BRS_DOWNLOAD_DOWN or even + * BRS_DOWNLOAD_UP. It is also possible to get BRS_ERROR on the + * top level. + * + * @param dc overall download this block belongs to + * @param dr block to reconstruct + */ +static void +try_top_down_reconstruction (struct GNUNET_FS_DownloadContext *dc, + struct DownloadRequest *dr) +{ + uint64_t off; + char block[DBLOCK_SIZE]; + GNUNET_HashCode key; + uint64_t total; + size_t len; + unsigned int i; + unsigned int chk_off; + struct DownloadRequest *drc; + uint64_t child_block_size; + const struct ContentHashKey *chks; + int up_done; + + GNUNET_assert (dc->rfh != NULL); + GNUNET_assert (dr->state == BRS_CHK_SET); + total = GNUNET_FS_uri_chk_get_file_size (dc->uri); + GNUNET_assert (dr->depth < dc->treedepth); + len = GNUNET_FS_tree_calculate_block_size (total, dr->offset, dr->depth); + GNUNET_assert (len <= DBLOCK_SIZE); + off = compute_disk_offset (total, dr->offset, dr->depth); + if (dc->old_file_size < off + len) + return; /* failure */ + if (off != GNUNET_DISK_file_seek (dc->rfh, off, GNUNET_DISK_SEEK_SET)) + { + GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "seek", dc->filename); + return; /* failure */ + } + if (len != GNUNET_DISK_file_read (dc->rfh, block, len)) + { + GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "read", dc->filename); + return; /* failure */ + } + GNUNET_CRYPTO_hash (block, len, &key); + if (0 != memcmp (&key, &dr->chk.key, sizeof (GNUNET_HashCode))) + return; /* mismatch */ + if (GNUNET_OK != + encrypt_existing_match (dc, &dr->chk, dr, block, len, GNUNET_NO)) + { + /* hash matches but encrypted block does not, really bad */ + dr->state = BRS_ERROR; + /* propagate up */ + while (dr->parent != NULL) + { + dr = dr->parent; + dr->state = BRS_ERROR; + } + return; + } + /* block matches */ + dr->state = BRS_DOWNLOAD_DOWN; + + /* set CHKs for children */ + up_done = GNUNET_YES; + chks = (const struct ContentHashKey *) block; + for (i = 0; i < dr->num_children; i++) + { + drc = dr->children[i]; + GNUNET_assert (drc->offset >= dr->offset); + child_block_size = GNUNET_FS_tree_compute_tree_size (drc->depth); + GNUNET_assert (0 == (drc->offset - dr->offset) % child_block_size); + chk_off = (drc->offset - dr->offset) / child_block_size; + if (drc->state == BRS_INIT) + { + drc->state = BRS_CHK_SET; + drc->chk = chks[chk_off]; + try_top_down_reconstruction (dc, drc); + } + if (drc->state != BRS_DOWNLOAD_UP) + up_done = GNUNET_NO; /* children not all done */ + } + if (up_done == GNUNET_YES) + propagate_up (dr); /* children all done (or no children...) */ +} + + +/** + * Schedule the download of the specified block in the tree. + * + * @param dc overall download this block belongs to + * @param dr request to schedule + */ +static void +schedule_block_download (struct GNUNET_FS_DownloadContext *dc, + struct DownloadRequest *dr) +{ + unsigned int i; + + switch (dr->state) + { + case BRS_INIT: + GNUNET_assert (0); + break; + case BRS_RECONSTRUCT_DOWN: + GNUNET_assert (0); + break; + case BRS_RECONSTRUCT_META_UP: + GNUNET_assert (0); + break; + case BRS_RECONSTRUCT_UP: + GNUNET_assert (0); + break; + case BRS_CHK_SET: + /* normal case, start download */ + break; + case BRS_DOWNLOAD_DOWN: + for (i = 0; i < dr->num_children; i++) + schedule_block_download (dc, dr->children[i]); + return; + case BRS_DOWNLOAD_UP: + /* We're done! */ + return; + case BRS_ERROR: + GNUNET_break (0); + return; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Scheduling download at offset %llu and depth %u for `%s'\n", + (unsigned long long) dr->offset, dr->depth, + GNUNET_h2s (&dr->chk.query)); + if (GNUNET_NO != + GNUNET_CONTAINER_multihashmap_contains_value (dc->active, &dr->chk.query, + dr)) + return; /* already active */ + GNUNET_CONTAINER_multihashmap_put (dc->active, &dr->chk.query, dr, + GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE); + if (dc->client == NULL) + return; /* download not active */ + GNUNET_CONTAINER_DLL_insert (dc->pending_head, dc->pending_tail, dr); + dr->is_pending = GNUNET_YES; + if (NULL == dc->th) + dc->th = + GNUNET_CLIENT_notify_transmit_ready (dc->client, + sizeof (struct SearchMessage), + GNUNET_CONSTANTS_SERVICE_TIMEOUT, + GNUNET_NO, + &transmit_download_request, dc); +} + + +#define GNUNET_FS_URI_CHK_PREFIX GNUNET_FS_URI_PREFIX GNUNET_FS_URI_CHK_INFIX + +/** + * We found an entry in a directory. Check if the respective child + * already exists and if not create the respective child download. + * + * @param cls the parent download + * @param filename name of the file in the directory + * @param uri URI of the file (CHK or LOC) + * @param meta meta data of the file + * @param length number of bytes in data + * @param data contents of the file (or NULL if they were not inlined) + */ +static void +trigger_recursive_download (void *cls, const char *filename, + const struct GNUNET_FS_Uri *uri, + const struct GNUNET_CONTAINER_MetaData *meta, + size_t length, const void *data) +{ + struct GNUNET_FS_DownloadContext *dc = cls; + struct GNUNET_FS_DownloadContext *cpos; + char *temp_name; + char *fn; + char *us; + char *ext; + char *dn; + char *pos; + char *full_name; + char *sfn; + + if (NULL == uri) + return; /* entry for the directory itself */ + cpos = dc->child_head; + while (cpos != NULL) + { + if ((GNUNET_FS_uri_test_equal (uri, cpos->uri)) || + ((filename != NULL) && (0 == strcmp (cpos->filename, filename)))) + break; + cpos = cpos->next; + } + if (cpos != NULL) + return; /* already exists */ + fn = NULL; + if (NULL == filename) + { + fn = GNUNET_FS_meta_data_suggest_filename (meta); + if (fn == NULL) + { + us = GNUNET_FS_uri_to_string (uri); + fn = GNUNET_strdup (&us[strlen (GNUNET_FS_URI_CHK_PREFIX)]); + GNUNET_free (us); + } + else if (fn[0] == '.') + { + ext = fn; + us = GNUNET_FS_uri_to_string (uri); + GNUNET_asprintf (&fn, "%s%s", &us[strlen (GNUNET_FS_URI_CHK_PREFIX)], + ext); + GNUNET_free (ext); + GNUNET_free (us); + } + /* change '\' to '/' (this should have happened + * during insertion, but malicious peers may + * not have done this) */ + while (NULL != (pos = strstr (fn, "\\"))) + *pos = '/'; + /* remove '../' everywhere (again, well-behaved + * peers don't do this, but don't trust that + * we did not get something nasty) */ + while (NULL != (pos = strstr (fn, "../"))) + { + pos[0] = '_'; + pos[1] = '_'; + pos[2] = '_'; + } + filename = fn; + } + if (dc->filename == NULL) + { + full_name = NULL; + } + else + { + dn = GNUNET_strdup (dc->filename); + GNUNET_break ((strlen (dn) >= strlen (GNUNET_FS_DIRECTORY_EXT)) && + (NULL != + strstr (dn + strlen (dn) - strlen (GNUNET_FS_DIRECTORY_EXT), + GNUNET_FS_DIRECTORY_EXT))); + sfn = GNUNET_strdup (filename); + while ((strlen (sfn) > 0) && (filename[strlen (sfn) - 1] == '/')) + sfn[strlen (sfn) - 1] = '\0'; + if ((strlen (dn) >= strlen (GNUNET_FS_DIRECTORY_EXT)) && + (NULL != + strstr (dn + strlen (dn) - strlen (GNUNET_FS_DIRECTORY_EXT), + GNUNET_FS_DIRECTORY_EXT))) + dn[strlen (dn) - strlen (GNUNET_FS_DIRECTORY_EXT)] = '\0'; + if ((GNUNET_YES == GNUNET_FS_meta_data_test_for_directory (meta)) && + ((strlen (filename) < strlen (GNUNET_FS_DIRECTORY_EXT)) || + (NULL == + strstr (filename + strlen (filename) - + strlen (GNUNET_FS_DIRECTORY_EXT), GNUNET_FS_DIRECTORY_EXT)))) + { + GNUNET_asprintf (&full_name, "%s%s%s%s", dn, DIR_SEPARATOR_STR, sfn, + GNUNET_FS_DIRECTORY_EXT); + } + else + { + GNUNET_asprintf (&full_name, "%s%s%s", dn, DIR_SEPARATOR_STR, sfn); + } + GNUNET_free (sfn); + GNUNET_free (dn); + } + if ((full_name != NULL) && + (GNUNET_OK != GNUNET_DISK_directory_create_for_file (full_name))) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + _ + ("Failed to create directory for recursive download of `%s'\n"), + full_name); + GNUNET_free (full_name); + GNUNET_free_non_null (fn); + return; + } + + temp_name = NULL; + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Triggering recursive download of size %llu with %u bytes MD\n", + (unsigned long long) GNUNET_FS_uri_chk_get_file_size (uri), + (unsigned int) + GNUNET_CONTAINER_meta_data_get_serialized_size (meta)); + GNUNET_FS_download_start (dc->h, uri, meta, full_name, temp_name, 0, + GNUNET_FS_uri_chk_get_file_size (uri), + dc->anonymity, dc->options, NULL, dc); + GNUNET_free_non_null (full_name); + GNUNET_free_non_null (temp_name); + GNUNET_free_non_null (fn); +} + + +/** + * (recursively) free download request structure + * + * @param dr request to free + */ +void +GNUNET_FS_free_download_request_ (struct DownloadRequest *dr) +{ + unsigned int i; + + if (dr == NULL) + return; + for (i = 0; i < dr->num_children; i++) + GNUNET_FS_free_download_request_ (dr->children[i]); + GNUNET_free_non_null (dr->children); + GNUNET_free (dr); +} + + +/** + * Iterator over entries in the pending requests in the 'active' map for the + * reply that we just got. + * + * @param cls closure (our 'struct ProcessResultClosure') + * @param key query for the given value / request + * @param value value in the hash map (a 'struct DownloadRequest') + * @return GNUNET_YES (we should continue to iterate); unless serious error + */ +static int +process_result_with_request (void *cls, const GNUNET_HashCode * key, + void *value) +{ + struct ProcessResultClosure *prc = cls; + struct DownloadRequest *dr = value; + struct GNUNET_FS_DownloadContext *dc = prc->dc; + struct DownloadRequest *drc; + struct GNUNET_DISK_FileHandle *fh = NULL; + struct GNUNET_CRYPTO_AesSessionKey skey; + struct GNUNET_CRYPTO_AesInitializationVector iv; + char pt[prc->size]; + struct GNUNET_FS_ProgressInfo pi; + uint64_t off; + size_t bs; + size_t app; + int i; + struct ContentHashKey *chkarr; + + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Received block `%s' matching pending request at depth %u and offset %llu/%llu\n", + GNUNET_h2s (key), dr->depth, (unsigned long long) dr->offset, + (unsigned long long) GNUNET_ntohll (dc->uri->data. + chk.file_length)); + bs = GNUNET_FS_tree_calculate_block_size (GNUNET_ntohll + (dc->uri->data.chk.file_length), + dr->offset, dr->depth); + if (prc->size != bs) + { + GNUNET_asprintf (&dc->emsg, + _ + ("Internal error or bogus download URI (expected %u bytes at depth %u and offset %llu/%llu, got %u bytes)\n"), + bs, dr->depth, (unsigned long long) dr->offset, + (unsigned long long) GNUNET_ntohll (dc->uri->data. + chk.file_length), + prc->size); + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "%s", dc->emsg); + while (dr->parent != NULL) + { + dr->state = BRS_ERROR; + dr = dr->parent; + } + dr->state = BRS_ERROR; + goto signal_error; + } + + (void) GNUNET_CONTAINER_multihashmap_remove (dc->active, &prc->query, dr); + if (GNUNET_YES == dr->is_pending) + { + GNUNET_CONTAINER_DLL_remove (dc->pending_head, dc->pending_tail, dr); + dr->is_pending = GNUNET_NO; + } + + GNUNET_CRYPTO_hash_to_aes_key (&dr->chk.key, &skey, &iv); + if (-1 == GNUNET_CRYPTO_aes_decrypt (prc->data, prc->size, &skey, &iv, pt)) + { + GNUNET_break (0); + dc->emsg = GNUNET_strdup (_("internal error decrypting content")); + goto signal_error; + } + off = + compute_disk_offset (GNUNET_ntohll (dc->uri->data.chk.file_length), + dr->offset, dr->depth); + /* save to disk */ + if ((GNUNET_YES == prc->do_store) && + ((dc->filename != NULL) || (is_recursive_download (dc))) && + ((dr->depth == dc->treedepth) || + (0 == (dc->options & GNUNET_FS_DOWNLOAD_NO_TEMPORARIES)))) + { + fh = GNUNET_DISK_file_open (dc->filename != + NULL ? dc->filename : dc->temp_filename, + GNUNET_DISK_OPEN_READWRITE | + GNUNET_DISK_OPEN_CREATE, + GNUNET_DISK_PERM_USER_READ | + GNUNET_DISK_PERM_USER_WRITE | + GNUNET_DISK_PERM_GROUP_READ | + GNUNET_DISK_PERM_OTHER_READ); + if (NULL == fh) + { + GNUNET_asprintf (&dc->emsg, + _("Download failed: could not open file `%s': %s\n"), + dc->filename, STRERROR (errno)); + goto signal_error; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Saving decrypted block to disk at offset %llu\n", + (unsigned long long) off); + if ((off != GNUNET_DISK_file_seek (fh, off, GNUNET_DISK_SEEK_SET))) + { + GNUNET_asprintf (&dc->emsg, + _("Failed to seek to offset %llu in file `%s': %s\n"), + (unsigned long long) off, dc->filename, + STRERROR (errno)); + goto signal_error; + } + if (prc->size != GNUNET_DISK_file_write (fh, pt, prc->size)) + { + GNUNET_asprintf (&dc->emsg, + _ + ("Failed to write block of %u bytes at offset %llu in file `%s': %s\n"), + (unsigned int) prc->size, (unsigned long long) off, + dc->filename, STRERROR (errno)); + goto signal_error; + } + GNUNET_break (GNUNET_OK == GNUNET_DISK_file_close (fh)); + fh = NULL; + } + + if (dr->depth == 0) + { + /* DBLOCK, update progress and try recursion if applicable */ + app = prc->size; + if (dr->offset < dc->offset) + { + /* starting offset begins in the middle of pt, + * do not count first bytes as progress */ + GNUNET_assert (app > (dc->offset - dr->offset)); + app -= (dc->offset - dr->offset); + } + if (dr->offset + prc->size > dc->offset + dc->length) + { + /* end of block is after relevant range, + * do not count last bytes as progress */ + GNUNET_assert (app > + (dr->offset + prc->size) - (dc->offset + dc->length)); + app -= (dr->offset + prc->size) - (dc->offset + dc->length); + } + dc->completed += app; + + /* do recursive download if option is set and either meta data + * says it is a directory or if no meta data is given AND filename + * ends in '.gnd' (top-level case) */ + if (is_recursive_download (dc)) + GNUNET_FS_directory_list_contents (prc->size, pt, off, + &trigger_recursive_download, dc); + + } + GNUNET_assert (dc->completed <= dc->length); + dr->state = BRS_DOWNLOAD_DOWN; + pi.status = GNUNET_FS_STATUS_DOWNLOAD_PROGRESS; + pi.value.download.specifics.progress.data = pt; + pi.value.download.specifics.progress.offset = dr->offset; + pi.value.download.specifics.progress.data_len = prc->size; + pi.value.download.specifics.progress.depth = dr->depth; + pi.value.download.specifics.progress.trust_offered = 0; + if (prc->last_transmission.abs_value != GNUNET_TIME_UNIT_FOREVER_ABS.abs_value) + pi.value.download.specifics.progress.block_download_duration = + GNUNET_TIME_absolute_get_duration (prc->last_transmission); + else + pi.value.download.specifics.progress.block_download_duration.rel_value = + GNUNET_TIME_UNIT_FOREVER_REL.rel_value; + GNUNET_FS_download_make_status_ (&pi, dc); + if (dr->depth == 0) + propagate_up (dr); + + if (dc->completed == dc->length) + { + /* download completed, signal */ + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Download completed, truncating file to desired length %llu\n", + (unsigned long long) GNUNET_ntohll (dc->uri->data. + chk.file_length)); + /* truncate file to size (since we store IBlocks at the end) */ + if (dc->filename != NULL) + { + if (0 != + truncate (dc->filename, + GNUNET_ntohll (dc->uri->data.chk.file_length))) + GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "truncate", + dc->filename); + } + GNUNET_assert (dr->depth == 0); + check_completed (dc); + } + if (dr->depth == 0) + { + /* bottom of the tree, no child downloads possible, just sync */ + GNUNET_FS_download_sync_ (dc); + return GNUNET_YES; + } + + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Triggering downloads of children (this block was at depth %u and offset %llu)\n", + dr->depth, (unsigned long long) dr->offset); + GNUNET_assert (0 == (prc->size % sizeof (struct ContentHashKey))); + chkarr = (struct ContentHashKey *) pt; + for (i = (prc->size / sizeof (struct ContentHashKey)) - 1; i >= 0; i--) + { + drc = dr->children[i]; + switch (drc->state) + { + case BRS_INIT: + drc->chk = chkarr[i]; + drc->state = BRS_CHK_SET; + schedule_block_download (dc, drc); + break; + case BRS_RECONSTRUCT_DOWN: + GNUNET_assert (0); + break; + case BRS_RECONSTRUCT_META_UP: + GNUNET_assert (0); + break; + case BRS_RECONSTRUCT_UP: + GNUNET_assert (0); + break; + case BRS_CHK_SET: + GNUNET_assert (0); + break; + case BRS_DOWNLOAD_DOWN: + GNUNET_assert (0); + break; + case BRS_DOWNLOAD_UP: + GNUNET_assert (0); + break; + case BRS_ERROR: + GNUNET_assert (0); + break; + default: + GNUNET_assert (0); + break; + } + } + GNUNET_FS_download_sync_ (dc); + return GNUNET_YES; + +signal_error: + if (fh != NULL) + GNUNET_DISK_file_close (fh); + pi.status = GNUNET_FS_STATUS_DOWNLOAD_ERROR; + pi.value.download.specifics.error.message = dc->emsg; + GNUNET_FS_download_make_status_ (&pi, dc); + /* abort all pending requests */ + if (NULL != dc->th) + { + GNUNET_CLIENT_notify_transmit_ready_cancel (dc->th); + dc->th = NULL; + } + GNUNET_CLIENT_disconnect (dc->client, GNUNET_NO); + dc->in_receive = GNUNET_NO; + dc->client = NULL; + GNUNET_FS_free_download_request_ (dc->top_request); + dc->top_request = NULL; + GNUNET_CONTAINER_multihashmap_destroy (dc->active); + dc->active = NULL; + dc->pending_head = NULL; + dc->pending_tail = NULL; + GNUNET_FS_download_sync_ (dc); + return GNUNET_NO; +} + + +/** + * Process a download result. + * + * @param dc our download context + * @param type type of the result + * @param last_transmission when was this block requested the last time? (FOREVER if unknown/not applicable) + * @param data the (encrypted) response + * @param size size of data + */ +static void +process_result (struct GNUNET_FS_DownloadContext *dc, + enum GNUNET_BLOCK_Type type, + struct GNUNET_TIME_Absolute last_transmission, + const void *data, size_t size) +{ + struct ProcessResultClosure prc; + + prc.dc = dc; + prc.data = data; + prc.size = size; + prc.type = type; + prc.do_store = GNUNET_YES; + prc.last_transmission = last_transmission; + GNUNET_CRYPTO_hash (data, size, &prc.query); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Received result for query `%s' from `%s'-service\n", + GNUNET_h2s (&prc.query), "FS"); + GNUNET_CONTAINER_multihashmap_get_multiple (dc->active, &prc.query, + &process_result_with_request, + &prc); +} + + +/** + * Type of a function to call when we receive a message + * from the service. + * + * @param cls closure + * @param msg message received, NULL on timeout or fatal error + */ +static void +receive_results (void *cls, const struct GNUNET_MessageHeader *msg) +{ + struct GNUNET_FS_DownloadContext *dc = cls; + const struct ClientPutMessage *cm; + uint16_t msize; + + if ((NULL == msg) || (ntohs (msg->type) != GNUNET_MESSAGE_TYPE_FS_PUT) || + (sizeof (struct ClientPutMessage) > ntohs (msg->size))) + { + GNUNET_break (msg == NULL); + try_reconnect (dc); + return; + } + msize = ntohs (msg->size); + cm = (const struct ClientPutMessage *) msg; + process_result (dc, ntohl (cm->type), + GNUNET_TIME_absolute_ntoh (cm->last_transmission), &cm[1], + msize - sizeof (struct ClientPutMessage)); + if (dc->client == NULL) + return; /* fatal error */ + /* continue receiving */ + GNUNET_CLIENT_receive (dc->client, &receive_results, dc, + GNUNET_TIME_UNIT_FOREVER_REL); +} + + + +/** + * We're ready to transmit a search request to the + * file-sharing service. Do it. If there is + * more than one request pending, try to send + * multiple or request another transmission. + * + * @param cls closure + * @param size number of bytes available in buf + * @param buf where the callee should write the message + * @return number of bytes written to buf + */ +static size_t +transmit_download_request (void *cls, size_t size, void *buf) +{ + struct GNUNET_FS_DownloadContext *dc = cls; + size_t msize; + struct SearchMessage *sm; + struct DownloadRequest *dr; + + dc->th = NULL; + if (NULL == buf) + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Transmitting download request failed, trying to reconnect\n"); + try_reconnect (dc); + return 0; + } + GNUNET_assert (size >= sizeof (struct SearchMessage)); + msize = 0; + sm = buf; + while ((NULL != (dr = dc->pending_head)) && + (size >= msize + sizeof (struct SearchMessage))) + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Transmitting download request for `%s' to `%s'-service\n", + GNUNET_h2s (&dr->chk.query), "FS"); + memset (sm, 0, sizeof (struct SearchMessage)); + sm->header.size = htons (sizeof (struct SearchMessage)); + sm->header.type = htons (GNUNET_MESSAGE_TYPE_FS_START_SEARCH); + if (0 != (dc->options & GNUNET_FS_DOWNLOAD_OPTION_LOOPBACK_ONLY)) + sm->options = htonl (GNUNET_FS_SEARCH_OPTION_LOOPBACK_ONLY); + else + sm->options = htonl (GNUNET_FS_SEARCH_OPTION_NONE); + if (dr->depth == 0) + sm->type = htonl (GNUNET_BLOCK_TYPE_FS_DBLOCK); + else + sm->type = htonl (GNUNET_BLOCK_TYPE_FS_IBLOCK); + sm->anonymity_level = htonl (dc->anonymity); + sm->target = dc->target.hashPubKey; + sm->query = dr->chk.query; + GNUNET_CONTAINER_DLL_remove (dc->pending_head, dc->pending_tail, dr); + dr->is_pending = GNUNET_NO; + msize += sizeof (struct SearchMessage); + sm++; + } + if (dc->pending_head != NULL) + { + dc->th = + GNUNET_CLIENT_notify_transmit_ready (dc->client, + sizeof (struct SearchMessage), + GNUNET_CONSTANTS_SERVICE_TIMEOUT, + GNUNET_NO, + &transmit_download_request, dc); + GNUNET_assert (dc->th != NULL); + } + if (GNUNET_NO == dc->in_receive) + { + dc->in_receive = GNUNET_YES; + GNUNET_CLIENT_receive (dc->client, &receive_results, dc, + GNUNET_TIME_UNIT_FOREVER_REL); + } + return msize; +} + + +/** + * Reconnect to the FS service and transmit our queries NOW. + * + * @param cls our download context + * @param tc unused + */ +static void +do_reconnect (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct GNUNET_FS_DownloadContext *dc = cls; + struct GNUNET_CLIENT_Connection *client; + + dc->task = GNUNET_SCHEDULER_NO_TASK; + client = GNUNET_CLIENT_connect ("fs", dc->h->cfg); + if (NULL == client) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + "Connecting to `%s'-service failed, will try again.\n", "FS"); + try_reconnect (dc); + return; + } + dc->client = client; + if (dc->pending_head != NULL) + { + dc->th = + GNUNET_CLIENT_notify_transmit_ready (client, + sizeof (struct SearchMessage), + GNUNET_CONSTANTS_SERVICE_TIMEOUT, + GNUNET_NO, + &transmit_download_request, dc); + GNUNET_assert (dc->th != NULL); + } +} + + +/** + * Add entries to the pending list. + * + * @param cls our download context + * @param key unused + * @param entry entry of type "struct DownloadRequest" + * @return GNUNET_OK + */ +static int +retry_entry (void *cls, const GNUNET_HashCode * key, void *entry) +{ + struct GNUNET_FS_DownloadContext *dc = cls; + struct DownloadRequest *dr = entry; + + dr->next = NULL; + dr->prev = NULL; + GNUNET_CONTAINER_DLL_insert (dc->pending_head, dc->pending_tail, dr); + dr->is_pending = GNUNET_YES; + return GNUNET_OK; +} + + +/** + * We've lost our connection with the FS service. + * Re-establish it and re-transmit all of our + * pending requests. + * + * @param dc download context that is having trouble + */ +static void +try_reconnect (struct GNUNET_FS_DownloadContext *dc) +{ + + if (NULL != dc->client) + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Moving all requests back to pending list\n"); + if (NULL != dc->th) + { + GNUNET_CLIENT_notify_transmit_ready_cancel (dc->th); + dc->th = NULL; + } + /* full reset of the pending list */ + dc->pending_head = NULL; + dc->pending_tail = NULL; + GNUNET_CONTAINER_multihashmap_iterate (dc->active, &retry_entry, dc); + GNUNET_CLIENT_disconnect (dc->client, GNUNET_NO); + dc->in_receive = GNUNET_NO; + dc->client = NULL; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Will try to reconnect in 1s\n"); + dc->task = + GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_SECONDS, &do_reconnect, + dc); +} + + +/** + * We're allowed to ask the FS service for our blocks. Start the download. + * + * @param cls the 'struct GNUNET_FS_DownloadContext' + * @param client handle to use for communcation with FS (we must destroy it!) + */ +static void +activate_fs_download (void *cls, struct GNUNET_CLIENT_Connection *client) +{ + struct GNUNET_FS_DownloadContext *dc = cls; + struct GNUNET_FS_ProgressInfo pi; + + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Download activated\n"); + GNUNET_assert (NULL != client); + GNUNET_assert (dc->client == NULL); + GNUNET_assert (dc->th == NULL); + dc->client = client; + pi.status = GNUNET_FS_STATUS_DOWNLOAD_ACTIVE; + GNUNET_FS_download_make_status_ (&pi, dc); + dc->pending_head = NULL; + dc->pending_tail = NULL; + GNUNET_CONTAINER_multihashmap_iterate (dc->active, &retry_entry, dc); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Asking for transmission to FS service\n"); + if (dc->pending_head != NULL) + { + dc->th = + GNUNET_CLIENT_notify_transmit_ready (dc->client, + sizeof (struct SearchMessage), + GNUNET_CONSTANTS_SERVICE_TIMEOUT, + GNUNET_NO, + &transmit_download_request, dc); + GNUNET_assert (dc->th != NULL); + } +} + + +/** + * We must stop to ask the FS service for our blocks. Pause the download. + * + * @param cls the 'struct GNUNET_FS_DownloadContext' + */ +static void +deactivate_fs_download (void *cls) +{ + struct GNUNET_FS_DownloadContext *dc = cls; + struct GNUNET_FS_ProgressInfo pi; + + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Download deactivated\n"); + if (NULL != dc->th) + { + GNUNET_CLIENT_notify_transmit_ready_cancel (dc->th); + dc->th = NULL; + } + if (NULL != dc->client) + { + GNUNET_CLIENT_disconnect (dc->client, GNUNET_NO); + dc->in_receive = GNUNET_NO; + dc->client = NULL; + } + dc->pending_head = NULL; + dc->pending_tail = NULL; + pi.status = GNUNET_FS_STATUS_DOWNLOAD_INACTIVE; + GNUNET_FS_download_make_status_ (&pi, dc); +} + + +/** + * (recursively) Create a download request structure. + * + * @param parent parent of the current entry + * @param depth depth of the current entry, 0 are the DBLOCKs, + * top level block is 'dc->treedepth - 1' + * @param dr_offset offset in the original file this block maps to + * (as in, offset of the first byte of the first DBLOCK + * in the subtree rooted in the returned download request tree) + * @param file_start_offset desired starting offset for the download + * in the original file; requesting tree should not contain + * DBLOCKs prior to the file_start_offset + * @param desired_length desired number of bytes the user wanted to access + * (from file_start_offset). Resulting tree should not contain + * DBLOCKs after file_start_offset + file_length. + * @return download request tree for the given range of DBLOCKs at + * the specified depth + */ +static struct DownloadRequest * +create_download_request (struct DownloadRequest *parent, unsigned int depth, + uint64_t dr_offset, uint64_t file_start_offset, + uint64_t desired_length) +{ + struct DownloadRequest *dr; + unsigned int i; + unsigned int head_skip; + uint64_t child_block_size; + + dr = GNUNET_malloc (sizeof (struct DownloadRequest)); + dr->parent = parent; + dr->depth = depth; + dr->offset = dr_offset; + if (depth > 0) + { + child_block_size = GNUNET_FS_tree_compute_tree_size (depth - 1); + + /* calculate how many blocks at this level are not interesting + * from the start (rounded down), either because of the requested + * file offset or because this IBlock is further along */ + if (dr_offset < file_start_offset) + head_skip = file_start_offset / child_block_size; + else + head_skip = dr_offset / child_block_size; + + /* calculate index of last block at this level that is interesting (rounded up) */ + dr->num_children = file_start_offset + desired_length / child_block_size; + if (dr->num_children * child_block_size < + file_start_offset + desired_length) + dr->num_children++; /* round up */ + + /* now we can get the total number of children for this block */ + dr->num_children -= head_skip; + if (dr->num_children > CHK_PER_INODE) + dr->num_children = CHK_PER_INODE; /* cap at max */ + + /* why else would we have gotten here to begin with? (that'd be a bad logic error) */ + GNUNET_assert (dr->num_children > 0); + + dr->children = + GNUNET_malloc (dr->num_children * sizeof (struct DownloadRequest *)); + for (i = 0; i < dr->num_children; i++) + dr->children[i] = + create_download_request (dr, depth - 1, + dr_offset + i * child_block_size, + file_start_offset, desired_length); + } + return dr; +} + + +/** + * Continuation after a possible attempt to reconstruct + * the current IBlock from the existing file. + * + * @param cls the 'struct ReconstructContext' + * @param tc scheduler context + */ +static void +reconstruct_cont (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct GNUNET_FS_DownloadContext *dc = cls; + + /* clean up state from tree encoder */ + if (dc->te != NULL) + { + GNUNET_FS_tree_encoder_finish (dc->te, NULL, NULL); + dc->te = NULL; + } + if (dc->task != GNUNET_SCHEDULER_NO_TASK) + { + GNUNET_SCHEDULER_cancel (dc->task); + dc->task = GNUNET_SCHEDULER_NO_TASK; + } + if (dc->rfh != NULL) + { + GNUNET_break (GNUNET_OK == GNUNET_DISK_file_close (dc->rfh)); + dc->rfh = NULL; + } + /* start "normal" download */ + schedule_block_download (dc, dc->top_request); +} + + +/** + * Task requesting the next block from the tree encoder. + * + * @param cls the 'struct GNUJNET_FS_DownloadContext' we're processing + * @param tc task context + */ +static void +get_next_block (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct GNUNET_FS_DownloadContext *dc = cls; + + dc->task = GNUNET_SCHEDULER_NO_TASK; + GNUNET_FS_tree_encoder_next (dc->te); +} + + + +/** + * Function called asking for the current (encoded) + * block to be processed. After processing the + * client should either call "GNUNET_FS_tree_encode_next" + * or (on error) "GNUNET_FS_tree_encode_finish". + * + * This function checks if the content on disk matches + * the expected content based on the URI. + * + * @param cls closure + * @param chk content hash key for the block + * @param offset offset of the block + * @param depth depth of the block, 0 for DBLOCK + * @param type type of the block (IBLOCK or DBLOCK) + * @param block the (encrypted) block + * @param block_size size of block (in bytes) + */ +static void +reconstruct_cb (void *cls, const struct ContentHashKey *chk, uint64_t offset, + unsigned int depth, enum GNUNET_BLOCK_Type type, + const void *block, uint16_t block_size) +{ + struct GNUNET_FS_DownloadContext *dc = cls; + struct GNUNET_FS_ProgressInfo pi; + struct DownloadRequest *dr; + uint64_t blen; + unsigned int chld; + + /* find corresponding request entry */ + dr = dc->top_request; + while (dr->depth > depth) + { + blen = GNUNET_FS_tree_compute_tree_size (dr->depth); + chld = (offset - dr->offset) / blen; + GNUNET_assert (chld < dr->num_children); + dr = dr->children[chld]; + } + /* FIXME: this code needs more testing and might + need to handle more states... */ + switch (dr->state) + { + case BRS_INIT: + break; + case BRS_RECONSTRUCT_DOWN: + break; + case BRS_RECONSTRUCT_META_UP: + break; + case BRS_RECONSTRUCT_UP: + break; + case BRS_CHK_SET: + if (0 == memcmp (chk, &dr->chk, sizeof (struct ContentHashKey))) + { + /* block matches, hence tree below matches; + * this request is done! */ + dr->state = BRS_DOWNLOAD_UP; + GNUNET_break (GNUNET_NO == + GNUNET_CONTAINER_multihashmap_remove (dc->active, &dr->chk.query, dr)); + if (GNUNET_YES == dr->is_pending) + { + GNUNET_break (0); /* how did we get here? */ + GNUNET_CONTAINER_DLL_remove (dc->pending_head, dc->pending_tail, dr); + dr->is_pending = GNUNET_NO; + } + /* calculate how many bytes of payload this block + * corresponds to */ + blen = GNUNET_FS_tree_compute_tree_size (dr->depth); + /* how many of those bytes are in the requested range? */ + blen = GNUNET_MIN (blen, dc->length + dc->offset - dr->offset); + /* signal progress */ + dc->completed += blen; + pi.status = GNUNET_FS_STATUS_DOWNLOAD_PROGRESS; + pi.value.download.specifics.progress.data = NULL; + pi.value.download.specifics.progress.offset = offset; + pi.value.download.specifics.progress.data_len = 0; + pi.value.download.specifics.progress.depth = 0; + pi.value.download.specifics.progress.trust_offered = 0; + GNUNET_FS_download_make_status_ (&pi, dc); + /* FIXME: duplicated code from 'process_result_with_request - refactor */ + if (dc->completed == dc->length) + { + /* download completed, signal */ + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Download completed, truncating file to desired length %llu\n", + (unsigned long long) GNUNET_ntohll (dc->uri->data. + chk.file_length)); + /* truncate file to size (since we store IBlocks at the end) */ + if (dc->filename != NULL) + { + if (0 != + truncate (dc->filename, + GNUNET_ntohll (dc->uri->data.chk.file_length))) + GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "truncate", + dc->filename); + } + } + } + break; + case BRS_DOWNLOAD_DOWN: + break; + case BRS_DOWNLOAD_UP: + break; + case BRS_ERROR: + break; + default: + GNUNET_assert (0); + break; + } + if ((dr == dc->top_request) && (dr->state == BRS_DOWNLOAD_UP)) + { + check_completed (dc); + return; + } + dc->task = GNUNET_SCHEDULER_add_now (&get_next_block, dc); +} + + +/** + * Function called by the tree encoder to obtain a block of plaintext + * data (for the lowest level of the tree). + * + * @param cls our 'struct ReconstructContext' + * @param offset identifies which block to get + * @param max (maximum) number of bytes to get; returning + * fewer will also cause errors + * @param buf where to copy the plaintext buffer + * @param emsg location to store an error message (on error) + * @return number of bytes copied to buf, 0 on error + */ +static size_t +fh_reader (void *cls, uint64_t offset, size_t max, void *buf, char **emsg) +{ + struct GNUNET_FS_DownloadContext *dc = cls; + struct GNUNET_DISK_FileHandle *fh = dc->rfh; + ssize_t ret; + + *emsg = NULL; + if (offset != GNUNET_DISK_file_seek (fh, offset, GNUNET_DISK_SEEK_SET)) + { + *emsg = GNUNET_strdup (strerror (errno)); + return 0; + } + ret = GNUNET_DISK_file_read (fh, buf, max); + if (ret < 0) + { + *emsg = GNUNET_strdup (strerror (errno)); + return 0; + } + return ret; +} + + +/** + * Task that creates the initial (top-level) download + * request for the file. + * + * @param cls the 'struct GNUNET_FS_DownloadContext' + * @param tc scheduler context + */ +void +GNUNET_FS_download_start_task_ (void *cls, + const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct GNUNET_FS_DownloadContext *dc = cls; + struct GNUNET_FS_ProgressInfo pi; + struct GNUNET_DISK_FileHandle *fh; + + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Start task running...\n"); + dc->task = GNUNET_SCHEDULER_NO_TASK; + if (dc->length == 0) + { + /* no bytes required! */ + if (dc->filename != NULL) + { + fh = GNUNET_DISK_file_open (dc->filename, + GNUNET_DISK_OPEN_READWRITE | + GNUNET_DISK_OPEN_CREATE | + ((0 == + GNUNET_FS_uri_chk_get_file_size (dc->uri)) ? + GNUNET_DISK_OPEN_TRUNCATE : 0), + GNUNET_DISK_PERM_USER_READ | + GNUNET_DISK_PERM_USER_WRITE | + GNUNET_DISK_PERM_GROUP_READ | + GNUNET_DISK_PERM_OTHER_READ); + GNUNET_DISK_file_close (fh); + } + GNUNET_FS_download_sync_ (dc); + pi.status = GNUNET_FS_STATUS_DOWNLOAD_START; + pi.value.download.specifics.start.meta = dc->meta; + GNUNET_FS_download_make_status_ (&pi, dc); + check_completed (dc); + return; + } + if (dc->emsg != NULL) + return; + if (dc->top_request == NULL) + { + dc->top_request = + create_download_request (NULL, dc->treedepth - 1, 0, dc->offset, + dc->length); + dc->top_request->state = BRS_CHK_SET; + dc->top_request->chk = + (dc->uri->type == + chk) ? dc->uri->data.chk.chk : dc->uri->data.loc.fi.chk; + /* signal start */ + GNUNET_FS_download_sync_ (dc); + if (NULL != dc->search) + GNUNET_FS_search_result_sync_ (dc->search); + pi.status = GNUNET_FS_STATUS_DOWNLOAD_START; + pi.value.download.specifics.start.meta = dc->meta; + GNUNET_FS_download_make_status_ (&pi, dc); + } + GNUNET_FS_download_start_downloading_ (dc); + /* attempt reconstruction from disk */ + if (GNUNET_YES == GNUNET_DISK_file_test (dc->filename)) + dc->rfh = + GNUNET_DISK_file_open (dc->filename, GNUNET_DISK_OPEN_READ, + GNUNET_DISK_PERM_NONE); + if (dc->top_request->state == BRS_CHK_SET) + { + if (dc->rfh != NULL) + { + /* first, try top-down */ + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Trying top-down reconstruction for `%s'\n", dc->filename); + try_top_down_reconstruction (dc, dc->top_request); + switch (dc->top_request->state) + { + case BRS_CHK_SET: + break; /* normal */ + case BRS_DOWNLOAD_DOWN: + break; /* normal, some blocks already down */ + case BRS_DOWNLOAD_UP: + /* already done entirely, party! */ + if (dc->rfh != NULL) + { + /* avoid hanging on to file handle longer than + * necessary */ + GNUNET_DISK_file_close (dc->rfh); + dc->rfh = NULL; + } + return; + case BRS_ERROR: + GNUNET_asprintf (&dc->emsg, _("Invalid URI")); + GNUNET_FS_download_sync_ (dc); + pi.status = GNUNET_FS_STATUS_DOWNLOAD_ERROR; + pi.value.download.specifics.error.message = dc->emsg; + GNUNET_FS_download_make_status_ (&pi, dc); + return; + default: + GNUNET_assert (0); + break; + } + } + } + /* attempt reconstruction from meta data */ + if ((GNUNET_FS_uri_chk_get_file_size (dc->uri) <= MAX_INLINE_SIZE) && + (NULL != dc->meta)) + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Trying to find embedded meta data for download of size %llu with %u bytes MD\n", + (unsigned long long) GNUNET_FS_uri_chk_get_file_size (dc->uri), + (unsigned int) + GNUNET_CONTAINER_meta_data_get_serialized_size (dc->meta)); + GNUNET_CONTAINER_meta_data_iterate (dc->meta, &match_full_data, dc); + if (dc->top_request->state == BRS_DOWNLOAD_UP) + { + if (dc->rfh != NULL) + { + /* avoid hanging on to file handle longer than + * necessary */ + GNUNET_DISK_file_close (dc->rfh); + dc->rfh = NULL; + } + return; /* finished, status update was already done for us */ + } + } + if (dc->rfh != NULL) + { + /* finally, try bottom-up */ + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Trying bottom-up reconstruction of file `%s'\n", dc->filename); + dc->te = + GNUNET_FS_tree_encoder_create (dc->h, dc->old_file_size, dc, &fh_reader, + &reconstruct_cb, NULL, + &reconstruct_cont); + dc->task = GNUNET_SCHEDULER_add_now (&get_next_block, dc); + } + else + { + /* simple, top-level download */ + schedule_block_download (dc, dc->top_request); + } + if (dc->top_request->state == BRS_DOWNLOAD_UP) + check_completed (dc); +} + + +/** + * Create SUSPEND event for the given download operation + * and then clean up our state (without stop signal). + * + * @param cls the 'struct GNUNET_FS_DownloadContext' to signal for + */ +void +GNUNET_FS_download_signal_suspend_ (void *cls) +{ + struct GNUNET_FS_DownloadContext *dc = cls; + struct GNUNET_FS_ProgressInfo pi; + + if (dc->top != NULL) + GNUNET_FS_end_top (dc->h, dc->top); + while (NULL != dc->child_head) + GNUNET_FS_download_signal_suspend_ (dc->child_head); + if (dc->search != NULL) + { + dc->search->download = NULL; + dc->search = NULL; + } + if (dc->job_queue != NULL) + { + GNUNET_FS_dequeue_ (dc->job_queue); + dc->job_queue = NULL; + } + if (dc->parent != NULL) + GNUNET_CONTAINER_DLL_remove (dc->parent->child_head, dc->parent->child_tail, + dc); + if (dc->task != GNUNET_SCHEDULER_NO_TASK) + { + GNUNET_SCHEDULER_cancel (dc->task); + dc->task = GNUNET_SCHEDULER_NO_TASK; + } + pi.status = GNUNET_FS_STATUS_DOWNLOAD_SUSPEND; + GNUNET_FS_download_make_status_ (&pi, dc); + if (dc->te != NULL) + { + GNUNET_FS_tree_encoder_finish (dc->te, NULL, NULL); + dc->te = NULL; + } + if (dc->rfh != NULL) + { + GNUNET_DISK_file_close (dc->rfh); + dc->rfh = NULL; + } + GNUNET_FS_free_download_request_ (dc->top_request); + if (dc->active != NULL) + { + GNUNET_CONTAINER_multihashmap_destroy (dc->active); + dc->active = NULL; + } + GNUNET_free_non_null (dc->filename); + GNUNET_CONTAINER_meta_data_destroy (dc->meta); + GNUNET_FS_uri_destroy (dc->uri); + GNUNET_free_non_null (dc->temp_filename); + GNUNET_free_non_null (dc->serialization); + GNUNET_free (dc); +} + + +/** + * Download parts of a file. Note that this will store + * the blocks at the respective offset in the given file. Also, the + * download is still using the blocking of the underlying FS + * encoding. As a result, the download may *write* outside of the + * given boundaries (if offset and length do not match the 32k FS + * block boundaries). <p> + * + * This function should be used to focus a download towards a + * particular portion of the file (optimization), not to strictly + * limit the download to exactly those bytes. + * + * @param h handle to the file sharing subsystem + * @param uri the URI of the file (determines what to download); CHK or LOC URI + * @param meta known metadata for the file (can be NULL) + * @param filename where to store the file, maybe NULL (then no file is + * created on disk and data must be grabbed from the callbacks) + * @param tempname where to store temporary file data, not used if filename is non-NULL; + * can be NULL (in which case we will pick a name if needed); the temporary file + * may already exist, in which case we will try to use the data that is there and + * if it is not what is desired, will overwrite it + * @param offset at what offset should we start the download (typically 0) + * @param length how many bytes should be downloaded starting at offset + * @param anonymity anonymity level to use for the download + * @param options various options + * @param cctx initial value for the client context for this download + * @param parent parent download to associate this download with (use NULL + * for top-level downloads; useful for manually-triggered recursive downloads) + * @return context that can be used to control this download + */ +struct GNUNET_FS_DownloadContext * +GNUNET_FS_download_start (struct GNUNET_FS_Handle *h, + const struct GNUNET_FS_Uri *uri, + const struct GNUNET_CONTAINER_MetaData *meta, + const char *filename, const char *tempname, + uint64_t offset, uint64_t length, uint32_t anonymity, + enum GNUNET_FS_DownloadOptions options, void *cctx, + struct GNUNET_FS_DownloadContext *parent) +{ + struct GNUNET_FS_DownloadContext *dc; + + GNUNET_assert (GNUNET_FS_uri_test_chk (uri) || GNUNET_FS_uri_test_loc (uri)); + + if ((offset + length < offset) || + (offset + length > GNUNET_FS_uri_chk_get_file_size (uri))) + { + GNUNET_break (0); + return NULL; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Starting download `%s' of %llu bytes\n", + filename, (unsigned long long) length); + dc = GNUNET_malloc (sizeof (struct GNUNET_FS_DownloadContext)); + dc->h = h; + dc->parent = parent; + if (parent != NULL) + { + GNUNET_CONTAINER_DLL_insert (parent->child_head, parent->child_tail, dc); + } + dc->uri = GNUNET_FS_uri_dup (uri); + dc->meta = GNUNET_CONTAINER_meta_data_duplicate (meta); + dc->client_info = cctx; + dc->start_time = GNUNET_TIME_absolute_get (); + if (NULL != filename) + { + dc->filename = GNUNET_strdup (filename); + if (GNUNET_YES == GNUNET_DISK_file_test (filename)) + GNUNET_DISK_file_size (filename, &dc->old_file_size, GNUNET_YES); + } + if (GNUNET_FS_uri_test_loc (dc->uri)) + GNUNET_assert (GNUNET_OK == + GNUNET_FS_uri_loc_get_peer_identity (dc->uri, &dc->target)); + dc->offset = offset; + dc->length = length; + dc->anonymity = anonymity; + dc->options = options; + dc->active = + GNUNET_CONTAINER_multihashmap_create (1 + 2 * (length / DBLOCK_SIZE)); + dc->treedepth = + GNUNET_FS_compute_depth (GNUNET_FS_uri_chk_get_file_size (dc->uri)); + if ((filename == NULL) && (is_recursive_download (dc))) + { + if (tempname != NULL) + dc->temp_filename = GNUNET_strdup (tempname); + else + dc->temp_filename = GNUNET_DISK_mktemp ("gnunet-directory-download-tmp"); + } + + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Download tree has depth %u\n", + dc->treedepth); + if (parent == NULL) + { + dc->top = + GNUNET_FS_make_top (dc->h, &GNUNET_FS_download_signal_suspend_, dc); + } + dc->task = GNUNET_SCHEDULER_add_now (&GNUNET_FS_download_start_task_, dc); + return dc; +} + + +/** + * Download parts of a file based on a search result. The download + * will be associated with the search result (and the association + * will be preserved when serializing/deserializing the state). + * If the search is stopped, the download will not be aborted but + * be 'promoted' to a stand-alone download. + * + * As with the other download function, this will store + * the blocks at the respective offset in the given file. Also, the + * download is still using the blocking of the underlying FS + * encoding. As a result, the download may *write* outside of the + * given boundaries (if offset and length do not match the 32k FS + * block boundaries). <p> + * + * The given range can be used to focus a download towards a + * particular portion of the file (optimization), not to strictly + * limit the download to exactly those bytes. + * + * @param h handle to the file sharing subsystem + * @param sr the search result to use for the download (determines uri and + * meta data and associations) + * @param filename where to store the file, maybe NULL (then no file is + * created on disk and data must be grabbed from the callbacks) + * @param tempname where to store temporary file data, not used if filename is non-NULL; + * can be NULL (in which case we will pick a name if needed); the temporary file + * may already exist, in which case we will try to use the data that is there and + * if it is not what is desired, will overwrite it + * @param offset at what offset should we start the download (typically 0) + * @param length how many bytes should be downloaded starting at offset + * @param anonymity anonymity level to use for the download + * @param options various download options + * @param cctx initial value for the client context for this download + * @return context that can be used to control this download + */ +struct GNUNET_FS_DownloadContext * +GNUNET_FS_download_start_from_search (struct GNUNET_FS_Handle *h, + struct GNUNET_FS_SearchResult *sr, + const char *filename, + const char *tempname, uint64_t offset, + uint64_t length, uint32_t anonymity, + enum GNUNET_FS_DownloadOptions options, + void *cctx) +{ + struct GNUNET_FS_DownloadContext *dc; + + if ((sr == NULL) || (sr->download != NULL)) + { + GNUNET_break (0); + return NULL; + } + GNUNET_assert (GNUNET_FS_uri_test_chk (sr->uri) || + GNUNET_FS_uri_test_loc (sr->uri)); + if ((offset + length < offset) || + (offset + length > sr->uri->data.chk.file_length)) + { + GNUNET_break (0); + return NULL; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Starting download `%s' of %llu bytes\n", + filename, (unsigned long long) length); + dc = GNUNET_malloc (sizeof (struct GNUNET_FS_DownloadContext)); + dc->h = h; + dc->search = sr; + sr->download = dc; + if (sr->probe_ctx != NULL) + { + GNUNET_FS_download_stop (sr->probe_ctx, GNUNET_YES); + sr->probe_ctx = NULL; + } + dc->uri = GNUNET_FS_uri_dup (sr->uri); + dc->meta = GNUNET_CONTAINER_meta_data_duplicate (sr->meta); + dc->client_info = cctx; + dc->start_time = GNUNET_TIME_absolute_get (); + if (NULL != filename) + { + dc->filename = GNUNET_strdup (filename); + if (GNUNET_YES == GNUNET_DISK_file_test (filename)) + GNUNET_DISK_file_size (filename, &dc->old_file_size, GNUNET_YES); + } + if (GNUNET_FS_uri_test_loc (dc->uri)) + GNUNET_assert (GNUNET_OK == + GNUNET_FS_uri_loc_get_peer_identity (dc->uri, &dc->target)); + dc->offset = offset; + dc->length = length; + dc->anonymity = anonymity; + dc->options = options; + dc->active = + GNUNET_CONTAINER_multihashmap_create (1 + 2 * (length / DBLOCK_SIZE)); + dc->treedepth = + GNUNET_FS_compute_depth (GNUNET_ntohll (dc->uri->data.chk.file_length)); + if ((filename == NULL) && (is_recursive_download (dc))) + { + if (tempname != NULL) + dc->temp_filename = GNUNET_strdup (tempname); + else + dc->temp_filename = GNUNET_DISK_mktemp ("gnunet-directory-download-tmp"); + } + + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Download tree has depth %u\n", + dc->treedepth); + dc->task = GNUNET_SCHEDULER_add_now (&GNUNET_FS_download_start_task_, dc); + return dc; +} + + +/** + * Start the downloading process (by entering the queue). + * + * @param dc our download context + */ +void +GNUNET_FS_download_start_downloading_ (struct GNUNET_FS_DownloadContext *dc) +{ + if (dc->completed == dc->length) + return; + GNUNET_assert (dc->job_queue == NULL); + dc->job_queue = + GNUNET_FS_queue_ (dc->h, &activate_fs_download, &deactivate_fs_download, + dc, (dc->length + DBLOCK_SIZE - 1) / DBLOCK_SIZE); +} + + +/** + * Stop a download (aborts if download is incomplete). + * + * @param dc handle for the download + * @param do_delete delete files of incomplete downloads + */ +void +GNUNET_FS_download_stop (struct GNUNET_FS_DownloadContext *dc, int do_delete) +{ + struct GNUNET_FS_ProgressInfo pi; + int have_children; + + if (dc->top != NULL) + GNUNET_FS_end_top (dc->h, dc->top); + + + if (dc->task != GNUNET_SCHEDULER_NO_TASK) + { + GNUNET_SCHEDULER_cancel (dc->task); + dc->task = GNUNET_SCHEDULER_NO_TASK; + } + if (dc->search != NULL) + { + dc->search->download = NULL; + GNUNET_FS_search_result_sync_ (dc->search); + dc->search = NULL; + } + if (dc->job_queue != NULL) + { + GNUNET_FS_dequeue_ (dc->job_queue); + dc->job_queue = NULL; + } + if (dc->te != NULL) + { + GNUNET_FS_tree_encoder_finish (dc->te, NULL, NULL); + dc->te = NULL; + } + have_children = (NULL != dc->child_head) ? GNUNET_YES : GNUNET_NO; + while (NULL != dc->child_head) + GNUNET_FS_download_stop (dc->child_head, do_delete); + if (dc->parent != NULL) + GNUNET_CONTAINER_DLL_remove (dc->parent->child_head, dc->parent->child_tail, + dc); + if (dc->serialization != NULL) + GNUNET_FS_remove_sync_file_ (dc->h, + ((dc->parent != NULL) || + (dc->search != + NULL)) ? GNUNET_FS_SYNC_PATH_CHILD_DOWNLOAD : + GNUNET_FS_SYNC_PATH_MASTER_DOWNLOAD, + dc->serialization); + if ((GNUNET_YES == have_children) && (dc->parent == NULL)) + GNUNET_FS_remove_sync_dir_ (dc->h, + (dc->search != + NULL) ? GNUNET_FS_SYNC_PATH_CHILD_DOWNLOAD : + GNUNET_FS_SYNC_PATH_MASTER_DOWNLOAD, + dc->serialization); + pi.status = GNUNET_FS_STATUS_DOWNLOAD_STOPPED; + GNUNET_FS_download_make_status_ (&pi, dc); + GNUNET_FS_free_download_request_ (dc->top_request); + dc->top_request = NULL; + if (dc->active != NULL) + { + GNUNET_CONTAINER_multihashmap_destroy (dc->active); + dc->active = NULL; + } + if (dc->filename != NULL) + { + if ((dc->completed != dc->length) && (GNUNET_YES == do_delete)) + { + if (0 != UNLINK (dc->filename)) + GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", + dc->filename); + } + GNUNET_free (dc->filename); + } + GNUNET_CONTAINER_meta_data_destroy (dc->meta); + GNUNET_FS_uri_destroy (dc->uri); + if (NULL != dc->temp_filename) + { + if (0 != UNLINK (dc->temp_filename)) + GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR, "unlink", + dc->temp_filename); + GNUNET_free (dc->temp_filename); + } + GNUNET_free_non_null (dc->serialization); + GNUNET_free (dc); +} + +/* end of fs_download.c */ diff --git a/src/fs/fs_file_information.c b/src/fs/fs_file_information.c new file mode 100644 index 0000000..85a076f --- /dev/null +++ b/src/fs/fs_file_information.c @@ -0,0 +1,453 @@ +/* + This file is part of GNUnet. + (C) 2009, 2011 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/fs_file_information.c + * @brief Manage information for publishing directory hierarchies + * @author Christian Grothoff + */ +#include "platform.h" +#include <extractor.h> +#include "gnunet_fs_service.h" +#include "fs_api.h" +#include "fs_tree.h" + + +/** + * Obtain the name under which this file information + * structure is stored on disk. Only works for top-level + * file information structures. + * + * @param s structure to get the filename for + * @return NULL on error, otherwise filename that + * can be passed to "GNUNET_FS_file_information_recover" + * to read this fi-struct from disk. + */ +const char * +GNUNET_FS_file_information_get_id (struct GNUNET_FS_FileInformation *s) +{ + if (NULL != s->dir) + return NULL; + return s->serialization; +} + +/** + * Obtain the filename from the file information structure. + * + * @param s structure to get the filename for + * @return "filename" field of the structure (can be NULL) + */ +const char * +GNUNET_FS_file_information_get_filename (struct GNUNET_FS_FileInformation *s) +{ + return s->filename; +} + + +/** + * Set the filename in the file information structure. + * If filename was already set, frees it before setting the new one. + * Makes a copy of the argument. + * + * @param s structure to get the filename for + * @param filename filename to set + */ +void +GNUNET_FS_file_information_set_filename (struct GNUNET_FS_FileInformation *s, + const char *filename) +{ + GNUNET_free_non_null (s->filename); + if (filename) + s->filename = GNUNET_strdup (filename); + else + s->filename = NULL; +} + +/** + * Create an entry for a file in a publish-structure. + * + * @param h handle to the file sharing subsystem + * @param client_info initial value for the client-info value for this entry + * @param filename name of the file or directory to publish + * @param keywords under which keywords should this file be available + * directly; can be NULL + * @param meta metadata for the file + * @param do_index GNUNET_YES for index, GNUNET_NO for insertion, + * GNUNET_SYSERR for simulation + * @param bo block options + * @return publish structure entry for the file + */ +struct GNUNET_FS_FileInformation * +GNUNET_FS_file_information_create_from_file (struct GNUNET_FS_Handle *h, + void *client_info, + const char *filename, + const struct GNUNET_FS_Uri + *keywords, + const struct + GNUNET_CONTAINER_MetaData *meta, + int do_index, + const struct GNUNET_FS_BlockOptions + *bo) +{ + struct FileInfo *fi; + struct stat sbuf; + struct GNUNET_FS_FileInformation *ret; + const char *fn; + const char *ss; + +#if WINDOWS + char fn_conv[MAX_PATH]; +#endif + + if (0 != STAT (filename, &sbuf)) + { + GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "stat", filename); + return NULL; + } + fi = GNUNET_FS_make_file_reader_context_ (filename); + if (fi == NULL) + { + GNUNET_break (0); + return NULL; + } + ret = + GNUNET_FS_file_information_create_from_reader (h, client_info, + sbuf.st_size, + &GNUNET_FS_data_reader_file_, + fi, keywords, meta, + do_index, bo); + if (ret == NULL) + return NULL; + ret->h = h; + ret->filename = GNUNET_strdup (filename); +#if !WINDOWS + fn = filename; +#else + plibc_conv_to_win_path (filename, fn_conv); + fn = fn_conv; +#endif + while (NULL != (ss = strstr (fn, DIR_SEPARATOR_STR))) + fn = ss + 1; +#if !WINDOWS + GNUNET_CONTAINER_meta_data_insert (ret->meta, "<gnunet>", + EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME, + EXTRACTOR_METAFORMAT_C_STRING, + "text/plain", fn, strlen (fn) + 1); +#else + GNUNET_CONTAINER_meta_data_insert (ret->meta, "<gnunet>", + EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", fn, strlen (fn) + 1); +#endif + return ret; +} + + +/** + * Create an entry for a file in a publish-structure. + * + * @param h handle to the file sharing subsystem + * @param client_info initial value for the client-info value for this entry + * @param length length of the file + * @param data data for the file (should not be used afterwards by + * the caller; callee will "free") + * @param keywords under which keywords should this file be available + * directly; can be NULL + * @param meta metadata for the file + * @param do_index GNUNET_YES for index, GNUNET_NO for insertion, + * GNUNET_SYSERR for simulation + * @param bo block options + * @return publish structure entry for the file + */ +struct GNUNET_FS_FileInformation * +GNUNET_FS_file_information_create_from_data (struct GNUNET_FS_Handle *h, + void *client_info, uint64_t length, + void *data, + const struct GNUNET_FS_Uri + *keywords, + const struct + GNUNET_CONTAINER_MetaData *meta, + int do_index, + const struct GNUNET_FS_BlockOptions + *bo) +{ + if (GNUNET_YES == do_index) + { + GNUNET_break (0); + return NULL; + } + return GNUNET_FS_file_information_create_from_reader (h, client_info, length, + &GNUNET_FS_data_reader_copy_, + data, keywords, meta, + do_index, bo); +} + + +/** + * Create an entry for a file in a publish-structure. + * + * @param h handle to the file sharing subsystem + * @param client_info initial value for the client-info value for this entry + * @param length length of the file + * @param reader function that can be used to obtain the data for the file + * @param reader_cls closure for "reader" + * @param keywords under which keywords should this file be available + * directly; can be NULL + * @param meta metadata for the file + * @param do_index GNUNET_YES for index, GNUNET_NO for insertion, + * GNUNET_SYSERR for simulation + * @param bo block options + * @return publish structure entry for the file + */ +struct GNUNET_FS_FileInformation * +GNUNET_FS_file_information_create_from_reader (struct GNUNET_FS_Handle *h, + void *client_info, + uint64_t length, + GNUNET_FS_DataReader reader, + void *reader_cls, + const struct GNUNET_FS_Uri + *keywords, + const struct + GNUNET_CONTAINER_MetaData *meta, + int do_index, + const struct + GNUNET_FS_BlockOptions *bo) +{ + struct GNUNET_FS_FileInformation *ret; + + if ((GNUNET_YES == do_index) && (reader != &GNUNET_FS_data_reader_file_)) + { + GNUNET_break (0); + return NULL; + } + ret = GNUNET_malloc (sizeof (struct GNUNET_FS_FileInformation)); + ret->h = h; + ret->client_info = client_info; + ret->meta = GNUNET_CONTAINER_meta_data_duplicate (meta); + if (ret->meta == NULL) + ret->meta = GNUNET_CONTAINER_meta_data_create (); + ret->keywords = (keywords == NULL) ? NULL : GNUNET_FS_uri_dup (keywords); + ret->data.file.reader = reader; + ret->data.file.reader_cls = reader_cls; + ret->data.file.do_index = do_index; + ret->data.file.file_size = length; + ret->bo = *bo; + return ret; +} + + +/** + * Test if a given entry represents a directory. + * + * @param ent check if this FI represents a directory + * @return GNUNET_YES if so, GNUNET_NO if not + */ +int +GNUNET_FS_file_information_is_directory (const struct GNUNET_FS_FileInformation + *ent) +{ + return ent->is_directory; +} + + +/** + * Create an entry for an empty directory in a publish-structure. + * This function should be used by applications for which the + * use of "GNUNET_FS_file_information_create_from_directory" + * is not appropriate. + * + * @param h handle to the file sharing subsystem + * @param client_info initial value for the client-info value for this entry + * @param meta metadata for the directory + * @param keywords under which keywords should this directory be available + * directly; can be NULL + * @param bo block options + * @param filename name of the directory; can be NULL + * @return publish structure entry for the directory , NULL on error + */ +struct GNUNET_FS_FileInformation * +GNUNET_FS_file_information_create_empty_directory (struct GNUNET_FS_Handle *h, + void *client_info, + const struct GNUNET_FS_Uri + *keywords, + const struct + GNUNET_CONTAINER_MetaData + *meta, + const struct + GNUNET_FS_BlockOptions *bo, + const char *filename) +{ + struct GNUNET_FS_FileInformation *ret; + + ret = GNUNET_malloc (sizeof (struct GNUNET_FS_FileInformation)); + ret->h = h; + ret->client_info = client_info; + ret->meta = GNUNET_CONTAINER_meta_data_duplicate (meta); + ret->keywords = GNUNET_FS_uri_dup (keywords); + ret->bo = *bo; + ret->is_directory = GNUNET_YES; + if (filename != NULL) + ret->filename = GNUNET_strdup (filename); + return ret; +} + + +/** + * Add an entry to a directory in a publish-structure. Clients + * should never modify publish structures that were passed to + * "GNUNET_FS_publish_start" already. + * + * @param dir the directory + * @param ent the entry to add; the entry must not have been + * added to any other directory at this point and + * must not include "dir" in its structure + * @return GNUNET_OK on success, GNUNET_SYSERR on error + */ +int +GNUNET_FS_file_information_add (struct GNUNET_FS_FileInformation *dir, + struct GNUNET_FS_FileInformation *ent) +{ + if ((ent->dir != NULL) || (ent->next != NULL) || (dir->is_directory != GNUNET_YES)) + { + GNUNET_break (0); + return GNUNET_SYSERR; + } + ent->dir = dir; + ent->next = dir->data.dir.entries; + dir->data.dir.entries = ent; + dir->data.dir.dir_size = 0; + return GNUNET_OK; +} + + +/** + * Inspect a file or directory in a publish-structure. Clients + * should never modify publish structures that were passed to + * "GNUNET_FS_publish_start" already. When called on a directory, + * this function will FIRST call "proc" with information about + * the directory itself and then for each of the files in the + * directory (but not for files in subdirectories). When called + * on a file, "proc" will be called exactly once (with information + * about the specific file). + * + * @param dir the directory + * @param proc function to call on each entry + * @param proc_cls closure for proc + */ +void +GNUNET_FS_file_information_inspect (struct GNUNET_FS_FileInformation *dir, + GNUNET_FS_FileInformationProcessor proc, + void *proc_cls) +{ + struct GNUNET_FS_FileInformation *pos; + int no; + + no = GNUNET_NO; + if (GNUNET_OK != + proc (proc_cls, dir, + (dir->is_directory == GNUNET_YES) ? dir->data.dir.dir_size : dir->data. + file.file_size, dir->meta, &dir->keywords, &dir->bo, + (dir->is_directory == GNUNET_YES) ? &no : &dir->data.file.do_index, + &dir->client_info)) + return; + if (dir->is_directory != GNUNET_YES) + return; + pos = dir->data.dir.entries; + while (pos != NULL) + { + no = GNUNET_NO; + if (GNUNET_OK != + proc (proc_cls, pos, + (pos->is_directory == GNUNET_YES) ? pos->data.dir.dir_size : pos->data. + file.file_size, pos->meta, &pos->keywords, &pos->bo, + (pos->is_directory == GNUNET_YES) ? &no : &pos->data.file.do_index, + &pos->client_info)) + break; + pos = pos->next; + } +} + + +/** + * Destroy publish-structure. Clients should never destroy publish + * structures that were passed to "GNUNET_FS_publish_start" already. + * + * @param fi structure to destroy + * @param cleaner function to call on each entry in the structure + * (useful to clean up client_info); can be NULL; return + * values are ignored + * @param cleaner_cls closure for cleaner + */ +void +GNUNET_FS_file_information_destroy (struct GNUNET_FS_FileInformation *fi, + GNUNET_FS_FileInformationProcessor cleaner, + void *cleaner_cls) +{ + struct GNUNET_FS_FileInformation *pos; + int no; + + no = GNUNET_NO; + if (fi->is_directory == GNUNET_YES) + { + /* clean up directory */ + while (NULL != (pos = fi->data.dir.entries)) + { + fi->data.dir.entries = pos->next; + GNUNET_FS_file_information_destroy (pos, cleaner, cleaner_cls); + } + /* clean up client-info */ + if (NULL != cleaner) + cleaner (cleaner_cls, fi, fi->data.dir.dir_size, fi->meta, &fi->keywords, + &fi->bo, &no, &fi->client_info); + GNUNET_free_non_null (fi->data.dir.dir_data); + } + else + { + /* call clean-up function of the reader */ + if (fi->data.file.reader != NULL) + fi->data.file.reader (fi->data.file.reader_cls, 0, 0, NULL, NULL); + /* clean up client-info */ + if (NULL != cleaner) + cleaner (cleaner_cls, fi, fi->data.file.file_size, fi->meta, + &fi->keywords, &fi->bo, &fi->data.file.do_index, + &fi->client_info); + } + GNUNET_free_non_null (fi->filename); + GNUNET_free_non_null (fi->emsg); + GNUNET_free_non_null (fi->chk_uri); + /* clean up serialization */ + if ((NULL != fi->serialization) && (0 != UNLINK (fi->serialization))) + GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", + fi->serialization); + if (NULL != fi->keywords) + GNUNET_FS_uri_destroy (fi->keywords); + if (NULL != fi->meta) + GNUNET_CONTAINER_meta_data_destroy (fi->meta); + GNUNET_free_non_null (fi->serialization); + if (fi->te != NULL) + { + GNUNET_FS_tree_encoder_finish (fi->te, NULL, NULL); + fi->te = NULL; + } + GNUNET_free (fi); +} + + +/* end of fs_file_information.c */ diff --git a/src/fs/fs_getopt.c b/src/fs/fs_getopt.c new file mode 100644 index 0000000..0374774 --- /dev/null +++ b/src/fs/fs_getopt.c @@ -0,0 +1,197 @@ +/* + This file is part of GNUnet. + (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/fs_getopt.c + * @brief helper functions for command-line argument processing + * @author Igor Wronsky, Christian Grothoff + */ +#include "platform.h" +#include "gnunet_fs_service.h" +#include "fs_api.h" + +/* ******************** command-line option parsing API ******************** */ + +/** + * Command-line option parser function that allows the user + * to specify one or more '-k' options with keywords. Each + * specified keyword will be added to the URI. A pointer to + * the URI must be passed as the "scls" argument. + * + * @param ctx command line processor context + * @param scls must be of type "struct GNUNET_FS_Uri **" + * @param option name of the option (typically 'k') + * @param value command line argument given + * @return GNUNET_OK on success + */ +int +GNUNET_FS_getopt_set_keywords (struct GNUNET_GETOPT_CommandLineProcessorContext + *ctx, void *scls, const char *option, + const char *value) +{ + struct GNUNET_FS_Uri **uri = scls; + struct GNUNET_FS_Uri *u = *uri; + char *val; + size_t slen; + + if (u == NULL) + { + u = GNUNET_malloc (sizeof (struct GNUNET_FS_Uri)); + *uri = u; + u->type = ksk; + u->data.ksk.keywordCount = 0; + u->data.ksk.keywords = NULL; + } + else + { + GNUNET_assert (u->type == ksk); + } + slen = strlen (value); + if (slen == 0) + return GNUNET_SYSERR; /* cannot be empty */ + if (value[0] == '+') + { + /* simply preserve the "mandatory" flag */ + if (slen < 2) + return GNUNET_SYSERR; /* empty keywords not allowed */ + if ((value[1] == '"') && (slen > 3) && (value[slen - 1] == '"')) + { + /* remove the quotes, keep the '+' */ + val = GNUNET_malloc (slen - 1); + val[0] = '+'; + memcpy (&val[1], &value[2], slen - 3); + val[slen - 2] = '\0'; + } + else + { + /* no quotes, just keep the '+' */ + val = GNUNET_strdup (value); + } + } + else + { + if ((value[0] == '"') && (slen > 2) && (value[slen - 1] == '"')) + { + /* remove the quotes, add a space */ + val = GNUNET_malloc (slen); + val[0] = ' '; + memcpy (&val[1], &value[1], slen - 2); + val[slen - 1] = '\0'; + } + else + { + /* add a space to indicate "not mandatory" */ + val = GNUNET_malloc (slen + 2); + strcpy (val, " "); + strcat (val, value); + } + } + GNUNET_array_append (u->data.ksk.keywords, u->data.ksk.keywordCount, val); + return GNUNET_OK; +} + + +/** + * Command-line option parser function that allows the user to specify + * one or more '-m' options with metadata. Each specified entry of + * the form "type=value" will be added to the metadata. A pointer to + * the metadata must be passed as the "scls" argument. + * + * @param ctx command line processor context + * @param scls must be of type "struct GNUNET_MetaData **" + * @param option name of the option (typically 'k') + * @param value command line argument given + * @return GNUNET_OK on success + */ +int +GNUNET_FS_getopt_set_metadata (struct GNUNET_GETOPT_CommandLineProcessorContext + *ctx, void *scls, const char *option, + const char *value) +{ + struct GNUNET_CONTAINER_MetaData **mm = scls; + enum EXTRACTOR_MetaType type; + const char *typename; + const char *typename_i18n; + struct GNUNET_CONTAINER_MetaData *meta; + char *tmp; + + meta = *mm; + if (meta == NULL) + { + meta = GNUNET_CONTAINER_meta_data_create (); + *mm = meta; + } + +#if ENABLE_NLS + tmp = GNUNET_STRINGS_to_utf8 (value, strlen (value), nl_langinfo (CODESET)); +#else + tmp = GNUNET_STRINGS_to_utf8 (value, strlen (value), "utf-8"); +#endif + type = EXTRACTOR_metatype_get_max (); + while (type > 0) + { + type--; + typename = EXTRACTOR_metatype_to_string (type); + typename_i18n = dgettext (LIBEXTRACTOR_GETTEXT_DOMAIN, typename); + if ((strlen (tmp) >= strlen (typename) + 1) && + (tmp[strlen (typename)] == ':') && + (0 == strncmp (typename, tmp, strlen (typename)))) + { + GNUNET_CONTAINER_meta_data_insert (meta, "<gnunet>", type, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + &tmp[strlen (typename) + 1], + strlen (&tmp[strlen (typename) + 1]) + + 1); + GNUNET_free (tmp); + tmp = NULL; + break; + } + if ((strlen (tmp) >= strlen (typename_i18n) + 1) && + (tmp[strlen (typename_i18n)] == ':') && + (0 == strncmp (typename_i18n, tmp, strlen (typename_i18n)))) + { + GNUNET_CONTAINER_meta_data_insert (meta, "<gnunet>", type, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + &tmp[strlen (typename_i18n) + 1], + strlen (&tmp + [strlen (typename_i18n) + 1]) + + 1); + GNUNET_free (tmp); + tmp = NULL; + break; + } + } + if (tmp != NULL) + { + GNUNET_CONTAINER_meta_data_insert (meta, "<gnunet>", + EXTRACTOR_METATYPE_UNKNOWN, + EXTRACTOR_METAFORMAT_UTF8, "text/plain", + tmp, strlen (tmp) + 1); + GNUNET_free (tmp); + printf (_ + ("Unknown metadata type in metadata option `%s'. Using metadata type `unknown' instead.\n"), + value); + } + return GNUNET_OK; +} + +/* end of fs_getopt.c */ diff --git a/src/fs/fs_list_indexed.c b/src/fs/fs_list_indexed.c new file mode 100644 index 0000000..784c988 --- /dev/null +++ b/src/fs/fs_list_indexed.c @@ -0,0 +1,184 @@ +/* + This file is part of GNUnet. + (C) 2003, 2004, 2006, 2009 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/fs_list_indexed.c + * @author Christian Grothoff + * @brief provide a list of all indexed files + */ + +#include "platform.h" +#include "gnunet_constants.h" +#include "gnunet_fs_service.h" +#include "gnunet_protocols.h" +#include "fs_api.h" + + +/** + * Context for "GNUNET_FS_get_indexed_files". + */ +struct GNUNET_FS_GetIndexedContext +{ + /** + * Handle to global FS context. + */ + struct GNUNET_FS_Handle *h; + + /** + * Connection to the FS service. + */ + struct GNUNET_CLIENT_Connection *client; + + /** + * Function to call for each indexed file. + */ + GNUNET_FS_IndexedFileProcessor iterator; + + /** + * Closure for iterator. + */ + void *iterator_cls; + + /** + * Continuation to trigger at the end. + */ + GNUNET_SCHEDULER_Task cont; + + /** + * Closure for cont. + */ + void *cont_cls; +}; + + +/** + * Function called on each response from the FS + * service with information about indexed files. + * + * @param cls closure (of type "struct GNUNET_FS_GetIndexedContext*") + * @param msg message with indexing information + */ +static void +handle_index_info (void *cls, const struct GNUNET_MessageHeader *msg) +{ + struct GNUNET_FS_GetIndexedContext *gic = cls; + const struct IndexInfoMessage *iim; + uint16_t msize; + const char *filename; + + if (NULL == msg) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _ + ("Failed to receive response for `%s' request from `%s' service.\n"), + "GET_INDEXED", "fs"); + (void) gic->iterator (gic->iterator_cls, NULL, NULL); + GNUNET_FS_get_indexed_files_cancel (gic); + return; + } + if (ntohs (msg->type) == GNUNET_MESSAGE_TYPE_FS_INDEX_LIST_END) + { + /* normal end-of-list */ + (void) gic->iterator (gic->iterator_cls, NULL, NULL); + GNUNET_FS_get_indexed_files_cancel (gic); + return; + } + msize = ntohs (msg->size); + iim = (const struct IndexInfoMessage *) msg; + filename = (const char *) &iim[1]; + if ((ntohs (msg->type) != GNUNET_MESSAGE_TYPE_FS_INDEX_LIST_ENTRY) || + (msize <= sizeof (struct IndexInfoMessage)) || + (filename[msize - sizeof (struct IndexInfoMessage) - 1] != '\0')) + { + /* bogus reply */ + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _ + ("Failed to receive valid response for `%s' request from `%s' service.\n"), + "GET_INDEXED", "fs"); + (void) gic->iterator (gic->iterator_cls, NULL, NULL); + GNUNET_FS_get_indexed_files_cancel (gic); + return; + } + if (GNUNET_OK != gic->iterator (gic->iterator_cls, filename, &iim->file_id)) + { + GNUNET_FS_get_indexed_files_cancel (gic); + return; + } + /* get more */ + GNUNET_CLIENT_receive (gic->client, &handle_index_info, gic, + GNUNET_CONSTANTS_SERVICE_TIMEOUT); +} + + +/** + * Iterate over all indexed files. + * + * @param h handle to the file sharing subsystem + * @param iterator function to call on each indexed file + * @param iterator_cls closure for iterator + * @return NULL on error ('iter' is not called) + */ +struct GNUNET_FS_GetIndexedContext * +GNUNET_FS_get_indexed_files (struct GNUNET_FS_Handle *h, + GNUNET_FS_IndexedFileProcessor iterator, + void *iterator_cls) +{ + struct GNUNET_CLIENT_Connection *client; + struct GNUNET_FS_GetIndexedContext *gic; + struct GNUNET_MessageHeader msg; + + client = GNUNET_CLIENT_connect ("fs", h->cfg); + if (NULL == client) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _("Failed to not connect to `%s' service.\n"), "fs"); + return NULL; + } + gic = GNUNET_malloc (sizeof (struct GNUNET_FS_GetIndexedContext)); + gic->h = h; + gic->client = client; + gic->iterator = iterator; + gic->iterator_cls = iterator_cls; + msg.size = htons (sizeof (struct GNUNET_MessageHeader)); + msg.type = htons (GNUNET_MESSAGE_TYPE_FS_INDEX_LIST_GET); + GNUNET_assert (GNUNET_OK == + GNUNET_CLIENT_transmit_and_get_response (client, &msg, + GNUNET_CONSTANTS_SERVICE_TIMEOUT, + GNUNET_YES, + &handle_index_info, + gic)); + return gic; +} + + +/** + * Cancel iteration over all indexed files. + * + * @param gic operation to cancel + */ +void +GNUNET_FS_get_indexed_files_cancel (struct GNUNET_FS_GetIndexedContext *gic) +{ + GNUNET_CLIENT_disconnect (gic->client, GNUNET_NO); + GNUNET_free (gic); +} + + +/* end of fs_list_indexed.c */ diff --git a/src/fs/fs_misc.c b/src/fs/fs_misc.c new file mode 100644 index 0000000..89dc486 --- /dev/null +++ b/src/fs/fs_misc.c @@ -0,0 +1,231 @@ +/* + This file is part of GNUnet. + (C) 2010, 2011 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ +/** + * @file fs/fs_misc.c + * @brief misc. functions related to file-sharing in general + * @author Christian Grothoff + */ +#include "platform.h" +#include "gnunet_constants.h" +#include "gnunet_fs_service.h" +#include "fs_api.h" + + +/** + * Suggest a filename based on given metadata. + * + * @param md given meta data + * @return NULL if meta data is useless for suggesting a filename + */ +char * +GNUNET_FS_meta_data_suggest_filename (const struct GNUNET_CONTAINER_MetaData + *md) +{ + static const char *mimeMap[][2] = { + {"application/bz2", ".bz2"}, + {"application/gnunet-directory", ".gnd"}, + {"application/java", ".class"}, + {"application/msword", ".doc"}, + {"application/ogg", ".ogg"}, + {"application/pdf", ".pdf"}, + {"application/pgp-keys", ".key"}, + {"application/pgp-signature", ".pgp"}, + {"application/postscript", ".ps"}, + {"application/rar", ".rar"}, + {"application/rtf", ".rtf"}, + {"application/xml", ".xml"}, + {"application/x-debian-package", ".deb"}, + {"application/x-dvi", ".dvi"}, + {"applixation/x-flac", ".flac"}, + {"applixation/x-gzip", ".gz"}, + {"application/x-java-archive", ".jar"}, + {"application/x-java-vm", ".class"}, + {"application/x-python-code", ".pyc"}, + {"application/x-redhat-package-manager", ".rpm"}, + {"application/x-rpm", ".rpm"}, + {"application/x-tar", ".tar"}, + {"application/x-tex-pk", ".pk"}, + {"application/x-texinfo", ".texinfo"}, + {"application/x-xcf", ".xcf"}, + {"application/x-xfig", ".xfig"}, + {"application/zip", ".zip"}, + + {"audio/midi", ".midi"}, + {"audio/mpeg", ".mp3"}, + {"audio/real", ".rm"}, + {"audio/x-wav", ".wav"}, + + {"image/gif", ".gif"}, + {"image/jpeg", ".jpg"}, + {"image/pcx", ".pcx"}, + {"image/png", ".png"}, + {"image/tiff", ".tiff"}, + {"image/x-ms-bmp", ".bmp"}, + {"image/x-xpixmap", ".xpm"}, + + {"text/css", ".css"}, + {"text/html", ".html"}, + {"text/plain", ".txt"}, + {"text/rtf", ".rtf"}, + {"text/x-c++hdr", ".h++"}, + {"text/x-c++src", ".c++"}, + {"text/x-chdr", ".h"}, + {"text/x-csrc", ".c"}, + {"text/x-java", ".java"}, + {"text/x-moc", ".moc"}, + {"text/x-pascal", ".pas"}, + {"text/x-perl", ".pl"}, + {"text/x-python", ".py"}, + {"text/x-tex", ".tex"}, + + {"video/avi", ".avi"}, + {"video/mpeg", ".mpeg"}, + {"video/quicktime", ".qt"}, + {"video/real", ".rm"}, + {"video/x-msvideo", ".avi"}, + {NULL, NULL}, + }; + char *ret; + unsigned int i; + char *mime; + char *base; + const char *ext; + + ret = + GNUNET_CONTAINER_meta_data_get_by_type (md, + EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME); + if (ret != NULL) + return ret; + ext = NULL; + mime = + GNUNET_CONTAINER_meta_data_get_by_type (md, EXTRACTOR_METATYPE_MIMETYPE); + if (mime != NULL) + { + i = 0; + while ((mimeMap[i][0] != NULL) && (0 != strcmp (mime, mimeMap[i][0]))) + i++; + if (mimeMap[i][1] == NULL) + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG | GNUNET_ERROR_TYPE_BULK, + _("Did not find mime type `%s' in extension list.\n"), mime); + else + ext = mimeMap[i][1]; + GNUNET_free (mime); + } + base = + GNUNET_CONTAINER_meta_data_get_first_by_types (md, + EXTRACTOR_METATYPE_TITLE, + EXTRACTOR_METATYPE_BOOK_TITLE, + EXTRACTOR_METATYPE_ORIGINAL_TITLE, + EXTRACTOR_METATYPE_PACKAGE_NAME, + EXTRACTOR_METATYPE_URL, + EXTRACTOR_METATYPE_URI, + EXTRACTOR_METATYPE_DESCRIPTION, + EXTRACTOR_METATYPE_ISRC, + EXTRACTOR_METATYPE_JOURNAL_NAME, + EXTRACTOR_METATYPE_AUTHOR_NAME, + EXTRACTOR_METATYPE_SUBJECT, + EXTRACTOR_METATYPE_ALBUM, + EXTRACTOR_METATYPE_ARTIST, + EXTRACTOR_METATYPE_KEYWORDS, + EXTRACTOR_METATYPE_COMMENT, + EXTRACTOR_METATYPE_UNKNOWN, + -1); + if ((base == NULL) && (ext == NULL)) + return NULL; + if (base == NULL) + return GNUNET_strdup (ext); + if (ext == NULL) + return base; + GNUNET_asprintf (&ret, "%s%s", base, ext); + GNUNET_free (base); + return ret; +} + + +/** + * Return the current year (i.e. '2011'). + */ +unsigned int +GNUNET_FS_get_current_year () +{ + time_t tp; + struct tm *t; + + tp = time (NULL); + t = gmtime (&tp); + if (t == NULL) + return 0; + return t->tm_year + 1900; +} + + +/** + * Convert a year to an expiration time of January 1st of that year. + * + * @param year a year (after 1970, please ;-)). + * @return absolute time for January 1st of that year. + */ +struct GNUNET_TIME_Absolute +GNUNET_FS_year_to_time (unsigned int year) +{ + struct GNUNET_TIME_Absolute ret; + time_t tp; + struct tm t; + + memset (&t, 0, sizeof (t)); + if (year < 1900) + { + GNUNET_break (0); + return GNUNET_TIME_absolute_get (); /* now */ + } + t.tm_year = year - 1900; + t.tm_mday = 1; + t.tm_mon = 1; + t.tm_wday = 1; + t.tm_yday = 1; + tp = mktime (&t); + GNUNET_break (tp != (time_t) - 1); + ret.abs_value = tp * 1000LL; /* seconds to ms */ + return ret; +} + + +/** + * Convert an expiration time to the respective year (rounds) + * + * @param at absolute time + * @return year a year (after 1970), 0 on error + */ +unsigned int +GNUNET_FS_time_to_year (struct GNUNET_TIME_Absolute at) +{ + struct tm *t; + time_t tp; + + tp = at.abs_value / 1000; /* ms to seconds */ + t = gmtime (&tp); + if (t == NULL) + return 0; + return t->tm_year + 1900; + +} + + +/* end of fs_misc.c */ diff --git a/src/fs/fs_namespace.c b/src/fs/fs_namespace.c new file mode 100644 index 0000000..bfd7594 --- /dev/null +++ b/src/fs/fs_namespace.c @@ -0,0 +1,954 @@ +/* + This file is part of GNUnet + (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/fs_namespace.c + * @brief create and destroy namespaces + * @author Christian Grothoff + */ +#include "platform.h" +#include "gnunet_constants.h" +#include "gnunet_signatures.h" +#include "gnunet_util_lib.h" +#include "gnunet_fs_service.h" +#include "fs_api.h" + + +/** + * Maximum legal size for an sblock. + */ +#define MAX_SBLOCK_SIZE (60 * 1024) + + +/** + * Return the name of the directory in which we store + * our local namespaces (or rather, their public keys). + * + * @param h global fs handle + * @return NULL on error, otherwise the name of the directory + */ +static char * +get_namespace_directory (struct GNUNET_FS_Handle *h) +{ + char *dn; + + if (GNUNET_OK != + GNUNET_CONFIGURATION_get_value_filename (h->cfg, "FS", "IDENTITY_DIR", + &dn)) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + _("Configuration fails to specify `%s' in section `%s'\n"), + "IDENTITY_DIR", "fs"); + return NULL; + } + return dn; +} + + +/** + * Return the name of the directory in which we store + * the update information graph for the given local namespace. + * + * @param ns namespace handle + * @return NULL on error, otherwise the name of the directory + */ +static char * +get_update_information_directory (struct GNUNET_FS_Namespace *ns) +{ + char *dn; + char *ret; + + if (GNUNET_OK != + GNUNET_CONFIGURATION_get_value_filename (ns->h->cfg, "FS", "UPDATE_DIR", + &dn)) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + _("Configuration fails to specify `%s' in section `%s'\n"), + "UPDATE_DIR", "fs"); + return NULL; + } + GNUNET_asprintf (&ret, "%s%s%s", dn, DIR_SEPARATOR_STR, ns->name); + GNUNET_free (dn); + return ret; +} + + +/** + * Write the namespace update node graph to a file. + * + * @param ns namespace to dump + */ +static void +write_update_information_graph (struct GNUNET_FS_Namespace *ns) +{ + char *fn; + struct GNUNET_BIO_WriteHandle *wh; + unsigned int i; + struct NamespaceUpdateNode *n; + char *uris; + + fn = get_update_information_directory (ns); + wh = GNUNET_BIO_write_open (fn); + if (wh == NULL) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + _("Failed to open `%s' for writing: %s\n"), STRERROR (errno)); + GNUNET_free (fn); + return; + } + if (GNUNET_OK != GNUNET_BIO_write_int32 (wh, ns->update_node_count)) + goto END; + for (i = 0; i < ns->update_node_count; i++) + { + n = ns->update_nodes[i]; + uris = GNUNET_FS_uri_to_string (n->uri); + if ((GNUNET_OK != GNUNET_BIO_write_string (wh, n->id)) || + (GNUNET_OK != GNUNET_BIO_write_meta_data (wh, n->md)) || + (GNUNET_OK != GNUNET_BIO_write_string (wh, n->update)) || + (GNUNET_OK != GNUNET_BIO_write_string (wh, uris))) + { + GNUNET_free (uris); + break; + } + GNUNET_free (uris); + } +END: + if (GNUNET_OK != GNUNET_BIO_write_close (wh)) + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, _("Failed to write `%s': %s\n"), + STRERROR (errno)); + GNUNET_free (fn); +} + + +/** + * Read the namespace update node graph from a file. + * + * @param ns namespace to read + */ +static void +read_update_information_graph (struct GNUNET_FS_Namespace *ns) +{ + char *fn; + struct GNUNET_BIO_ReadHandle *rh; + unsigned int i; + struct NamespaceUpdateNode *n; + char *uris; + uint32_t count; + char *emsg; + + fn = get_update_information_directory (ns); + if (GNUNET_YES != GNUNET_DISK_file_test (fn)) + { + GNUNET_free (fn); + return; + } + rh = GNUNET_BIO_read_open (fn); + if (rh == NULL) + { + GNUNET_free (fn); + return; + } + if (GNUNET_OK != GNUNET_BIO_read_int32 (rh, &count)) + { + GNUNET_break (0); + goto END; + } + if (count > 1024 * 1024) + { + GNUNET_break (0); + goto END; + } + if (count == 0) + { + GNUNET_break (GNUNET_OK == GNUNET_BIO_read_close (rh, NULL)); + GNUNET_free (fn); + return; + } + ns->update_nodes = + GNUNET_malloc (count * sizeof (struct NamespaceUpdateNode *)); + + for (i = 0; i < count; i++) + { + n = GNUNET_malloc (sizeof (struct NamespaceUpdateNode)); + if ((GNUNET_OK != GNUNET_BIO_read_string (rh, "identifier", &n->id, 1024)) + || (GNUNET_OK != GNUNET_BIO_read_meta_data (rh, "meta", &n->md)) || + (GNUNET_OK != + GNUNET_BIO_read_string (rh, "update-id", &n->update, 1024)) || + (GNUNET_OK != GNUNET_BIO_read_string (rh, "uri", &uris, 1024 * 2))) + { + GNUNET_break (0); + GNUNET_free_non_null (n->id); + GNUNET_free_non_null (n->update); + if (n->md != NULL) + GNUNET_CONTAINER_meta_data_destroy (n->md); + GNUNET_free (n); + break; + } + n->uri = GNUNET_FS_uri_parse (uris, &emsg); + GNUNET_free (uris); + if (n->uri == NULL) + { + GNUNET_break (0); + GNUNET_free (emsg); + GNUNET_free (n->id); + GNUNET_free_non_null (n->update); + GNUNET_CONTAINER_meta_data_destroy (n->md); + GNUNET_free (n); + break; + } + ns->update_nodes[i] = n; + } + ns->update_node_count = i; +END: + if (GNUNET_OK != GNUNET_BIO_read_close (rh, &emsg)) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, _("Failed to write `%s': %s\n"), emsg); + GNUNET_free (emsg); + } + GNUNET_free (fn); +} + + + + +/** + * Create a namespace with the given name; if one already + * exists, return a handle to the existing namespace. + * + * @param h handle to the file sharing subsystem + * @param name name to use for the namespace + * @return handle to the namespace, NULL on error + */ +struct GNUNET_FS_Namespace * +GNUNET_FS_namespace_create (struct GNUNET_FS_Handle *h, const char *name) +{ + char *dn; + char *fn; + struct GNUNET_FS_Namespace *ret; + + dn = get_namespace_directory (h); + GNUNET_asprintf (&fn, "%s%s%s", dn, DIR_SEPARATOR_STR, name); + GNUNET_free (dn); + ret = GNUNET_malloc (sizeof (struct GNUNET_FS_Namespace)); + ret->h = h; + ret->rc = 1; + ret->key = GNUNET_CRYPTO_rsa_key_create_from_file (fn); + if (ret->key == NULL) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + _("Failed to create or read private key for namespace `%s'\n"), + name); + GNUNET_free (ret); + GNUNET_free (fn); + return NULL; + } + ret->name = GNUNET_strdup (name); + ret->filename = fn; + return ret; +} + + +/** + * Duplicate a namespace handle. + * + * @param ns namespace handle + * @return duplicated handle to the namespace + */ +struct GNUNET_FS_Namespace * +GNUNET_FS_namespace_dup (struct GNUNET_FS_Namespace *ns) +{ + ns->rc++; + return ns; +} + + +/** + * Delete a namespace handle. Can be used for a clean shutdown (free + * memory) or also to freeze the namespace to prevent further + * insertions by anyone. + * + * @param namespace handle to the namespace that should be deleted / freed + * @param freeze prevents future insertions; creating a namespace + * with the same name again will create a fresh namespace instead + * + * @return GNUNET_OK on success, GNUNET_SYSERR on error + */ +int +GNUNET_FS_namespace_delete (struct GNUNET_FS_Namespace *namespace, int freeze) +{ + unsigned int i; + struct NamespaceUpdateNode *nsn; + + namespace->rc--; + if (freeze) + { + if (0 != UNLINK (namespace->filename)) + GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR, "unlink", + namespace->filename); + } + if (0 != namespace->rc) + return GNUNET_OK; + GNUNET_CRYPTO_rsa_key_free (namespace->key); + GNUNET_free (namespace->filename); + GNUNET_free (namespace->name); + for (i = 0; i < namespace->update_node_count; i++) + { + nsn = namespace->update_nodes[i]; + GNUNET_CONTAINER_meta_data_destroy (nsn->md); + GNUNET_FS_uri_destroy (nsn->uri); + GNUNET_free (nsn->id); + GNUNET_free (nsn->update); + GNUNET_free (nsn); + } + GNUNET_array_grow (namespace->update_nodes, namespace->update_node_count, + 0); + if (namespace->update_map != NULL) + GNUNET_CONTAINER_multihashmap_destroy (namespace->update_map); + GNUNET_free (namespace); + return GNUNET_OK; +} + + +/** + * Context for the 'process_namespace' callback. + * Specifies a function to call on each namespace. + */ +struct ProcessNamespaceContext +{ + /** + * Function to call. + */ + GNUNET_FS_NamespaceInfoProcessor cb; + + /** + * Closure for 'cb'. + */ + void *cb_cls; +}; + + +/** + * Function called with a filename of a namespace. Reads the key and + * calls the callback. + * + * @param cls closure (struct ProcessNamespaceContext) + * @param filename complete filename (absolute path) + * @return GNUNET_OK to continue to iterate, + * GNUNET_SYSERR to abort iteration with error! + */ +static int +process_namespace (void *cls, const char *filename) +{ + struct ProcessNamespaceContext *pnc = cls; + struct GNUNET_CRYPTO_RsaPrivateKey *key; + struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded pk; + GNUNET_HashCode id; + const char *name; + const char *t; + + key = GNUNET_CRYPTO_rsa_key_create_from_file (filename); + if (key == NULL) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + _ + ("Failed to read namespace private key file `%s', deleting it!\n"), + filename); + if (0 != UNLINK (filename)) + GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", filename); + return GNUNET_OK; + } + GNUNET_CRYPTO_rsa_key_get_public (key, &pk); + GNUNET_CRYPTO_rsa_key_free (key); + GNUNET_CRYPTO_hash (&pk, sizeof (pk), &id); + name = filename; + while (NULL != (t = strstr (name, DIR_SEPARATOR_STR))) + name = t + 1; + pnc->cb (pnc->cb_cls, name, &id); + return GNUNET_OK; +} + + +/** + * Build a list of all available local (!) namespaces The returned + * names are only the nicknames since we only iterate over the local + * namespaces. + * + * @param h handle to the file sharing subsystem + * @param cb function to call on each known namespace + * @param cb_cls closure for cb + */ +void +GNUNET_FS_namespace_list (struct GNUNET_FS_Handle *h, + GNUNET_FS_NamespaceInfoProcessor cb, void *cb_cls) +{ + char *dn; + struct ProcessNamespaceContext ctx; + + dn = get_namespace_directory (h); + if (dn == NULL) + return; + ctx.cb = cb; + ctx.cb_cls = cb_cls; + GNUNET_DISK_directory_scan (dn, &process_namespace, &ctx); + GNUNET_free (dn); +} + + +/** + * Context for the SKS publication. + */ +struct GNUNET_FS_PublishSksContext +{ + + /** + * URI of the new entry in the namespace. + */ + struct GNUNET_FS_Uri *uri; + + /** + * Namespace update node to add to namespace on success (or to be + * deleted if publishing failed). + */ + struct NamespaceUpdateNode *nsn; + + /** + * Namespace we're publishing to. + */ + struct GNUNET_FS_Namespace *namespace; + + /** + * Handle to the datastore. + */ + struct GNUNET_DATASTORE_Handle *dsh; + + /** + * Function to call once we're done. + */ + GNUNET_FS_PublishContinuation cont; + + /** + * Closure for cont. + */ + void *cont_cls; + + /** + * Handle for our datastore request. + */ + struct GNUNET_DATASTORE_QueueEntry *dqe; +}; + + +/** + * Function called by the datastore API with + * the result from the PUT (SBlock) request. + * + * @param cls closure of type "struct GNUNET_FS_PublishSksContext*" + * @param success GNUNET_OK on success + * @param min_expiration minimum expiration time required for content to be stored + * @param msg error message (or NULL) + */ +static void +sb_put_cont (void *cls, int success, + struct GNUNET_TIME_Absolute min_expiration, + const char *msg) +{ + struct GNUNET_FS_PublishSksContext *psc = cls; + GNUNET_HashCode hc; + + psc->dqe = NULL; + if (GNUNET_OK != success) + { + if (NULL != psc->cont) + psc->cont (psc->cont_cls, NULL, msg); + GNUNET_FS_publish_sks_cancel (psc); + return; + } + if (NULL != psc->nsn) + { + /* FIXME: this can be done much more + * efficiently by simply appending to the + * file and overwriting the 4-byte header */ + if (psc->namespace->update_nodes == NULL) + read_update_information_graph (psc->namespace); + GNUNET_array_append (psc->namespace->update_nodes, + psc->namespace->update_node_count, psc->nsn); + if (psc->namespace->update_map != NULL) + { + GNUNET_CRYPTO_hash (psc->nsn->id, strlen (psc->nsn->id), &hc); + GNUNET_CONTAINER_multihashmap_put (psc->namespace->update_map, &hc, + psc->nsn, + GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE); + } + psc->nsn = NULL; + write_update_information_graph (psc->namespace); + } + if (NULL != psc->cont) + psc->cont (psc->cont_cls, psc->uri, NULL); + GNUNET_FS_publish_sks_cancel (psc); +} + + +/** + * Publish an SBlock on GNUnet. + * + * @param h handle to the file sharing subsystem + * @param namespace namespace to publish in + * @param identifier identifier to use + * @param update update identifier to use + * @param meta metadata to use + * @param uri URI to refer to in the SBlock + * @param bo block options + * @param options publication options + * @param cont continuation + * @param cont_cls closure for cont + * @return NULL on error ('cont' will still be called) + */ +struct GNUNET_FS_PublishSksContext * +GNUNET_FS_publish_sks (struct GNUNET_FS_Handle *h, + struct GNUNET_FS_Namespace *namespace, + const char *identifier, const char *update, + const struct GNUNET_CONTAINER_MetaData *meta, + const struct GNUNET_FS_Uri *uri, + const struct GNUNET_FS_BlockOptions *bo, + enum GNUNET_FS_PublishOptions options, + GNUNET_FS_PublishContinuation cont, void *cont_cls) +{ + struct GNUNET_FS_PublishSksContext *psc; + struct GNUNET_CRYPTO_AesSessionKey sk; + struct GNUNET_CRYPTO_AesInitializationVector iv; + struct GNUNET_FS_Uri *sks_uri; + char *uris; + size_t size; + size_t slen; + size_t nidlen; + size_t idlen; + ssize_t mdsize; + struct SBlock *sb; + struct SBlock *sb_enc; + char *dest; + struct GNUNET_CONTAINER_MetaData *mmeta; + GNUNET_HashCode key; /* hash of thisId = key */ + GNUNET_HashCode id; /* hash of hc = identifier */ + GNUNET_HashCode query; /* id ^ nsid = DB query */ + + if (NULL == meta) + mmeta = GNUNET_CONTAINER_meta_data_create (); + else + mmeta = GNUNET_CONTAINER_meta_data_duplicate (meta); + uris = GNUNET_FS_uri_to_string (uri); + slen = strlen (uris) + 1; + idlen = strlen (identifier); + if (update != NULL) + nidlen = strlen (update) + 1; + else + nidlen = 1; + mdsize = GNUNET_CONTAINER_meta_data_get_serialized_size (mmeta); + size = sizeof (struct SBlock) + slen + nidlen + mdsize; + if (size > MAX_SBLOCK_SIZE) + { + size = MAX_SBLOCK_SIZE; + mdsize = size - (sizeof (struct SBlock) + slen + nidlen); + } + sb = GNUNET_malloc (sizeof (struct SBlock) + size); + dest = (char *) &sb[1]; + if (update != NULL) + memcpy (dest, update, nidlen); + else + memset (dest, 0, 1); + dest += nidlen; + memcpy (dest, uris, slen); + GNUNET_free (uris); + dest += slen; + mdsize = + GNUNET_CONTAINER_meta_data_serialize (mmeta, &dest, mdsize, + GNUNET_CONTAINER_META_DATA_SERIALIZE_PART); + GNUNET_CONTAINER_meta_data_destroy (mmeta); + if (mdsize == -1) + { + GNUNET_break (0); + GNUNET_free (sb); + if (NULL != cont) + cont (cont_cls, NULL, _("Internal error.")); + return NULL; + } + size = sizeof (struct SBlock) + mdsize + slen + nidlen; + sb_enc = GNUNET_malloc (size); + GNUNET_CRYPTO_hash (identifier, idlen, &key); + GNUNET_CRYPTO_hash (&key, sizeof (GNUNET_HashCode), &id); + sks_uri = GNUNET_malloc (sizeof (struct GNUNET_FS_Uri)); + sks_uri->type = sks; + GNUNET_CRYPTO_rsa_key_get_public (namespace->key, &sb_enc->subspace); + GNUNET_CRYPTO_hash (&sb_enc->subspace, + sizeof (struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded), + &sks_uri->data.sks.namespace); + sks_uri->data.sks.identifier = GNUNET_strdup (identifier); + GNUNET_CRYPTO_hash_xor (&id, &sks_uri->data.sks.namespace, + &sb_enc->identifier); + GNUNET_CRYPTO_hash_to_aes_key (&key, &sk, &iv); + GNUNET_CRYPTO_aes_encrypt (&sb[1], size - sizeof (struct SBlock), &sk, &iv, + &sb_enc[1]); + sb_enc->purpose.purpose = htonl (GNUNET_SIGNATURE_PURPOSE_FS_SBLOCK); + sb_enc->purpose.size = + htonl (slen + mdsize + nidlen + sizeof (struct SBlock) - + sizeof (struct GNUNET_CRYPTO_RsaSignature)); + GNUNET_assert (GNUNET_OK == + GNUNET_CRYPTO_rsa_sign (namespace->key, &sb_enc->purpose, + &sb_enc->signature)); + psc = GNUNET_malloc (sizeof (struct GNUNET_FS_PublishSksContext)); + psc->uri = sks_uri; + psc->cont = cont; + psc->namespace = GNUNET_FS_namespace_dup (namespace); + psc->cont_cls = cont_cls; + if (0 != (options & GNUNET_FS_PUBLISH_OPTION_SIMULATE_ONLY)) + { + GNUNET_free (sb_enc); + GNUNET_free (sb); + sb_put_cont (psc, GNUNET_OK, GNUNET_TIME_UNIT_ZERO_ABS, NULL); + return NULL; + } + psc->dsh = GNUNET_DATASTORE_connect (h->cfg); + if (NULL == psc->dsh) + { + GNUNET_free (sb_enc); + GNUNET_free (sb); + sb_put_cont (psc, GNUNET_NO, GNUNET_TIME_UNIT_ZERO_ABS, _("Failed to connect to datastore.")); + return NULL; + } + GNUNET_CRYPTO_hash_xor (&sks_uri->data.sks.namespace, &id, &query); + if (NULL != update) + { + psc->nsn = GNUNET_malloc (sizeof (struct NamespaceUpdateNode)); + psc->nsn->id = GNUNET_strdup (identifier); + psc->nsn->update = GNUNET_strdup (update); + psc->nsn->md = GNUNET_CONTAINER_meta_data_duplicate (meta); + psc->nsn->uri = GNUNET_FS_uri_dup (uri); + } + psc->dqe = GNUNET_DATASTORE_put (psc->dsh, 0, &sb_enc->identifier, size, sb_enc, + GNUNET_BLOCK_TYPE_FS_SBLOCK, bo->content_priority, + bo->anonymity_level, bo->replication_level, + bo->expiration_time, -2, 1, + GNUNET_CONSTANTS_SERVICE_TIMEOUT, &sb_put_cont, psc); + GNUNET_free (sb); + GNUNET_free (sb_enc); + return psc; +} + + +/** + * Abort the SKS publishing operation. + * + * @param psc context of the operation to abort. + */ +void +GNUNET_FS_publish_sks_cancel (struct GNUNET_FS_PublishSksContext *psc) +{ + if (NULL != psc->dqe) + { + GNUNET_DATASTORE_cancel (psc->dqe); + psc->dqe = NULL; + } + if (NULL != psc->dsh) + { + GNUNET_DATASTORE_disconnect (psc->dsh, GNUNET_NO); + psc->dsh = NULL; + } + GNUNET_FS_namespace_delete (psc->namespace, GNUNET_NO); + GNUNET_FS_uri_destroy (psc->uri); + if (NULL != psc->nsn) + { + GNUNET_CONTAINER_meta_data_destroy (psc->nsn->md); + GNUNET_FS_uri_destroy (psc->nsn->uri); + GNUNET_free (psc->nsn->id); + GNUNET_free (psc->nsn->update); + GNUNET_free (psc->nsn); + } + GNUNET_free (psc); +} + + +/** + * Closure for 'process_update_node'. + */ +struct ProcessUpdateClosure +{ + /** + * Function to call for each node. + */ + GNUNET_FS_IdentifierProcessor ip; + + /** + * Closure for 'ip'. + */ + void *ip_cls; +}; + + +/** + * Call the iterator in the closure for each node. + * + * @param cls closure (of type 'struct ProcessUpdateClosure *') + * @param key current key code + * @param value value in the hash map (of type 'struct NamespaceUpdateNode *') + * @return GNUNET_YES if we should continue to + * iterate, + * GNUNET_NO if not. + */ +static int +process_update_node (void *cls, const GNUNET_HashCode * key, void *value) +{ + struct ProcessUpdateClosure *pc = cls; + struct NamespaceUpdateNode *nsn = value; + + pc->ip (pc->ip_cls, nsn->id, nsn->uri, nsn->md, nsn->update); + return GNUNET_YES; +} + + +/** + * Closure for 'find_trees'. + */ +struct FindTreeClosure +{ + /** + * Namespace we are operating on. + */ + struct GNUNET_FS_Namespace *namespace; + + /** + * Array with 'head's of TREEs. + */ + struct NamespaceUpdateNode **tree_array; + + /** + * Size of 'tree_array' + */ + unsigned int tree_array_size; + + /** + * Current generational ID used. + */ + unsigned int nug; + + /** + * Identifier for the current TREE, or UINT_MAX for none yet. + */ + unsigned int id; +}; + + +/** + * Find all nodes reachable from the current node (including the + * current node itself). If they are in no tree, add them to the + * current one. If they are the head of another tree, merge the + * trees. If they are in the middle of another tree, let them be. + * We can tell that a node is already in an tree by checking if + * its 'nug' field is set to the current 'nug' value. It is the + * head of an tree if it is in the 'tree_array' under its respective + * 'tree_id'. + * + * In short, we're trying to find the smallest number of tree to + * cover a directed graph. + * + * @param cls closure (of type 'struct FindTreeClosure') + * @param key current key code + * @param value value in the hash map + * @return GNUNET_YES if we should continue to + * iterate, + * GNUNET_NO if not. + */ +static int +find_trees (void *cls, const GNUNET_HashCode * key, void *value) +{ + struct FindTreeClosure *fc = cls; + struct NamespaceUpdateNode *nsn = value; + GNUNET_HashCode hc; + + if (nsn->nug == fc->nug) + { + if (nsn->tree_id == UINT_MAX) + return GNUNET_YES; /* circular */ + GNUNET_assert (nsn->tree_id < fc->tree_array_size); + if (fc->tree_array[nsn->tree_id] != nsn) + return GNUNET_YES; /* part of "another" (directed) TREE, + * and not root of it, end trace */ + if (nsn->tree_id == fc->id) + return GNUNET_YES; /* that's our own root (can this be?) */ + /* merge existing TREE, we have a root for both */ + fc->tree_array[nsn->tree_id] = NULL; + if (fc->id == UINT_MAX) + fc->id = nsn->tree_id; /* take over ID */ + } + else + { + nsn->nug = fc->nug; + nsn->tree_id = UINT_MAX; /* mark as undef */ + /* trace */ + GNUNET_CRYPTO_hash (nsn->update, strlen (nsn->update), &hc); + GNUNET_CONTAINER_multihashmap_get_multiple (fc->namespace->update_map, &hc, + &find_trees, fc); + } + return GNUNET_YES; +} + + +/** + * List all of the identifiers in the namespace for which we could + * produce an update. Namespace updates form a graph where each node + * has a name. Each node can have any number of URI/meta-data entries + * which can each be linked to other nodes. Cycles are possible. + * + * Calling this function with "next_id" NULL will cause the library to + * call "ip" with a root for each strongly connected component of the + * graph (a root being a node from which all other nodes in the Tree + * are reachable). + * + * Calling this function with "next_id" being the name of a node will + * cause the library to call "ip" with all children of the node. Note + * that cycles within the final tree are possible (including self-loops). + * I know, odd definition of a tree, but the GUI will display an actual + * tree (GtkTreeView), so that's what counts for the term here. + * + * @param namespace namespace to inspect for updateable content + * @param next_id ID to look for; use NULL to look for tree roots + * @param ip function to call on each updateable identifier + * @param ip_cls closure for ip + */ +void +GNUNET_FS_namespace_list_updateable (struct GNUNET_FS_Namespace *namespace, + const char *next_id, + GNUNET_FS_IdentifierProcessor ip, + void *ip_cls) +{ + unsigned int i; + unsigned int nug; + GNUNET_HashCode hc; + struct NamespaceUpdateNode *nsn; + struct ProcessUpdateClosure pc; + struct FindTreeClosure fc; + + if (namespace->update_nodes == NULL) + read_update_information_graph (namespace); + if (namespace->update_nodes == NULL) + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "No updateable nodes found for ID `%s'\n", next_id); + return; /* no nodes */ + } + if (namespace->update_map == NULL) + { + /* need to construct */ + namespace->update_map = + GNUNET_CONTAINER_multihashmap_create (2 + + 3 * namespace->update_node_count / + 4); + for (i = 0; i < namespace->update_node_count; i++) + { + nsn = namespace->update_nodes[i]; + GNUNET_CRYPTO_hash (nsn->id, strlen (nsn->id), &hc); + GNUNET_CONTAINER_multihashmap_put (namespace->update_map, &hc, nsn, + GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE); + } + } + if (next_id != NULL) + { + GNUNET_CRYPTO_hash (next_id, strlen (next_id), &hc); + pc.ip = ip; + pc.ip_cls = ip_cls; + GNUNET_CONTAINER_multihashmap_get_multiple (namespace->update_map, &hc, + &process_update_node, &pc); + return; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Calculating TREEs to find roots of update trees\n"); + /* Find heads of TREEs in update graph */ + nug = ++namespace->nug_gen; + fc.tree_array = NULL; + fc.tree_array_size = 0; + + for (i = 0; i < namespace->update_node_count; i++) + { + nsn = namespace->update_nodes[i]; + if (nsn->nug == nug) + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "TREE of node `%s' is %u\n", nsn->id, + nsn->nug); + continue; /* already placed in TREE */ + } + GNUNET_CRYPTO_hash (nsn->update, strlen (nsn->update), &hc); + nsn->nug = nug; + nsn->tree_id = UINT_MAX; + fc.id = UINT_MAX; + fc.nug = nug; + fc.namespace = namespace; + GNUNET_CONTAINER_multihashmap_get_multiple (namespace->update_map, &hc, + &find_trees, &fc); + if (fc.id == UINT_MAX) + { + /* start new TREE */ + for (fc.id = 0; fc.id < fc.tree_array_size; fc.id++) + { + if (fc.tree_array[fc.id] == NULL) + { + fc.tree_array[fc.id] = nsn; + nsn->tree_id = fc.id; + break; + } + } + if (fc.id == fc.tree_array_size) + { + GNUNET_array_append (fc.tree_array, fc.tree_array_size, nsn); + nsn->tree_id = fc.id; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Starting new TREE %u with node `%s'\n", nsn->tree_id, + nsn->id); + /* put all nodes with same identifier into this TREE */ + GNUNET_CRYPTO_hash (nsn->id, strlen (nsn->id), &hc); + fc.id = nsn->tree_id; + fc.nug = nug; + fc.namespace = namespace; + GNUNET_CONTAINER_multihashmap_get_multiple (namespace->update_map, &hc, + &find_trees, &fc); + } + else + { + /* make head of TREE "id" */ + fc.tree_array[fc.id] = nsn; + nsn->tree_id = fc.id; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "TREE of node `%s' is %u\n", nsn->id, + fc.id); + } + for (i = 0; i < fc.tree_array_size; i++) + { + nsn = fc.tree_array[i]; + if (NULL != nsn) + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Root of TREE %u is node `%s'\n", i, + nsn->id); + ip (ip_cls, nsn->id, nsn->uri, nsn->md, nsn->update); + } + } + GNUNET_array_grow (fc.tree_array, fc.tree_array_size, 0); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Done processing TREEs\n"); +} + + +/* end of fs_namespace.c */ diff --git a/src/fs/fs_namespace_advertise.c b/src/fs/fs_namespace_advertise.c new file mode 100644 index 0000000..e0226ca --- /dev/null +++ b/src/fs/fs_namespace_advertise.c @@ -0,0 +1,323 @@ +/* + This file is part of GNUnet + (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2012 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/fs_namespace_advertise.c + * @brief advertise namespaces (creating NBlocks) + * @author Christian Grothoff + */ +#include "platform.h" +#include "gnunet_constants.h" +#include "gnunet_signatures.h" +#include "gnunet_util_lib.h" +#include "gnunet_fs_service.h" +#include "fs_api.h" + + +/** + * Maximum legal size for an nblock. + */ +#define MAX_NBLOCK_SIZE (60 * 1024) + + +/** + * Context for advertising a namespace. + */ +struct GNUNET_FS_AdvertisementContext +{ + /** + * Function to call with the result. + */ + GNUNET_FS_PublishContinuation cont; + + /** + * Closure for cont. + */ + void *cont_cls; + + /** + * Datastore handle. + */ + struct GNUNET_DATASTORE_Handle *dsh; + + /** + * Our KSK URI. + */ + struct GNUNET_FS_Uri *ksk_uri; + + /** + * Plaintext. + */ + char *pt; + + /** + * NBlock to sign and store. + */ + struct NBlock *nb; + + /** + * The namespace. + */ + struct GNUNET_FS_Namespace *ns; + + /** + * Current datastore queue entry for advertising. + */ + struct GNUNET_DATASTORE_QueueEntry *dqe; + + /** + * Block options. + */ + struct GNUNET_FS_BlockOptions bo; + + /** + * Number of bytes of plaintext. + */ + size_t pt_size; + + /** + * Current keyword offset. + */ + unsigned int pos; +}; + + +// FIXME: I see no good reason why this should need to be done +// in a new task (anymore). Integrate with 'cancel' function below? +/** + * Disconnect from the datastore. + * + * @param cls datastore handle + * @param tc scheduler context + */ +static void +do_disconnect (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct GNUNET_DATASTORE_Handle *dsh = cls; + + GNUNET_DATASTORE_disconnect (dsh, GNUNET_NO); +} + + +/** + * Continuation called to notify client about result of the + * operation. + * + * @param cls closure (our struct GNUNET_FS_AdvertismentContext) + * @param success GNUNET_SYSERR on failure + * @param min_expiration minimum expiration time required for content to be stored + * @param msg NULL on success, otherwise an error message + */ +static void +advertisement_cont (void *cls, int success, + struct GNUNET_TIME_Absolute min_expiration, + const char *msg) +{ + struct GNUNET_FS_AdvertisementContext *ac = cls; + const char *keyword; + GNUNET_HashCode key; + GNUNET_HashCode query; + struct GNUNET_CRYPTO_AesSessionKey skey; + struct GNUNET_CRYPTO_AesInitializationVector iv; + struct GNUNET_CRYPTO_RsaPrivateKey *pk; + + ac->dqe = NULL; + if (GNUNET_SYSERR == success) + { + /* error! */ + (void) GNUNET_SCHEDULER_add_now (&do_disconnect, ac->dsh); + ac->dsh = NULL; + if (msg == NULL) + { + GNUNET_break (0); + msg = _("Unknown error"); + } + if (ac->cont != NULL) + { + ac->cont (ac->cont_cls, NULL, msg); + ac->cont = NULL; + } + GNUNET_FS_namespace_advertise_cancel (ac); + return; + } + if (ac->pos == ac->ksk_uri->data.ksk.keywordCount) + { + /* done! */ + (void) GNUNET_SCHEDULER_add_now (&do_disconnect, ac->dsh); + ac->dsh = NULL; + if (ac->cont != NULL) + { + ac->cont (ac->cont_cls, ac->ksk_uri, NULL); + ac->cont = NULL; + } + GNUNET_FS_namespace_advertise_cancel (ac); + return; + } + keyword = ac->ksk_uri->data.ksk.keywords[ac->pos++]; + /* first character of keyword indicates if it is + * mandatory or not -- ignore for hashing */ + GNUNET_CRYPTO_hash (&keyword[1], strlen (&keyword[1]), &key); + GNUNET_CRYPTO_hash_to_aes_key (&key, &skey, &iv); + GNUNET_CRYPTO_aes_encrypt (ac->pt, ac->pt_size, &skey, &iv, &ac->nb[1]); + GNUNET_break (GNUNET_OK == + GNUNET_CRYPTO_rsa_sign (ac->ns->key, &ac->nb->ns_purpose, + &ac->nb->ns_signature)); + pk = GNUNET_CRYPTO_rsa_key_create_from_hash (&key); + GNUNET_assert (pk != NULL); + GNUNET_CRYPTO_rsa_key_get_public (pk, &ac->nb->keyspace); + GNUNET_CRYPTO_hash (&ac->nb->keyspace, + sizeof (struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded), + &query); + GNUNET_break (GNUNET_OK == + GNUNET_CRYPTO_rsa_sign (pk, &ac->nb->ksk_purpose, + &ac->nb->ksk_signature)); + GNUNET_CRYPTO_rsa_key_free (pk); + ac->dqe = GNUNET_DATASTORE_put (ac->dsh, 0 /* no reservation */ , + &query, ac->pt_size + sizeof (struct NBlock), ac->nb, + GNUNET_BLOCK_TYPE_FS_NBLOCK, ac->bo.content_priority, + ac->bo.anonymity_level, ac->bo.replication_level, + ac->bo.expiration_time, -2, 1, + GNUNET_CONSTANTS_SERVICE_TIMEOUT, &advertisement_cont, + ac); +} + + +/** + * Publish an advertismement for a namespace. + * + * @param h handle to the file sharing subsystem + * @param ksk_uri keywords to use for advertisment + * @param namespace handle for the namespace that should be advertised + * @param meta meta-data for the namespace advertisement + * @param bo block options + * @param rootEntry name of the root of the namespace + * @param cont continuation + * @param cont_cls closure for cont + * @return NULL on error ('cont' is still called) + */ +struct GNUNET_FS_AdvertisementContext * +GNUNET_FS_namespace_advertise (struct GNUNET_FS_Handle *h, + struct GNUNET_FS_Uri *ksk_uri, + struct GNUNET_FS_Namespace *namespace, + const struct GNUNET_CONTAINER_MetaData *meta, + const struct GNUNET_FS_BlockOptions *bo, + const char *rootEntry, + GNUNET_FS_PublishContinuation cont, + void *cont_cls) +{ + size_t reslen; + size_t size; + ssize_t mdsize; + struct NBlock *nb; + char *mdst; + struct GNUNET_DATASTORE_Handle *dsh; + struct GNUNET_FS_AdvertisementContext *ctx; + char *pt; + + /* create advertisements */ + mdsize = GNUNET_CONTAINER_meta_data_get_serialized_size (meta); + if (-1 == mdsize) + { + cont (cont_cls, NULL, _("Failed to serialize meta data")); + return NULL; + } + reslen = strlen (rootEntry) + 1; + size = mdsize + sizeof (struct NBlock) + reslen; + if (size > MAX_NBLOCK_SIZE) + { + size = MAX_NBLOCK_SIZE; + mdsize = size - sizeof (struct NBlock) - reslen; + } + + pt = GNUNET_malloc (mdsize + reslen); + memcpy (pt, rootEntry, reslen); + mdst = &pt[reslen]; + mdsize = + GNUNET_CONTAINER_meta_data_serialize (meta, &mdst, mdsize, + GNUNET_CONTAINER_META_DATA_SERIALIZE_PART); + if (-1 == mdsize) + { + GNUNET_break (0); + GNUNET_free (pt); + cont (cont_cls, NULL, _("Failed to serialize meta data")); + return NULL; + } + size = mdsize + sizeof (struct NBlock) + reslen; + nb = GNUNET_malloc (size); + GNUNET_CRYPTO_rsa_key_get_public (namespace->key, &nb->subspace); + nb->ns_purpose.size = + htonl (mdsize + reslen + + sizeof (struct GNUNET_CRYPTO_RsaSignaturePurpose) + + sizeof (struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded)); + nb->ns_purpose.purpose = htonl (GNUNET_SIGNATURE_PURPOSE_FS_NBLOCK); + nb->ksk_purpose.size = + htonl (size - sizeof (struct GNUNET_CRYPTO_RsaSignature)); + nb->ksk_purpose.purpose = htonl (GNUNET_SIGNATURE_PURPOSE_FS_NBLOCK_KSIG); + dsh = GNUNET_DATASTORE_connect (h->cfg); + if (NULL == dsh) + { + GNUNET_free (nb); + GNUNET_free (pt); + cont (cont_cls, NULL, _("Failed to connect to datastore service")); + return NULL; + } + ctx = GNUNET_malloc (sizeof (struct GNUNET_FS_AdvertisementContext)); + ctx->cont = cont; + ctx->cont_cls = cont_cls; + ctx->dsh = dsh; + ctx->ksk_uri = GNUNET_FS_uri_dup (ksk_uri); + ctx->nb = nb; + ctx->pt = pt; + ctx->pt_size = mdsize + reslen; + ctx->ns = namespace; + ctx->ns->rc++; + ctx->bo = *bo; + advertisement_cont (ctx, GNUNET_OK, GNUNET_TIME_UNIT_ZERO_ABS, NULL); + return ctx; +} + + +/** + * Abort the namespace advertisement operation. + * + * @param ac context of the operation to abort. + */ +void +GNUNET_FS_namespace_advertise_cancel (struct GNUNET_FS_AdvertisementContext *ac) +{ + if (NULL != ac->dqe) + { + GNUNET_DATASTORE_cancel (ac->dqe); + ac->dqe = NULL; + } + if (NULL != ac->dsh) + { + GNUNET_DATASTORE_disconnect (ac->dsh, GNUNET_NO); + ac->dsh = NULL; + } + GNUNET_FS_uri_destroy (ac->ksk_uri); + GNUNET_free (ac->pt); + GNUNET_free (ac->nb); + GNUNET_FS_namespace_delete (ac->ns, GNUNET_NO); + GNUNET_free (ac); +} + + +/* end of fs_namespace_advertise.c */ diff --git a/src/fs/fs_publish.c b/src/fs/fs_publish.c new file mode 100644 index 0000000..1657e29 --- /dev/null +++ b/src/fs/fs_publish.c @@ -0,0 +1,1270 @@ +/* + This file is part of GNUnet. + (C) 2009, 2010 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/fs_publish.c + * @brief publish a file or directory in GNUnet + * @see https://gnunet.org/encoding + * @author Krista Bennett + * @author Christian Grothoff + */ + +#include "platform.h" +#include "gnunet_constants.h" +#include "gnunet_signatures.h" +#include "gnunet_util_lib.h" +#include "gnunet_fs_service.h" +#include "fs_api.h" +#include "fs_tree.h" + + +/** + * Fill in all of the generic fields for + * a publish event and call the callback. + * + * @param pi structure to fill in + * @param pc overall publishing context + * @param p file information for the file being published + * @param offset where in the file are we so far + * @return value returned from callback + */ +void * +GNUNET_FS_publish_make_status_ (struct GNUNET_FS_ProgressInfo *pi, + struct GNUNET_FS_PublishContext *pc, + const struct GNUNET_FS_FileInformation *p, + uint64_t offset) +{ + pi->value.publish.pc = pc; + pi->value.publish.fi = p; + pi->value.publish.cctx = p->client_info; + pi->value.publish.pctx = (NULL == p->dir) ? NULL : p->dir->client_info; + pi->value.publish.filename = p->filename; + pi->value.publish.size = + (p->is_directory == GNUNET_YES) ? p->data.dir.dir_size : p->data.file.file_size; + pi->value.publish.eta = + GNUNET_TIME_calculate_eta (p->start_time, offset, pi->value.publish.size); + pi->value.publish.completed = offset; + pi->value.publish.duration = + GNUNET_TIME_absolute_get_duration (p->start_time); + pi->value.publish.anonymity = p->bo.anonymity_level; + return pc->h->upcb (pc->h->upcb_cls, pi); +} + + +/** + * Cleanup the publish context, we're done with it. + * + * @param pc struct to clean up + */ +static void +publish_cleanup (struct GNUNET_FS_PublishContext *pc) +{ + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Cleaning up publish context (done!)\n"); + if (pc->fhc != NULL) + { + GNUNET_CRYPTO_hash_file_cancel (pc->fhc); + pc->fhc = NULL; + } + GNUNET_FS_file_information_destroy (pc->fi, NULL, NULL); + if (pc->namespace != NULL) + { + GNUNET_FS_namespace_delete (pc->namespace, GNUNET_NO); + pc->namespace = NULL; + } + GNUNET_free_non_null (pc->nid); + GNUNET_free_non_null (pc->nuid); + GNUNET_free_non_null (pc->serialization); + if (pc->dsh != NULL) + { + GNUNET_DATASTORE_disconnect (pc->dsh, GNUNET_NO); + pc->dsh = NULL; + } + if (pc->client != NULL) + { + GNUNET_CLIENT_disconnect (pc->client, GNUNET_NO); + pc->client = NULL; + } + GNUNET_assert (GNUNET_SCHEDULER_NO_TASK == pc->upload_task); + GNUNET_free (pc); +} + + +/** + * Function called by the datastore API with + * the result from the PUT request. + * + * @param cls the 'struct GNUNET_FS_PublishContext' + * @param success GNUNET_OK on success + * @param min_expiration minimum expiration time required for content to be stored + * @param msg error message (or NULL) + */ +static void +ds_put_cont (void *cls, int success, + struct GNUNET_TIME_Absolute min_expiration, + const char *msg) +{ + struct GNUNET_FS_PublishContext *pc = cls; + struct GNUNET_FS_ProgressInfo pi; + + pc->qre = NULL; + if (GNUNET_SYSERR == success) + { + GNUNET_asprintf (&pc->fi_pos->emsg, _("Publishing failed: %s"), msg); + pi.status = GNUNET_FS_STATUS_PUBLISH_ERROR; + pi.value.publish.eta = GNUNET_TIME_UNIT_FOREVER_REL; + pi.value.publish.specifics.error.message = pc->fi_pos->emsg; + pc->fi_pos->client_info = + GNUNET_FS_publish_make_status_ (&pi, pc, pc->fi_pos, 0); + if ((pc->fi_pos->is_directory != GNUNET_YES) && + (pc->fi_pos->filename != NULL) && + (pc->fi_pos->data.file.do_index == GNUNET_YES)) + { + /* run unindex to clean up */ + GNUNET_FS_unindex_start (pc->h, pc->fi_pos->filename, NULL); + } + } + GNUNET_assert (GNUNET_SCHEDULER_NO_TASK == pc->upload_task); + pc->upload_task = + GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_BACKGROUND, + &GNUNET_FS_publish_main_, pc); +} + + +/** + * Generate the callback that signals clients + * that a file (or directory) has been completely + * published. + * + * @param p the completed upload + * @param pc context of the publication + */ +static void +signal_publish_completion (struct GNUNET_FS_FileInformation *p, + struct GNUNET_FS_PublishContext *pc) +{ + struct GNUNET_FS_ProgressInfo pi; + + pi.status = GNUNET_FS_STATUS_PUBLISH_COMPLETED; + pi.value.publish.eta = GNUNET_TIME_UNIT_ZERO; + pi.value.publish.specifics.completed.chk_uri = p->chk_uri; + p->client_info = + GNUNET_FS_publish_make_status_ (&pi, pc, p, + GNUNET_ntohll (p->chk_uri->data. + chk.file_length)); +} + + +/** + * Generate the callback that signals clients + * that a file (or directory) has encountered + * a problem during publication. + * + * @param p the upload that had trouble + * @param pc context of the publication + * @param emsg error message + */ +static void +signal_publish_error (struct GNUNET_FS_FileInformation *p, + struct GNUNET_FS_PublishContext *pc, const char *emsg) +{ + struct GNUNET_FS_ProgressInfo pi; + + p->emsg = GNUNET_strdup (emsg); + pi.status = GNUNET_FS_STATUS_PUBLISH_ERROR; + pi.value.publish.eta = GNUNET_TIME_UNIT_FOREVER_REL; + pi.value.publish.specifics.error.message = emsg; + p->client_info = GNUNET_FS_publish_make_status_ (&pi, pc, p, 0); + if ((p->is_directory != GNUNET_YES) && (p->filename != NULL) && + (p->data.file.do_index == GNUNET_YES)) + { + /* run unindex to clean up */ + GNUNET_FS_unindex_start (pc->h, p->filename, NULL); + } + +} + + +/** + * Datastore returns from reservation cancel request. + * + * @param cls the 'struct GNUNET_FS_PublishContext' + * @param success success code (not used) + * @param min_expiration minimum expiration time required for content to be stored + * @param msg error message (typically NULL, not used) + */ +static void +finish_release_reserve (void *cls, int success, + struct GNUNET_TIME_Absolute min_expiration, + const char *msg) +{ + struct GNUNET_FS_PublishContext *pc = cls; + + pc->qre = NULL; + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Releasing reserve done!\n"); + signal_publish_completion (pc->fi, pc); + pc->all_done = GNUNET_YES; + GNUNET_FS_publish_sync_ (pc); +} + + +/** + * We've finished publishing the SBlock as part of a larger upload. + * Check the result and complete the larger upload. + * + * @param cls the "struct GNUNET_FS_PublishContext*" of the larger upload + * @param uri URI of the published SBlock + * @param emsg NULL on success, otherwise error message + */ +static void +publish_sblocks_cont (void *cls, const struct GNUNET_FS_Uri *uri, + const char *emsg) +{ + struct GNUNET_FS_PublishContext *pc = cls; + + pc->sks_pc = NULL; + if (NULL != emsg) + { + signal_publish_error (pc->fi, pc, emsg); + GNUNET_FS_publish_sync_ (pc); + return; + } + GNUNET_assert (pc->qre == NULL); + if ((pc->dsh != NULL) && (pc->rid != 0)) + { + pc->qre = + GNUNET_DATASTORE_release_reserve (pc->dsh, pc->rid, UINT_MAX, UINT_MAX, + GNUNET_TIME_UNIT_FOREVER_REL, + &finish_release_reserve, pc); + } + else + { + finish_release_reserve (pc, GNUNET_OK, GNUNET_TIME_UNIT_ZERO_ABS, NULL); + } +} + + +/** + * We are almost done publishing the structure, + * add SBlocks (if needed). + * + * @param pc overall upload data + */ +static void +publish_sblock (struct GNUNET_FS_PublishContext *pc) +{ + if (NULL != pc->namespace) + pc->sks_pc = GNUNET_FS_publish_sks (pc->h, pc->namespace, pc->nid, pc->nuid, + pc->fi->meta, pc->fi->chk_uri, &pc->fi->bo, + pc->options, &publish_sblocks_cont, pc); + else + publish_sblocks_cont (pc, NULL, NULL); +} + + +/** + * We've finished publishing a KBlock as part of a larger upload. + * Check the result and continue the larger upload. + * + * @param cls the "struct GNUNET_FS_PublishContext*" + * of the larger upload + * @param uri URI of the published blocks + * @param emsg NULL on success, otherwise error message + */ +static void +publish_kblocks_cont (void *cls, const struct GNUNET_FS_Uri *uri, + const char *emsg) +{ + struct GNUNET_FS_PublishContext *pc = cls; + struct GNUNET_FS_FileInformation *p = pc->fi_pos; + + pc->ksk_pc = NULL; + if (NULL != emsg) + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Error uploading KSK blocks: %s\n", + emsg); + signal_publish_error (p, pc, emsg); + GNUNET_FS_file_information_sync_ (p); + GNUNET_FS_publish_sync_ (pc); + GNUNET_assert (GNUNET_SCHEDULER_NO_TASK == pc->upload_task); + pc->upload_task = + GNUNET_SCHEDULER_add_with_priority + (GNUNET_SCHEDULER_PRIORITY_BACKGROUND, &GNUNET_FS_publish_main_, pc); + return; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "KSK blocks published, moving on to next file\n"); + if (NULL != p->dir) + signal_publish_completion (p, pc); + /* move on to next file */ + if (NULL != p->next) + pc->fi_pos = p->next; + else + pc->fi_pos = p->dir; + GNUNET_FS_publish_sync_ (pc); + GNUNET_assert (GNUNET_SCHEDULER_NO_TASK == pc->upload_task); + pc->upload_task = + GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_BACKGROUND, + &GNUNET_FS_publish_main_, pc); +} + + +/** + * Function called by the tree encoder to obtain + * a block of plaintext data (for the lowest level + * of the tree). + * + * @param cls our publishing context + * @param offset identifies which block to get + * @param max (maximum) number of bytes to get; returning + * fewer will also cause errors + * @param buf where to copy the plaintext buffer + * @param emsg location to store an error message (on error) + * @return number of bytes copied to buf, 0 on error + */ +static size_t +block_reader (void *cls, uint64_t offset, size_t max, void *buf, char **emsg) +{ + struct GNUNET_FS_PublishContext *pc = cls; + struct GNUNET_FS_FileInformation *p; + size_t pt_size; + const char *dd; + + p = pc->fi_pos; + if (p->is_directory == GNUNET_YES) + { + pt_size = GNUNET_MIN (max, p->data.dir.dir_size - offset); + dd = p->data.dir.dir_data; + memcpy (buf, &dd[offset], pt_size); + } + else + { + pt_size = GNUNET_MIN (max, p->data.file.file_size - offset); + if (pt_size == 0) + return 0; /* calling reader with pt_size==0 + * might free buf, so don't! */ + if (pt_size != + p->data.file.reader (p->data.file.reader_cls, offset, pt_size, buf, + emsg)) + return 0; + } + return pt_size; +} + + +/** + * The tree encoder has finished processing a + * file. Call it's finish method and deal with + * the final result. + * + * @param cls our publishing context + * @param tc scheduler's task context (not used) + */ +static void +encode_cont (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct GNUNET_FS_PublishContext *pc = cls; + struct GNUNET_FS_FileInformation *p; + struct GNUNET_FS_ProgressInfo pi; + char *emsg; + uint64_t flen; + + p = pc->fi_pos; + GNUNET_FS_tree_encoder_finish (p->te, &p->chk_uri, &emsg); + p->te = NULL; + GNUNET_FS_file_information_sync_ (p); + if (NULL != emsg) + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Error during tree walk: %s\n", emsg); + GNUNET_asprintf (&p->emsg, _("Publishing failed: %s"), emsg); + GNUNET_free (emsg); + pi.status = GNUNET_FS_STATUS_PUBLISH_ERROR; + pi.value.publish.eta = GNUNET_TIME_UNIT_FOREVER_REL; + pi.value.publish.specifics.error.message = p->emsg; + p->client_info = GNUNET_FS_publish_make_status_ (&pi, pc, p, 0); + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Finished with tree encoder\n"); + /* final progress event */ + flen = GNUNET_FS_uri_chk_get_file_size (p->chk_uri); + pi.status = GNUNET_FS_STATUS_PUBLISH_PROGRESS; + pi.value.publish.specifics.progress.data = NULL; + pi.value.publish.specifics.progress.offset = flen; + pi.value.publish.specifics.progress.data_len = 0; + pi.value.publish.specifics.progress.depth = GNUNET_FS_compute_depth (flen); + p->client_info = GNUNET_FS_publish_make_status_ (&pi, pc, p, flen); + + /* continue with main */ + GNUNET_assert (GNUNET_SCHEDULER_NO_TASK == pc->upload_task); + pc->upload_task = + GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_BACKGROUND, + &GNUNET_FS_publish_main_, pc); +} + + +/** + * Function called asking for the current (encoded) + * block to be processed. After processing the + * client should either call "GNUNET_FS_tree_encode_next" + * or (on error) "GNUNET_FS_tree_encode_finish". + * + * @param cls closure + * @param chk content hash key for the block + * @param offset offset of the block in the file + * @param depth depth of the block in the file, 0 for DBLOCK + * @param type type of the block (IBLOCK or DBLOCK) + * @param block the (encrypted) block + * @param block_size size of block (in bytes) + */ +static void +block_proc (void *cls, const struct ContentHashKey *chk, uint64_t offset, + unsigned int depth, enum GNUNET_BLOCK_Type type, const void *block, + uint16_t block_size) +{ + struct GNUNET_FS_PublishContext *pc = cls; + struct GNUNET_FS_FileInformation *p; + struct OnDemandBlock odb; + + p = pc->fi_pos; + if (NULL == pc->dsh) + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Waiting for datastore connection\n"); + GNUNET_assert (GNUNET_SCHEDULER_NO_TASK == pc->upload_task); + pc->upload_task = + GNUNET_SCHEDULER_add_with_priority + (GNUNET_SCHEDULER_PRIORITY_BACKGROUND, &GNUNET_FS_publish_main_, pc); + return; + } + + if ((p->is_directory != GNUNET_YES) && (GNUNET_YES == p->data.file.do_index) && + (type == GNUNET_BLOCK_TYPE_FS_DBLOCK)) + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Indexing block `%s' for offset %llu with index size %u\n", + GNUNET_h2s (&chk->query), (unsigned long long) offset, + sizeof (struct OnDemandBlock)); + odb.offset = GNUNET_htonll (offset); + odb.file_id = p->data.file.file_id; + GNUNET_assert (pc->qre == NULL); + pc->qre = + GNUNET_DATASTORE_put (pc->dsh, (p->is_directory == GNUNET_YES) ? 0 : pc->rid, + &chk->query, sizeof (struct OnDemandBlock), &odb, + GNUNET_BLOCK_TYPE_FS_ONDEMAND, + p->bo.content_priority, p->bo.anonymity_level, + p->bo.replication_level, p->bo.expiration_time, + -2, 1, GNUNET_CONSTANTS_SERVICE_TIMEOUT, + &ds_put_cont, pc); + return; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Publishing block `%s' for offset %llu with size %u\n", + GNUNET_h2s (&chk->query), (unsigned long long) offset, + (unsigned int) block_size); + GNUNET_assert (pc->qre == NULL); + pc->qre = + GNUNET_DATASTORE_put (pc->dsh, (p->is_directory == GNUNET_YES) ? 0 : pc->rid, + &chk->query, block_size, block, type, + p->bo.content_priority, p->bo.anonymity_level, + p->bo.replication_level, p->bo.expiration_time, -2, + 1, GNUNET_CONSTANTS_SERVICE_TIMEOUT, &ds_put_cont, + pc); +} + + +/** + * Function called with information about our + * progress in computing the tree encoding. + * + * @param cls closure + * @param offset where are we in the file + * @param pt_block plaintext of the currently processed block + * @param pt_size size of pt_block + * @param depth depth of the block in the tree, 0 for DBLOCK + */ +static void +progress_proc (void *cls, uint64_t offset, const void *pt_block, size_t pt_size, + unsigned int depth) +{ + struct GNUNET_FS_PublishContext *pc = cls; + struct GNUNET_FS_FileInformation *p; + struct GNUNET_FS_ProgressInfo pi; + + p = pc->fi_pos; + pi.status = GNUNET_FS_STATUS_PUBLISH_PROGRESS; + pi.value.publish.specifics.progress.data = pt_block; + pi.value.publish.specifics.progress.offset = offset; + pi.value.publish.specifics.progress.data_len = pt_size; + pi.value.publish.specifics.progress.depth = depth; + p->client_info = GNUNET_FS_publish_make_status_ (&pi, pc, p, offset); +} + + +/** + * We are uploading a file or directory; load (if necessary) the next + * block into memory, encrypt it and send it to the FS service. Then + * continue with the main task. + * + * @param pc overall upload data + */ +static void +publish_content (struct GNUNET_FS_PublishContext *pc) +{ + struct GNUNET_FS_FileInformation *p; + char *emsg; + struct GNUNET_FS_DirectoryBuilder *db; + struct GNUNET_FS_FileInformation *dirpos; + void *raw_data; + uint64_t size; + + p = pc->fi_pos; + GNUNET_assert (p != NULL); + if (NULL == p->te) + { + if (p->is_directory == GNUNET_YES) + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Creating directory\n"); + db = GNUNET_FS_directory_builder_create (p->meta); + dirpos = p->data.dir.entries; + while (NULL != dirpos) + { + if (dirpos->is_directory == GNUNET_YES) + { + raw_data = dirpos->data.dir.dir_data; + dirpos->data.dir.dir_data = NULL; + } + else + { + raw_data = NULL; + if ((dirpos->data.file.file_size < MAX_INLINE_SIZE) && + (dirpos->data.file.file_size > 0)) + { + raw_data = GNUNET_malloc (dirpos->data.file.file_size); + emsg = NULL; + if (dirpos->data.file.file_size != + dirpos->data.file.reader (dirpos->data.file.reader_cls, 0, + dirpos->data.file.file_size, raw_data, + &emsg)) + { + GNUNET_free_non_null (emsg); + GNUNET_free (raw_data); + raw_data = NULL; + } + } + } + GNUNET_FS_directory_builder_add (db, dirpos->chk_uri, dirpos->meta, + raw_data); + GNUNET_free_non_null (raw_data); + dirpos = dirpos->next; + } + GNUNET_free_non_null (p->data.dir.dir_data); + p->data.dir.dir_data = NULL; + p->data.dir.dir_size = 0; + GNUNET_FS_directory_builder_finish (db, &p->data.dir.dir_size, + &p->data.dir.dir_data); + GNUNET_FS_file_information_sync_ (p); + } + size = (p->is_directory == GNUNET_YES) ? p->data.dir.dir_size : p->data.file.file_size; + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Creating tree encoder\n"); + p->te = + GNUNET_FS_tree_encoder_create (pc->h, size, pc, &block_reader, + &block_proc, &progress_proc, + &encode_cont); + + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Processing next block from tree\n"); + GNUNET_FS_tree_encoder_next (p->te); +} + + +/** + * Process the response (or lack thereof) from + * the "fs" service to our 'start index' request. + * + * @param cls closure (of type "struct GNUNET_FS_PublishContext*"_) + * @param msg the response we got + */ +static void +process_index_start_response (void *cls, const struct GNUNET_MessageHeader *msg) +{ + struct GNUNET_FS_PublishContext *pc = cls; + struct GNUNET_FS_FileInformation *p; + const char *emsg; + uint16_t msize; + + GNUNET_CLIENT_disconnect (pc->client, GNUNET_NO); + pc->client = NULL; + p = pc->fi_pos; + if (msg == NULL) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _ + ("Can not index file `%s': %s. Will try to insert instead.\n"), + p->filename, + _("timeout on index-start request to `fs' service")); + p->data.file.do_index = GNUNET_NO; + GNUNET_FS_file_information_sync_ (p); + publish_content (pc); + return; + } + if (ntohs (msg->type) != GNUNET_MESSAGE_TYPE_FS_INDEX_START_OK) + { + msize = ntohs (msg->size); + emsg = (const char *) &msg[1]; + if ((msize <= sizeof (struct GNUNET_MessageHeader)) || + (emsg[msize - sizeof (struct GNUNET_MessageHeader) - 1] != '\0')) + emsg = gettext_noop ("unknown error"); + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _ + ("Can not index file `%s': %s. Will try to insert instead.\n"), + p->filename, gettext (emsg)); + p->data.file.do_index = GNUNET_NO; + GNUNET_FS_file_information_sync_ (p); + publish_content (pc); + return; + } + p->data.file.index_start_confirmed = GNUNET_YES; + /* success! continue with indexing */ + GNUNET_FS_file_information_sync_ (p); + publish_content (pc); +} + + +/** + * Function called once the hash computation over an + * indexed file has completed. + * + * @param cls closure, our publishing context + * @param res resulting hash, NULL on error + */ +static void +hash_for_index_cb (void *cls, const GNUNET_HashCode * res) +{ + struct GNUNET_FS_PublishContext *pc = cls; + struct GNUNET_FS_FileInformation *p; + struct IndexStartMessage *ism; + size_t slen; + struct GNUNET_CLIENT_Connection *client; + uint64_t dev; + uint64_t ino; + char *fn; + + pc->fhc = NULL; + p = pc->fi_pos; + if (NULL == res) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _ + ("Can not index file `%s': %s. Will try to insert instead.\n"), + p->filename, _("failed to compute hash")); + p->data.file.do_index = GNUNET_NO; + GNUNET_FS_file_information_sync_ (p); + publish_content (pc); + return; + } + if (GNUNET_YES == p->data.file.index_start_confirmed) + { + publish_content (pc); + return; + } + fn = GNUNET_STRINGS_filename_expand (p->filename); + GNUNET_assert (fn != NULL); + slen = strlen (fn) + 1; + if (slen >= + GNUNET_SERVER_MAX_MESSAGE_SIZE - sizeof (struct IndexStartMessage)) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _ + ("Can not index file `%s': %s. Will try to insert instead.\n"), + fn, _("filename too long")); + GNUNET_free (fn); + p->data.file.do_index = GNUNET_NO; + GNUNET_FS_file_information_sync_ (p); + publish_content (pc); + return; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Hash of indexed file `%s' is `%s'\n", + p->filename, GNUNET_h2s (res)); + if (0 != (pc->options & GNUNET_FS_PUBLISH_OPTION_SIMULATE_ONLY)) + { + p->data.file.file_id = *res; + p->data.file.have_hash = GNUNET_YES; + p->data.file.index_start_confirmed = GNUNET_YES; + GNUNET_FS_file_information_sync_ (p); + publish_content (pc); + GNUNET_free (fn); + return; + } + client = GNUNET_CLIENT_connect ("fs", pc->h->cfg); + if (NULL == client) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _ + ("Can not index file `%s': %s. Will try to insert instead.\n"), + p->filename, _("could not connect to `fs' service")); + p->data.file.do_index = GNUNET_NO; + publish_content (pc); + GNUNET_free (fn); + return; + } + if (p->data.file.have_hash != GNUNET_YES) + { + p->data.file.file_id = *res; + p->data.file.have_hash = GNUNET_YES; + GNUNET_FS_file_information_sync_ (p); + } + ism = GNUNET_malloc (sizeof (struct IndexStartMessage) + slen); + ism->header.size = htons (sizeof (struct IndexStartMessage) + slen); + ism->header.type = htons (GNUNET_MESSAGE_TYPE_FS_INDEX_START); + if (GNUNET_OK == GNUNET_DISK_file_get_identifiers (p->filename, &dev, &ino)) + { + ism->device = GNUNET_htonll (dev); + ism->inode = GNUNET_htonll (ino); + } + else + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + _("Failed to get file identifiers for `%s'\n"), p->filename); + } + ism->file_id = *res; + memcpy (&ism[1], fn, slen); + GNUNET_free (fn); + pc->client = client; + GNUNET_break (GNUNET_YES == + GNUNET_CLIENT_transmit_and_get_response (client, &ism->header, + GNUNET_TIME_UNIT_FOREVER_REL, + GNUNET_YES, + &process_index_start_response, + pc)); + GNUNET_free (ism); +} + + +/** + * Main function that performs the upload. + * + * @param cls "struct GNUNET_FS_PublishContext" identifies the upload + * @param tc task context + */ +void +GNUNET_FS_publish_main_ (void *cls, + const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct GNUNET_FS_PublishContext *pc = cls; + struct GNUNET_FS_ProgressInfo pi; + struct GNUNET_FS_FileInformation *p; + struct GNUNET_FS_Uri *loc; + char *fn; + + pc->upload_task = GNUNET_SCHEDULER_NO_TASK; + p = pc->fi_pos; + if (NULL == p) + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Publishing complete, now publishing SKS and KSK blocks.\n"); + /* upload of entire hierarchy complete, + * publish namespace entries */ + GNUNET_FS_publish_sync_ (pc); + publish_sblock (pc); + return; + } + /* find starting position */ + while ((p->is_directory == GNUNET_YES) && (NULL != p->data.dir.entries) && (NULL == p->emsg) + && (NULL == p->data.dir.entries->chk_uri)) + { + p = p->data.dir.entries; + pc->fi_pos = p; + GNUNET_FS_publish_sync_ (pc); + } + /* abort on error */ + if (NULL != p->emsg) + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Error uploading: %s\n", p->emsg); + /* error with current file, abort all + * related files as well! */ + while (NULL != p->dir) + { + fn = GNUNET_CONTAINER_meta_data_get_by_type (p->meta, + EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME); + p = p->dir; + if (fn != NULL) + { + GNUNET_asprintf (&p->emsg, _("Recursive upload failed at `%s': %s"), fn, + p->emsg); + GNUNET_free (fn); + } + else + { + GNUNET_asprintf (&p->emsg, _("Recursive upload failed: %s"), p->emsg); + } + pi.status = GNUNET_FS_STATUS_PUBLISH_ERROR; + pi.value.publish.eta = GNUNET_TIME_UNIT_FOREVER_REL; + pi.value.publish.specifics.error.message = p->emsg; + p->client_info = GNUNET_FS_publish_make_status_ (&pi, pc, p, 0); + } + pc->all_done = GNUNET_YES; + GNUNET_FS_publish_sync_ (pc); + return; + } + /* handle completion */ + if (NULL != p->chk_uri) + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "File upload complete, now publishing KSK blocks.\n"); + if (0 == p->bo.anonymity_level) + { + /* zero anonymity, box CHK URI in LOC URI */ + loc = + GNUNET_FS_uri_loc_create (p->chk_uri, pc->h->cfg, + p->bo.expiration_time); + GNUNET_FS_uri_destroy (p->chk_uri); + p->chk_uri = loc; + } + GNUNET_FS_publish_sync_ (pc); + /* upload of "p" complete, publish KBlocks! */ + if (p->keywords != NULL) + { + pc->ksk_pc = GNUNET_FS_publish_ksk (pc->h, p->keywords, p->meta, p->chk_uri, &p->bo, + pc->options, &publish_kblocks_cont, pc); + } + else + { + publish_kblocks_cont (pc, p->chk_uri, NULL); + } + return; + } + if ((p->is_directory != GNUNET_YES) && (p->data.file.do_index)) + { + if (NULL == p->filename) + { + p->data.file.do_index = GNUNET_NO; + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _ + ("Can not index file `%s': %s. Will try to insert instead.\n"), + "<no-name>", _("needs to be an actual file")); + GNUNET_FS_file_information_sync_ (p); + publish_content (pc); + return; + } + if (p->data.file.have_hash) + { + hash_for_index_cb (pc, &p->data.file.file_id); + } + else + { + p->start_time = GNUNET_TIME_absolute_get (); + pc->fhc = + GNUNET_CRYPTO_hash_file (GNUNET_SCHEDULER_PRIORITY_IDLE, p->filename, + HASHING_BLOCKSIZE, &hash_for_index_cb, pc); + } + return; + } + publish_content (pc); +} + + +/** + * Signal the FS's progress function that we are starting + * an upload. + * + * @param cls closure (of type "struct GNUNET_FS_PublishContext*") + * @param fi the entry in the publish-structure + * @param length length of the file or directory + * @param meta metadata for the file or directory (can be modified) + * @param uri pointer to the keywords that will be used for this entry (can be modified) + * @param bo block options + * @param do_index should we index? + * @param client_info pointer to client context set upon creation (can be modified) + * @return GNUNET_OK to continue (always) + */ +static int +fip_signal_start (void *cls, struct GNUNET_FS_FileInformation *fi, + uint64_t length, struct GNUNET_CONTAINER_MetaData *meta, + struct GNUNET_FS_Uri **uri, struct GNUNET_FS_BlockOptions *bo, + int *do_index, void **client_info) +{ + struct GNUNET_FS_PublishContext *pc = cls; + struct GNUNET_FS_ProgressInfo pi; + unsigned int kc; + uint64_t left; + + if (GNUNET_YES == pc->skip_next_fi_callback) + { + pc->skip_next_fi_callback = GNUNET_NO; + return GNUNET_OK; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Starting publish operation\n"); + if (*do_index) + { + /* space for on-demand blocks */ + pc->reserve_space += + ((length + DBLOCK_SIZE - + 1) / DBLOCK_SIZE) * sizeof (struct OnDemandBlock); + } + else + { + /* space for DBlocks */ + pc->reserve_space += length; + } + /* entries for IBlocks and DBlocks, space for IBlocks */ + left = length; + while (1) + { + left = (left + DBLOCK_SIZE - 1) / DBLOCK_SIZE; + pc->reserve_entries += left; + if (left <= 1) + break; + left = left * sizeof (struct ContentHashKey); + pc->reserve_space += left; + } + pc->reserve_entries++; + /* entries and space for keywords */ + if (NULL != *uri) + { + kc = GNUNET_FS_uri_ksk_get_keyword_count (*uri); + pc->reserve_entries += kc; + pc->reserve_space += GNUNET_SERVER_MAX_MESSAGE_SIZE * kc; + } + pi.status = GNUNET_FS_STATUS_PUBLISH_START; + *client_info = GNUNET_FS_publish_make_status_ (&pi, pc, fi, 0); + GNUNET_FS_file_information_sync_ (fi); + if (GNUNET_YES == GNUNET_FS_meta_data_test_for_directory (meta) + && (fi->dir != NULL)) + { + /* process entries in directory */ + pc->skip_next_fi_callback = GNUNET_YES; + GNUNET_FS_file_information_inspect (fi, &fip_signal_start, pc); + } + return GNUNET_OK; +} + + +/** + * Signal the FS's progress function that we are suspending + * an upload. + * + * @param cls closure (of type "struct GNUNET_FS_PublishContext*") + * @param fi the entry in the publish-structure + * @param length length of the file or directory + * @param meta metadata for the file or directory (can be modified) + * @param uri pointer to the keywords that will be used for this entry (can be modified) + * @param bo block options + * @param do_index should we index? + * @param client_info pointer to client context set upon creation (can be modified) + * @return GNUNET_OK to continue (always) + */ +static int +fip_signal_suspend (void *cls, struct GNUNET_FS_FileInformation *fi, + uint64_t length, struct GNUNET_CONTAINER_MetaData *meta, + struct GNUNET_FS_Uri **uri, + struct GNUNET_FS_BlockOptions *bo, int *do_index, + void **client_info) +{ + struct GNUNET_FS_PublishContext *pc = cls; + struct GNUNET_FS_ProgressInfo pi; + uint64_t off; + + if (GNUNET_YES == pc->skip_next_fi_callback) + { + pc->skip_next_fi_callback = GNUNET_NO; + return GNUNET_OK; + } + if (GNUNET_YES == GNUNET_FS_meta_data_test_for_directory (meta)) + { + /* process entries in directory */ + pc->skip_next_fi_callback = GNUNET_YES; + GNUNET_FS_file_information_inspect (fi, &fip_signal_suspend, pc); + } + if (NULL != pc->ksk_pc) + { + GNUNET_FS_publish_ksk_cancel (pc->ksk_pc); + pc->ksk_pc = NULL; + } + if (NULL != pc->sks_pc) + { + GNUNET_FS_publish_sks_cancel (pc->sks_pc); + pc->sks_pc = NULL; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Suspending publish operation\n"); + GNUNET_free_non_null (fi->serialization); + fi->serialization = NULL; + off = (fi->chk_uri == NULL) ? 0 : length; + pi.status = GNUNET_FS_STATUS_PUBLISH_SUSPEND; + GNUNET_break (NULL == GNUNET_FS_publish_make_status_ (&pi, pc, fi, off)); + *client_info = NULL; + if (NULL != pc->qre) + { + GNUNET_DATASTORE_cancel (pc->qre); + pc->qre = NULL; + } + if (NULL != pc->dsh) + { + GNUNET_DATASTORE_disconnect (pc->dsh, GNUNET_NO); + pc->dsh = NULL; + } + pc->rid = 0; + return GNUNET_OK; +} + + +/** + * Create SUSPEND event for the given publish operation + * and then clean up our state (without stop signal). + * + * @param cls the 'struct GNUNET_FS_PublishContext' to signal for + */ +void +GNUNET_FS_publish_signal_suspend_ (void *cls) +{ + struct GNUNET_FS_PublishContext *pc = cls; + + if (GNUNET_SCHEDULER_NO_TASK != pc->upload_task) + { + GNUNET_SCHEDULER_cancel (pc->upload_task); + pc->upload_task = GNUNET_SCHEDULER_NO_TASK; + } + GNUNET_FS_file_information_inspect (pc->fi, &fip_signal_suspend, pc); + GNUNET_FS_end_top (pc->h, pc->top); + pc->top = NULL; + publish_cleanup (pc); +} + + +/** + * We have gotten a reply for our space reservation request. + * Either fail (insufficient space) or start publishing for good. + * + * @param cls the 'struct GNUNET_FS_PublishContext*' + * @param success positive reservation ID on success + * @param min_expiration minimum expiration time required for content to be stored + * @param msg error message on error, otherwise NULL + */ +static void +finish_reserve (void *cls, int success, + struct GNUNET_TIME_Absolute min_expiration, + const char *msg) +{ + struct GNUNET_FS_PublishContext *pc = cls; + + pc->qre = NULL; + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Reservation complete (%d)!\n", success); + if ((msg != NULL) || (success <= 0)) + { + GNUNET_asprintf (&pc->fi->emsg, _("Insufficient space for publishing: %s"), + msg); + signal_publish_error (pc->fi, pc, pc->fi->emsg); + return; + } + pc->rid = success; + GNUNET_assert (GNUNET_SCHEDULER_NO_TASK == pc->upload_task); + pc->upload_task = + GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_BACKGROUND, + &GNUNET_FS_publish_main_, pc); +} + + +/** + * Publish a file or directory. + * + * @param h handle to the file sharing subsystem + * @param fi information about the file or directory structure to publish + * @param namespace namespace to publish the file in, NULL for no namespace + * @param nid identifier to use for the publishd content in the namespace + * (can be NULL, must be NULL if namespace is NULL) + * @param nuid update-identifier that will be used for future updates + * (can be NULL, must be NULL if namespace or nid is NULL) + * @param options options for the publication + * @return context that can be used to control the publish operation + */ +struct GNUNET_FS_PublishContext * +GNUNET_FS_publish_start (struct GNUNET_FS_Handle *h, + struct GNUNET_FS_FileInformation *fi, + struct GNUNET_FS_Namespace *namespace, const char *nid, + const char *nuid, + enum GNUNET_FS_PublishOptions options) +{ + struct GNUNET_FS_PublishContext *ret; + struct GNUNET_DATASTORE_Handle *dsh; + + GNUNET_assert (NULL != h); + if (0 == (options & GNUNET_FS_PUBLISH_OPTION_SIMULATE_ONLY)) + { + dsh = GNUNET_DATASTORE_connect (h->cfg); + if (NULL == dsh) + return NULL; + } + else + { + dsh = NULL; + } + ret = GNUNET_malloc (sizeof (struct GNUNET_FS_PublishContext)); + ret->dsh = dsh; + ret->h = h; + ret->fi = fi; + ret->namespace = namespace; + ret->options = options; + if (namespace != NULL) + { + namespace->rc++; + GNUNET_assert (NULL != nid); + ret->nid = GNUNET_strdup (nid); + if (NULL != nuid) + ret->nuid = GNUNET_strdup (nuid); + } + /* signal start */ + GNUNET_FS_file_information_inspect (ret->fi, &fip_signal_start, ret); + ret->fi_pos = ret->fi; + ret->top = GNUNET_FS_make_top (h, &GNUNET_FS_publish_signal_suspend_, ret); + GNUNET_FS_publish_sync_ (ret); + if (NULL != ret->dsh) + { + GNUNET_assert (NULL == ret->qre); + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + _ + ("Reserving space for %u entries and %llu bytes for publication\n"), + (unsigned int) ret->reserve_entries, + (unsigned long long) ret->reserve_space); + ret->qre = + GNUNET_DATASTORE_reserve (ret->dsh, ret->reserve_space, + ret->reserve_entries, UINT_MAX, UINT_MAX, + GNUNET_TIME_UNIT_FOREVER_REL, &finish_reserve, + ret); + } + else + { + GNUNET_assert (GNUNET_SCHEDULER_NO_TASK == ret->upload_task); + ret->upload_task = + GNUNET_SCHEDULER_add_with_priority + (GNUNET_SCHEDULER_PRIORITY_BACKGROUND, &GNUNET_FS_publish_main_, ret); + } + return ret; +} + + +/** + * Signal the FS's progress function that we are stopping + * an upload. + * + * @param cls closure (of type "struct GNUNET_FS_PublishContext*") + * @param fi the entry in the publish-structure + * @param length length of the file or directory + * @param meta metadata for the file or directory (can be modified) + * @param uri pointer to the keywords that will be used for this entry (can be modified) + * @param bo block options (can be modified) + * @param do_index should we index? + * @param client_info pointer to client context set upon creation (can be modified) + * @return GNUNET_OK to continue (always) + */ +static int +fip_signal_stop (void *cls, struct GNUNET_FS_FileInformation *fi, + uint64_t length, struct GNUNET_CONTAINER_MetaData *meta, + struct GNUNET_FS_Uri **uri, struct GNUNET_FS_BlockOptions *bo, + int *do_index, void **client_info) +{ + struct GNUNET_FS_PublishContext *pc = cls; + struct GNUNET_FS_ProgressInfo pi; + uint64_t off; + + if (GNUNET_YES == pc->skip_next_fi_callback) + { + pc->skip_next_fi_callback = GNUNET_NO; + return GNUNET_OK; + } + if (GNUNET_YES == GNUNET_FS_meta_data_test_for_directory (meta)) + { + /* process entries in directory first */ + pc->skip_next_fi_callback = GNUNET_YES; + GNUNET_FS_file_information_inspect (fi, &fip_signal_stop, pc); + } + if (fi->serialization != NULL) + { + GNUNET_FS_remove_sync_file_ (pc->h, GNUNET_FS_SYNC_PATH_FILE_INFO, + fi->serialization); + GNUNET_free (fi->serialization); + fi->serialization = NULL; + } + off = (fi->chk_uri == NULL) ? 0 : length; + pi.status = GNUNET_FS_STATUS_PUBLISH_STOPPED; + GNUNET_break (NULL == GNUNET_FS_publish_make_status_ (&pi, pc, fi, off)); + *client_info = NULL; + return GNUNET_OK; +} + + +/** + * Stop an upload. Will abort incomplete uploads (but + * not remove blocks that have already been publishd) or + * simply clean up the state for completed uploads. + * Must NOT be called from within the event callback! + * + * @param pc context for the upload to stop + */ +void +GNUNET_FS_publish_stop (struct GNUNET_FS_PublishContext *pc) +{ + struct GNUNET_FS_ProgressInfo pi; + uint64_t off; + + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Publish stop called\n"); + GNUNET_FS_end_top (pc->h, pc->top); + if (NULL != pc->ksk_pc) + { + GNUNET_FS_publish_ksk_cancel (pc->ksk_pc); + pc->ksk_pc = NULL; + } + if (NULL != pc->sks_pc) + { + GNUNET_FS_publish_sks_cancel (pc->sks_pc); + pc->sks_pc = NULL; + } + if (GNUNET_SCHEDULER_NO_TASK != pc->upload_task) + { + GNUNET_SCHEDULER_cancel (pc->upload_task); + pc->upload_task = GNUNET_SCHEDULER_NO_TASK; + } + pc->skip_next_fi_callback = GNUNET_YES; + GNUNET_FS_file_information_inspect (pc->fi, &fip_signal_stop, pc); + + if (pc->fi->serialization != NULL) + { + GNUNET_FS_remove_sync_file_ (pc->h, GNUNET_FS_SYNC_PATH_FILE_INFO, + pc->fi->serialization); + GNUNET_free (pc->fi->serialization); + pc->fi->serialization = NULL; + } + off = (pc->fi->chk_uri == NULL) ? 0 : GNUNET_ntohll (pc->fi->chk_uri->data.chk.file_length); + + if (pc->serialization != NULL) + { + GNUNET_FS_remove_sync_file_ (pc->h, GNUNET_FS_SYNC_PATH_MASTER_PUBLISH, + pc->serialization); + GNUNET_free (pc->serialization); + pc->serialization = NULL; + } + if (NULL != pc->qre) + { + GNUNET_DATASTORE_cancel (pc->qre); + pc->qre = NULL; + } + pi.status = GNUNET_FS_STATUS_PUBLISH_STOPPED; + GNUNET_break (NULL == GNUNET_FS_publish_make_status_ (&pi, pc, pc->fi, off)); + publish_cleanup (pc); +} + + + +/* end of fs_publish.c */ diff --git a/src/fs/fs_publish_ksk.c b/src/fs/fs_publish_ksk.c new file mode 100644 index 0000000..5119de4 --- /dev/null +++ b/src/fs/fs_publish_ksk.c @@ -0,0 +1,342 @@ +/* + This file is part of GNUnet. + (C) 2009, 2010, 2012 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/fs_publish_ksk.c + * @brief publish a URI under a keyword in GNUnet + * @see https://gnunet.org/encoding + * @author Krista Bennett + * @author Christian Grothoff + */ + +#include "platform.h" +#include "gnunet_constants.h" +#include "gnunet_signatures.h" +#include "gnunet_util_lib.h" +#include "gnunet_fs_service.h" +#include "fs_api.h" +#include "fs_tree.h" + + +/** + * Maximum legal size for a kblock. + */ +#define MAX_KBLOCK_SIZE (60 * 1024) + + +/** + * Context for the KSK publication. + */ +struct GNUNET_FS_PublishKskContext +{ + + /** + * Keywords to use. + */ + struct GNUNET_FS_Uri *ksk_uri; + + /** + * Global FS context. + */ + struct GNUNET_FS_Handle *h; + + /** + * The master block that we are sending + * (in plaintext), has "mdsize+slen" more + * bytes than the struct would suggest. + */ + struct KBlock *kb; + + /** + * Buffer of the same size as "kb" for + * the encrypted version. + */ + struct KBlock *cpy; + + /** + * Handle to the datastore, NULL if we are just + * simulating. + */ + struct GNUNET_DATASTORE_Handle *dsh; + + /** + * Handle to datastore PUT request. + */ + struct GNUNET_DATASTORE_QueueEntry *qre; + + /** + * Current task. + */ + GNUNET_SCHEDULER_TaskIdentifier ksk_task; + + /** + * Function to call once we're done. + */ + GNUNET_FS_PublishContinuation cont; + + /** + * Closure for cont. + */ + void *cont_cls; + + /** + * When should the KBlocks expire? + */ + struct GNUNET_FS_BlockOptions bo; + + /** + * Size of the serialized metadata. + */ + ssize_t mdsize; + + /** + * Size of the (CHK) URI as a string. + */ + size_t slen; + + /** + * Keyword that we are currently processing. + */ + unsigned int i; + +}; + + +/** + * Continuation of "GNUNET_FS_publish_ksk" that performs + * the actual publishing operation (iterating over all + * of the keywords). + * + * @param cls closure of type "struct PublishKskContext*" + * @param tc unused + */ +static void +publish_ksk_cont (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc); + + +/** + * Function called by the datastore API with + * the result from the PUT request. + * + * @param cls closure of type "struct GNUNET_FS_PublishKskContext*" + * @param success GNUNET_OK on success + * @param min_expiration minimum expiration time required for content to be stored + * @param msg error message (or NULL) + */ +static void +kb_put_cont (void *cls, int success, + struct GNUNET_TIME_Absolute min_expiration, + const char *msg) +{ + struct GNUNET_FS_PublishKskContext *pkc = cls; + + pkc->qre = NULL; + if (GNUNET_OK != success) + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "KBlock PUT operation failed: %s\n", msg); + pkc->cont (pkc->cont_cls, NULL, msg); + GNUNET_FS_publish_ksk_cancel (pkc); + return; + } + pkc->ksk_task = GNUNET_SCHEDULER_add_now (&publish_ksk_cont, pkc); +} + + +/** + * Continuation of "GNUNET_FS_publish_ksk" that performs the actual + * publishing operation (iterating over all of the keywords). + * + * @param cls closure of type "struct GNUNET_FS_PublishKskContext*" + * @param tc unused + */ +static void +publish_ksk_cont (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct GNUNET_FS_PublishKskContext *pkc = cls; + const char *keyword; + GNUNET_HashCode key; + GNUNET_HashCode query; + struct GNUNET_CRYPTO_AesSessionKey skey; + struct GNUNET_CRYPTO_AesInitializationVector iv; + struct GNUNET_CRYPTO_RsaPrivateKey *pk; + + pkc->ksk_task = GNUNET_SCHEDULER_NO_TASK; + if ((pkc->i == pkc->ksk_uri->data.ksk.keywordCount) || (NULL == pkc->dsh)) + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "KSK PUT operation complete\n"); + pkc->cont (pkc->cont_cls, pkc->ksk_uri, NULL); + GNUNET_FS_publish_ksk_cancel (pkc); + return; + } + keyword = pkc->ksk_uri->data.ksk.keywords[pkc->i++]; + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Publishing under keyword `%s'\n", + keyword); + /* first character of keyword indicates if it is + * mandatory or not -- ignore for hashing */ + GNUNET_CRYPTO_hash (&keyword[1], strlen (&keyword[1]), &key); + GNUNET_CRYPTO_hash_to_aes_key (&key, &skey, &iv); + GNUNET_CRYPTO_aes_encrypt (&pkc->kb[1], pkc->slen + pkc->mdsize, &skey, &iv, + &pkc->cpy[1]); + pk = GNUNET_CRYPTO_rsa_key_create_from_hash (&key); + GNUNET_assert (NULL != pk); + GNUNET_CRYPTO_rsa_key_get_public (pk, &pkc->cpy->keyspace); + GNUNET_CRYPTO_hash (&pkc->cpy->keyspace, + sizeof (struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded), + &query); + GNUNET_assert (GNUNET_OK == + GNUNET_CRYPTO_rsa_sign (pk, &pkc->cpy->purpose, + &pkc->cpy->signature)); + GNUNET_CRYPTO_rsa_key_free (pk); + pkc->qre = + GNUNET_DATASTORE_put (pkc->dsh, 0, &query, + pkc->mdsize + sizeof (struct KBlock) + pkc->slen, + pkc->cpy, GNUNET_BLOCK_TYPE_FS_KBLOCK, + pkc->bo.content_priority, pkc->bo.anonymity_level, + pkc->bo.replication_level, pkc->bo.expiration_time, + -2, 1, GNUNET_CONSTANTS_SERVICE_TIMEOUT, + &kb_put_cont, pkc); +} + + +/** + * Publish a CHK under various keywords on GNUnet. + * + * @param h handle to the file sharing subsystem + * @param ksk_uri keywords to use + * @param meta metadata to use + * @param uri URI to refer to in the KBlock + * @param bo per-block options + * @param options publication options + * @param cont continuation + * @param cont_cls closure for cont + * @return NULL on error ('cont' will still be called) + */ +struct GNUNET_FS_PublishKskContext * +GNUNET_FS_publish_ksk (struct GNUNET_FS_Handle *h, + const struct GNUNET_FS_Uri *ksk_uri, + const struct GNUNET_CONTAINER_MetaData *meta, + const struct GNUNET_FS_Uri *uri, + const struct GNUNET_FS_BlockOptions *bo, + enum GNUNET_FS_PublishOptions options, + GNUNET_FS_PublishContinuation cont, void *cont_cls) +{ + struct GNUNET_FS_PublishKskContext *pkc; + char *uris; + size_t size; + char *kbe; + char *sptr; + + GNUNET_assert (NULL != uri); + pkc = GNUNET_malloc (sizeof (struct GNUNET_FS_PublishKskContext)); + pkc->h = h; + pkc->bo = *bo; + pkc->cont = cont; + pkc->cont_cls = cont_cls; + if (0 == (options & GNUNET_FS_PUBLISH_OPTION_SIMULATE_ONLY)) + { + pkc->dsh = GNUNET_DATASTORE_connect (h->cfg); + if (NULL == pkc->dsh) + { + cont (cont_cls, NULL, _("Could not connect to datastore.")); + GNUNET_free (pkc); + return NULL; + } + } + if (meta == NULL) + pkc->mdsize = 0; + else + pkc->mdsize = GNUNET_CONTAINER_meta_data_get_serialized_size (meta); + GNUNET_assert (pkc->mdsize >= 0); + uris = GNUNET_FS_uri_to_string (uri); + pkc->slen = strlen (uris) + 1; + size = pkc->mdsize + sizeof (struct KBlock) + pkc->slen; + if (size > MAX_KBLOCK_SIZE) + { + size = MAX_KBLOCK_SIZE; + pkc->mdsize = size - sizeof (struct KBlock) - pkc->slen; + } + pkc->kb = GNUNET_malloc (size); + kbe = (char *) &pkc->kb[1]; + memcpy (kbe, uris, pkc->slen); + GNUNET_free (uris); + sptr = &kbe[pkc->slen]; + if (meta != NULL) + pkc->mdsize = + GNUNET_CONTAINER_meta_data_serialize (meta, &sptr, pkc->mdsize, + GNUNET_CONTAINER_META_DATA_SERIALIZE_PART); + if (-1 == pkc->mdsize) + { + GNUNET_break (0); + GNUNET_free (pkc->kb); + if (pkc->dsh != NULL) + { + GNUNET_DATASTORE_disconnect (pkc->dsh, GNUNET_NO); + pkc->dsh = NULL; + } + GNUNET_free (pkc); + cont (cont_cls, NULL, _("Internal error.")); + return NULL; + } + size = sizeof (struct KBlock) + pkc->slen + pkc->mdsize; + + pkc->cpy = GNUNET_malloc (size); + pkc->cpy->purpose.size = + htonl (sizeof (struct GNUNET_CRYPTO_RsaSignaturePurpose) + + sizeof (struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded) + + pkc->mdsize + pkc->slen); + pkc->cpy->purpose.purpose = htonl (GNUNET_SIGNATURE_PURPOSE_FS_KBLOCK); + pkc->ksk_uri = GNUNET_FS_uri_dup (ksk_uri); + pkc->ksk_task = GNUNET_SCHEDULER_add_now (&publish_ksk_cont, pkc); + return pkc; +} + + +/** + * Abort the KSK publishing operation. + * + * @param pkc context of the operation to abort. + */ +void +GNUNET_FS_publish_ksk_cancel (struct GNUNET_FS_PublishKskContext *pkc) +{ + if (GNUNET_SCHEDULER_NO_TASK != pkc->ksk_task) + { + GNUNET_SCHEDULER_cancel (pkc->ksk_task); + pkc->ksk_task = GNUNET_SCHEDULER_NO_TASK; + } + if (NULL != pkc->qre) + { + GNUNET_DATASTORE_cancel (pkc->qre); + pkc->qre = NULL; + } + if (NULL != pkc->dsh) + { + GNUNET_DATASTORE_disconnect (pkc->dsh, GNUNET_NO); + pkc->dsh = NULL; + } + GNUNET_free (pkc->cpy); + GNUNET_free (pkc->kb); + GNUNET_FS_uri_destroy (pkc->ksk_uri); + GNUNET_free (pkc); +} + + +/* end of fs_publish_ksk.c */ diff --git a/src/fs/fs_search.c b/src/fs/fs_search.c new file mode 100644 index 0000000..a163d97 --- /dev/null +++ b/src/fs/fs_search.c @@ -0,0 +1,1548 @@ +/* + This file is part of GNUnet. + (C) 2001-2006, 2008-2011 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/fs_search.c + * @brief Helper functions for searching. + * @author Christian Grothoff + */ + +#include "platform.h" +#include "gnunet_constants.h" +#include "gnunet_fs_service.h" +#include "gnunet_protocols.h" +#include "fs_api.h" + +#define DEBUG_SEARCH GNUNET_EXTRA_LOGGING + +/** + * Number of availability trials we perform per search result. + */ +#define AVAILABILITY_TRIALS_MAX 8 + +/** + * Fill in all of the generic fields for a search event and + * call the callback. + * + * @param pi structure to fill in + * @param sc overall search context + * @return value returned by the callback + */ +void * +GNUNET_FS_search_make_status_ (struct GNUNET_FS_ProgressInfo *pi, + struct GNUNET_FS_SearchContext *sc) +{ + void *ret; + + pi->value.search.sc = sc; + pi->value.search.cctx = sc->client_info; + pi->value.search.pctx = + (sc->psearch_result == NULL) ? NULL : sc->psearch_result->client_info; + pi->value.search.query = sc->uri; + pi->value.search.duration = + GNUNET_TIME_absolute_get_duration (sc->start_time); + pi->value.search.anonymity = sc->anonymity; + ret = sc->h->upcb (sc->h->upcb_cls, pi); + return ret; +} + + +/** + * Check if the given result is identical + * to the given URI. + * + * @param cls points to the URI we check against + * @param key not used + * @param value a "struct GNUNET_FS_SearchResult" who's URI we + * should compare with + * @return GNUNET_SYSERR if the result is present, + * GNUNET_OK otherwise + */ +static int +test_result_present (void *cls, const GNUNET_HashCode * key, void *value) +{ + const struct GNUNET_FS_Uri *uri = cls; + struct GNUNET_FS_SearchResult *sr = value; + + if (GNUNET_FS_uri_test_equal (uri, sr->uri)) + return GNUNET_SYSERR; + return GNUNET_OK; +} + + +/** + * We've found a new CHK result. Let the client + * know about it. + * + * @param sc the search context + * @param sr the specific result + */ +static void +notify_client_chk_result (struct GNUNET_FS_SearchContext *sc, + struct GNUNET_FS_SearchResult *sr) +{ + struct GNUNET_FS_ProgressInfo pi; + + pi.status = GNUNET_FS_STATUS_SEARCH_RESULT; + pi.value.search.specifics.result.meta = sr->meta; + pi.value.search.specifics.result.uri = sr->uri; + pi.value.search.specifics.result.result = sr; + pi.value.search.specifics.result.applicability_rank = sr->optional_support; + sr->client_info = GNUNET_FS_search_make_status_ (&pi, sc); +} + + +/** + * We've found new information about an existing CHK result. Let the + * client know about it. + * + * @param sc the search context + * @param sr the specific result + */ +static void +notify_client_chk_update (struct GNUNET_FS_SearchContext *sc, + struct GNUNET_FS_SearchResult *sr) +{ + struct GNUNET_FS_ProgressInfo pi; + + pi.status = GNUNET_FS_STATUS_SEARCH_UPDATE; + pi.value.search.specifics.update.cctx = sr->client_info; + pi.value.search.specifics.update.meta = sr->meta; + pi.value.search.specifics.update.uri = sr->uri; + pi.value.search.specifics.update.availability_rank = + 2 * sr->availability_success - sr->availability_trials; + pi.value.search.specifics.update.availability_certainty = + sr->availability_trials; + pi.value.search.specifics.update.applicability_rank = sr->optional_support; + sr->client_info = GNUNET_FS_search_make_status_ (&pi, sc); +} + + +/** + * Context for "get_result_present". + */ +struct GetResultContext +{ + /** + * The URI we're looking for. + */ + const struct GNUNET_FS_Uri *uri; + + /** + * Where to store a pointer to the search + * result struct if we found a match. + */ + struct GNUNET_FS_SearchResult *sr; +}; + + +/** + * Check if the given result is identical to the given URI and if so + * return it. + * + * @param cls a "struct GetResultContext" + * @param key not used + * @param value a "struct GNUNET_FS_SearchResult" who's URI we + * should compare with + * @return GNUNET_OK + */ +static int +get_result_present (void *cls, const GNUNET_HashCode * key, void *value) +{ + struct GetResultContext *grc = cls; + struct GNUNET_FS_SearchResult *sr = value; + + if (GNUNET_FS_uri_test_equal (grc->uri, sr->uri)) + grc->sr = sr; + return GNUNET_OK; +} + + +/** + * Signal result of last probe to client and then schedule next + * probe. + */ +static void +signal_probe_result (struct GNUNET_FS_SearchResult *sr) +{ + struct GNUNET_FS_ProgressInfo pi; + + pi.status = GNUNET_FS_STATUS_SEARCH_START; + pi.value.search.specifics.update.cctx = sr->client_info; + pi.value.search.specifics.update.meta = sr->meta; + pi.value.search.specifics.update.uri = sr->uri; + pi.value.search.specifics.update.availability_rank = sr->availability_success; + pi.value.search.specifics.update.availability_certainty = + sr->availability_trials; + pi.value.search.specifics.update.applicability_rank = sr->optional_support; + sr->sc->client_info = GNUNET_FS_search_make_status_ (&pi, sr->sc); + GNUNET_FS_search_start_probe_ (sr); +} + + +/** + * Handle the case where we have failed to receive a response for our probe. + * + * @param cls our 'struct GNUNET_FS_SearchResult*' + * @param tc scheduler context + */ +static void +probe_failure_handler (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct GNUNET_FS_SearchResult *sr = cls; + + sr->availability_trials++; + GNUNET_FS_search_result_sync_ (sr); + signal_probe_result (sr); +} + + +/** + * Handle the case where we have gotten a response for our probe. + * + * @param cls our 'struct GNUNET_FS_SearchResult*' + * @param tc scheduler context + */ +static void +probe_success_handler (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct GNUNET_FS_SearchResult *sr = cls; + + sr->availability_trials++; + sr->availability_success++; + GNUNET_FS_search_result_sync_ (sr); + signal_probe_result (sr); +} + + +/** + * Notification of FS that a search probe has made progress. + * This function is used INSTEAD of the client's event handler + * for downloads where the GNUNET_FS_DOWNLOAD_IS_PROBE flag is set. + * + * @param cls closure, always NULL (!), actual closure + * is in the client-context of the info struct + * @param info details about the event, specifying the event type + * and various bits about the event + * @return client-context (for the next progress call + * for this operation; should be set to NULL for + * SUSPEND and STOPPED events). The value returned + * will be passed to future callbacks in the respective + * field in the GNUNET_FS_ProgressInfo struct. + */ +void * +GNUNET_FS_search_probe_progress_ (void *cls, + const struct GNUNET_FS_ProgressInfo *info) +{ + struct GNUNET_FS_SearchResult *sr = info->value.download.cctx; + struct GNUNET_TIME_Relative dur; + + switch (info->status) + { + case GNUNET_FS_STATUS_DOWNLOAD_START: + /* ignore */ + break; + case GNUNET_FS_STATUS_DOWNLOAD_RESUME: + /* probes should never be resumed */ + GNUNET_assert (0); + break; + case GNUNET_FS_STATUS_DOWNLOAD_SUSPEND: + /* probes should never be suspended */ + GNUNET_break (0); + break; + case GNUNET_FS_STATUS_DOWNLOAD_PROGRESS: + /* ignore */ + break; + case GNUNET_FS_STATUS_DOWNLOAD_ERROR: + if (sr->probe_cancel_task != GNUNET_SCHEDULER_NO_TASK) + { + GNUNET_SCHEDULER_cancel (sr->probe_cancel_task); + sr->probe_cancel_task = GNUNET_SCHEDULER_NO_TASK; + } + sr->probe_cancel_task = + GNUNET_SCHEDULER_add_delayed (sr->remaining_probe_time, + &probe_failure_handler, sr); + break; + case GNUNET_FS_STATUS_DOWNLOAD_COMPLETED: + if (sr->probe_cancel_task != GNUNET_SCHEDULER_NO_TASK) + { + GNUNET_SCHEDULER_cancel (sr->probe_cancel_task); + sr->probe_cancel_task = GNUNET_SCHEDULER_NO_TASK; + } + sr->probe_cancel_task = + GNUNET_SCHEDULER_add_delayed (sr->remaining_probe_time, + &probe_success_handler, sr); + break; + case GNUNET_FS_STATUS_DOWNLOAD_STOPPED: + if (sr->probe_cancel_task != GNUNET_SCHEDULER_NO_TASK) + { + GNUNET_SCHEDULER_cancel (sr->probe_cancel_task); + sr->probe_cancel_task = GNUNET_SCHEDULER_NO_TASK; + } + sr = NULL; + break; + case GNUNET_FS_STATUS_DOWNLOAD_ACTIVE: + GNUNET_assert (sr->probe_cancel_task == GNUNET_SCHEDULER_NO_TASK); + sr->probe_active_time = GNUNET_TIME_absolute_get (); + sr->probe_cancel_task = + GNUNET_SCHEDULER_add_delayed (sr->remaining_probe_time, + &probe_failure_handler, sr); + break; + case GNUNET_FS_STATUS_DOWNLOAD_INACTIVE: + if (sr->probe_cancel_task != GNUNET_SCHEDULER_NO_TASK) + { + GNUNET_SCHEDULER_cancel (sr->probe_cancel_task); + sr->probe_cancel_task = GNUNET_SCHEDULER_NO_TASK; + } + dur = GNUNET_TIME_absolute_get_duration (sr->probe_active_time); + sr->remaining_probe_time = + GNUNET_TIME_relative_subtract (sr->remaining_probe_time, dur); + GNUNET_FS_search_result_sync_ (sr); + break; + default: + GNUNET_break (0); + return NULL; + } + return sr; +} + + +/** + * Start download probes for the given search result. + * + * @param sr the search result + */ +void +GNUNET_FS_search_start_probe_ (struct GNUNET_FS_SearchResult *sr) +{ + uint64_t off; + uint64_t len; + + if (sr->probe_ctx != NULL) + return; + if (sr->download != NULL) + return; + if (0 == (sr->sc->h->flags & GNUNET_FS_FLAGS_DO_PROBES)) + return; + if (sr->availability_trials > AVAILABILITY_TRIALS_MAX) + return; + len = GNUNET_FS_uri_chk_get_file_size (sr->uri); + if (len == 0) + return; + if ((len <= DBLOCK_SIZE) && (sr->availability_success > 0)) + return; + off = len / DBLOCK_SIZE; + if (off > 0) + off = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, off); + off *= DBLOCK_SIZE; + if (len - off < DBLOCK_SIZE) + len = len - off; + else + len = DBLOCK_SIZE; + sr->remaining_probe_time = + GNUNET_TIME_relative_multiply (sr->sc->h->avg_block_latency, + 2 * (1 + sr->availability_trials)); + sr->probe_ctx = + GNUNET_FS_download_start (sr->sc->h, sr->uri, sr->meta, NULL, NULL, off, + len, sr->sc->anonymity, + GNUNET_FS_DOWNLOAD_NO_TEMPORARIES | + GNUNET_FS_DOWNLOAD_IS_PROBE, sr, NULL); +} + + +/** + * We have received a KSK result. Check how it fits in with the + * overall query and notify the client accordingly. + * + * @param sc context for the overall query + * @param ent entry for the specific keyword + * @param uri the URI that was found + * @param meta metadata associated with the URI + * under the "ent" keyword + */ +static void +process_ksk_result (struct GNUNET_FS_SearchContext *sc, + struct SearchRequestEntry *ent, + const struct GNUNET_FS_Uri *uri, + const struct GNUNET_CONTAINER_MetaData *meta) +{ + GNUNET_HashCode key; + struct GNUNET_FS_SearchResult *sr; + struct GetResultContext grc; + int is_new; + unsigned int koff; + + /* check if new */ + GNUNET_assert (NULL != sc); + GNUNET_FS_uri_to_key (uri, &key); + if (GNUNET_SYSERR == + GNUNET_CONTAINER_multihashmap_get_multiple (ent->results, &key, + &test_result_present, + (void *) uri)) + return; /* duplicate result */ + /* try to find search result in master map */ + grc.sr = NULL; + grc.uri = uri; + GNUNET_CONTAINER_multihashmap_get_multiple (sc->master_result_map, &key, + &get_result_present, &grc); + sr = grc.sr; + is_new = (NULL == sr) || (sr->mandatory_missing > 0); + if (NULL == sr) + { + sr = GNUNET_malloc (sizeof (struct GNUNET_FS_SearchResult)); + sr->sc = sc; + sr->uri = GNUNET_FS_uri_dup (uri); + sr->meta = GNUNET_CONTAINER_meta_data_duplicate (meta); + sr->mandatory_missing = sc->mandatory_count; + sr->key = key; + sr->keyword_bitmap = GNUNET_malloc ((sc->uri->data.ksk.keywordCount + 7) / 8); /* round up, count bits */ + GNUNET_CONTAINER_multihashmap_put (sc->master_result_map, &key, sr, + GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE); + } + else + { + GNUNET_CONTAINER_meta_data_merge (sr->meta, meta); + } + koff = ent - sc->requests; + GNUNET_assert ( (ent >= sc->requests) && (koff < sc->uri->data.ksk.keywordCount)); + sr->keyword_bitmap[koff / 8] |= (1 << (koff % 8)); + /* check if mandatory satisfied */ + if (ent->mandatory) + sr->mandatory_missing--; + else + sr->optional_support++; + if (0 != sr->mandatory_missing) + return; + if (is_new) + notify_client_chk_result (sc, sr); + else + notify_client_chk_update (sc, sr); + GNUNET_FS_search_result_sync_ (sr); + GNUNET_FS_search_start_probe_ (sr); +} + + +/** + * Start search for content, internal API. + * + * @param h handle to the file sharing subsystem + * @param uri specifies the search parameters; can be + * a KSK URI or an SKS URI. + * @param anonymity desired level of anonymity + * @param options options for the search + * @param cctx client context + * @param psearch parent search result (for namespace update searches) + * @return context that can be used to control the search + */ +static struct GNUNET_FS_SearchContext * +search_start (struct GNUNET_FS_Handle *h, const struct GNUNET_FS_Uri *uri, + uint32_t anonymity, enum GNUNET_FS_SearchOptions options, + void *cctx, struct GNUNET_FS_SearchResult *psearch); + + +/** + * We have received an SKS result. Start searching for updates and + * notify the client if it is a new result. + * + * @param sc context for the overall query + * @param id_update identifier for updates, NULL for none + * @param uri the URI that was found + * @param meta metadata associated with the URI + */ +static void +process_sks_result (struct GNUNET_FS_SearchContext *sc, const char *id_update, + const struct GNUNET_FS_Uri *uri, + const struct GNUNET_CONTAINER_MetaData *meta) +{ + struct GNUNET_FS_Uri uu; + GNUNET_HashCode key; + struct GNUNET_FS_SearchResult *sr; + + /* check if new */ + GNUNET_assert (NULL != sc); + GNUNET_FS_uri_to_key (uri, &key); + GNUNET_CRYPTO_hash_xor (&uri->data.chk.chk.key, &uri->data.chk.chk.query, + &key); + if (GNUNET_SYSERR == + GNUNET_CONTAINER_multihashmap_get_multiple (sc->master_result_map, &key, + &test_result_present, + (void *) uri)) + return; /* duplicate result */ + sr = GNUNET_malloc (sizeof (struct GNUNET_FS_SearchResult)); + sr->sc = sc; + sr->uri = GNUNET_FS_uri_dup (uri); + sr->meta = GNUNET_CONTAINER_meta_data_duplicate (meta); + sr->key = key; + GNUNET_CONTAINER_multihashmap_put (sc->master_result_map, &key, sr, + GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE); + GNUNET_FS_search_result_sync_ (sr); + GNUNET_FS_search_start_probe_ (sr); + /* notify client */ + notify_client_chk_result (sc, sr); + /* search for updates */ + if (strlen (id_update) == 0) + return; /* no updates */ + uu.type = sks; + uu.data.sks.namespace = sc->uri->data.sks.namespace; + uu.data.sks.identifier = GNUNET_strdup (id_update); + (void) search_start (sc->h, &uu, sc->anonymity, sc->options, NULL, sr); + GNUNET_free (uu.data.sks.identifier); +} + + +/** + * Process a keyword-search result. + * + * @param sc our search context + * @param kb the kblock + * @param size size of kb + */ +static void +process_kblock (struct GNUNET_FS_SearchContext *sc, const struct KBlock *kb, + size_t size) +{ + unsigned int i; + size_t j; + GNUNET_HashCode q; + char pt[size - sizeof (struct KBlock)]; + struct GNUNET_CRYPTO_AesSessionKey skey; + struct GNUNET_CRYPTO_AesInitializationVector iv; + const char *eos; + struct GNUNET_CONTAINER_MetaData *meta; + struct GNUNET_FS_Uri *uri; + char *emsg; + + GNUNET_CRYPTO_hash (&kb->keyspace, + sizeof (struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded), + &q); + /* find key */ + for (i = 0; i < sc->uri->data.ksk.keywordCount; i++) + if (0 == memcmp (&q, &sc->requests[i].query, sizeof (GNUNET_HashCode))) + break; + if (i == sc->uri->data.ksk.keywordCount) + { + /* oops, does not match any of our keywords!? */ + GNUNET_break (0); + return; + } + /* decrypt */ + GNUNET_CRYPTO_hash_to_aes_key (&sc->requests[i].key, &skey, &iv); + if (-1 == + GNUNET_CRYPTO_aes_decrypt (&kb[1], size - sizeof (struct KBlock), &skey, + &iv, pt)) + { + GNUNET_break (0); + return; + } + /* parse */ + eos = memchr (pt, 0, sizeof (pt)); + if (NULL == eos) + { + GNUNET_break_op (0); + return; + } + j = eos - pt + 1; + if (sizeof (pt) == j) + meta = GNUNET_CONTAINER_meta_data_create (); + else + meta = GNUNET_CONTAINER_meta_data_deserialize (&pt[j], sizeof (pt) - j); + if (meta == NULL) + { + GNUNET_break_op (0); /* kblock malformed */ + return; + } + uri = GNUNET_FS_uri_parse (pt, &emsg); + if (uri == NULL) + { + GNUNET_break_op (0); /* kblock malformed */ + GNUNET_free_non_null (emsg); + GNUNET_CONTAINER_meta_data_destroy (meta); + return; + } + /* process */ + process_ksk_result (sc, &sc->requests[i], uri, meta); + + /* clean up */ + GNUNET_CONTAINER_meta_data_destroy (meta); + GNUNET_FS_uri_destroy (uri); +} + + +/** + * Process a keyword-search result with a namespace advertisment. + * + * @param sc our search context + * @param nb the nblock + * @param size size of nb + */ +static void +process_nblock (struct GNUNET_FS_SearchContext *sc, const struct NBlock *nb, + size_t size) +{ + unsigned int i; + size_t j; + GNUNET_HashCode q; + char pt[size - sizeof (struct NBlock)]; + struct GNUNET_CRYPTO_AesSessionKey skey; + struct GNUNET_CRYPTO_AesInitializationVector iv; + const char *eos; + struct GNUNET_CONTAINER_MetaData *meta; + struct GNUNET_FS_Uri *uri; + char *uris; + + GNUNET_CRYPTO_hash (&nb->keyspace, + sizeof (struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded), + &q); + /* find key */ + for (i = 0; i < sc->uri->data.ksk.keywordCount; i++) + if (0 == memcmp (&q, &sc->requests[i].query, sizeof (GNUNET_HashCode))) + break; + if (i == sc->uri->data.ksk.keywordCount) + { + /* oops, does not match any of our keywords!? */ + GNUNET_break (0); + return; + } + /* decrypt */ + GNUNET_CRYPTO_hash_to_aes_key (&sc->requests[i].key, &skey, &iv); + if (-1 == + GNUNET_CRYPTO_aes_decrypt (&nb[1], size - sizeof (struct NBlock), &skey, + &iv, pt)) + { + GNUNET_break (0); + return; + } + /* parse */ + eos = memchr (pt, 0, sizeof (pt)); + if (NULL == eos) + { + GNUNET_break_op (0); + return; + } + j = eos - pt + 1; + if (sizeof (pt) == j) + meta = GNUNET_CONTAINER_meta_data_create (); + else + meta = GNUNET_CONTAINER_meta_data_deserialize (&pt[j], sizeof (pt) - j); + if (meta == NULL) + { + GNUNET_break_op (0); /* nblock malformed */ + return; + } + + uri = GNUNET_malloc (sizeof (struct GNUNET_FS_Uri)); + uri->type = sks; + uri->data.sks.identifier = GNUNET_strdup (pt); + GNUNET_CRYPTO_hash (&nb->subspace, + sizeof (struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded), + &uri->data.sks.namespace); + uris = GNUNET_FS_uri_to_string (uri); + GNUNET_CONTAINER_meta_data_insert (meta, "<gnunet>", EXTRACTOR_METATYPE_URI, + EXTRACTOR_METAFORMAT_UTF8, "text/plain", + uris, strlen (uris) + 1); + GNUNET_free (uris); + GNUNET_PSEUDONYM_add (sc->h->cfg, &uri->data.sks.namespace, meta); + /* process */ + process_ksk_result (sc, &sc->requests[i], uri, meta); + + /* clean up */ + GNUNET_CONTAINER_meta_data_destroy (meta); + GNUNET_FS_uri_destroy (uri); +} + + +/** + * Process a namespace-search result. + * + * @param sc our search context + * @param sb the sblock + * @param size size of sb + */ +static void +process_sblock (struct GNUNET_FS_SearchContext *sc, const struct SBlock *sb, + size_t size) +{ + size_t len = size - sizeof (struct SBlock); + char pt[len]; + struct GNUNET_CRYPTO_AesSessionKey skey; + struct GNUNET_CRYPTO_AesInitializationVector iv; + struct GNUNET_FS_Uri *uri; + struct GNUNET_CONTAINER_MetaData *meta; + const char *id; + const char *uris; + size_t off; + char *emsg; + GNUNET_HashCode key; + char *identifier; + + /* decrypt */ + identifier = sc->uri->data.sks.identifier; + GNUNET_CRYPTO_hash (identifier, strlen (identifier), &key); + GNUNET_CRYPTO_hash_to_aes_key (&key, &skey, &iv); + if (-1 == GNUNET_CRYPTO_aes_decrypt (&sb[1], len, &skey, &iv, pt)) + { + GNUNET_break (0); + return; + } + /* parse */ + off = GNUNET_STRINGS_buffer_tokenize (pt, len, 2, &id, &uris); + if (off == 0) + { + GNUNET_break_op (0); /* sblock malformed */ + return; + } + meta = GNUNET_CONTAINER_meta_data_deserialize (&pt[off], len - off); + if (meta == NULL) + { + GNUNET_break_op (0); /* sblock malformed */ + return; + } + uri = GNUNET_FS_uri_parse (uris, &emsg); + if (uri == NULL) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Failed to parse URI `%s': %s\n", uris, + emsg); + GNUNET_break_op (0); /* sblock malformed */ + GNUNET_free_non_null (emsg); + GNUNET_CONTAINER_meta_data_destroy (meta); + return; + } + /* process */ + process_sks_result (sc, id, uri, meta); + /* clean up */ + GNUNET_FS_uri_destroy (uri); + GNUNET_CONTAINER_meta_data_destroy (meta); +} + + +/** + * Process a search result. + * + * @param sc our search context + * @param type type of the result + * @param expiration when it will expire + * @param data the (encrypted) response + * @param size size of data + */ +static void +process_result (struct GNUNET_FS_SearchContext *sc, enum GNUNET_BLOCK_Type type, + struct GNUNET_TIME_Absolute expiration, const void *data, + size_t size) +{ + if (GNUNET_TIME_absolute_get_duration (expiration).rel_value > 0) + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Result received has already expired.\n"); + return; /* result expired */ + } + switch (type) + { + case GNUNET_BLOCK_TYPE_FS_KBLOCK: + if (!GNUNET_FS_uri_test_ksk (sc->uri)) + { + GNUNET_break (0); + return; + } + if (sizeof (struct KBlock) > size) + { + GNUNET_break_op (0); + return; + } + process_kblock (sc, data, size); + break; + case GNUNET_BLOCK_TYPE_FS_SBLOCK: + if (!GNUNET_FS_uri_test_sks (sc->uri)) + { + GNUNET_break (0); + return; + } + if (sizeof (struct SBlock) > size) + { + GNUNET_break_op (0); + return; + } + process_sblock (sc, data, size); + break; + case GNUNET_BLOCK_TYPE_FS_NBLOCK: + if (!GNUNET_FS_uri_test_ksk (sc->uri)) + { + GNUNET_break (0); + return; + } + if (sizeof (struct NBlock) > size) + { + GNUNET_break_op (0); + return; + } + process_nblock (sc, data, size); + break; + case GNUNET_BLOCK_TYPE_ANY: + GNUNET_break (0); + break; + case GNUNET_BLOCK_TYPE_FS_DBLOCK: + GNUNET_break (0); + break; + case GNUNET_BLOCK_TYPE_FS_ONDEMAND: + GNUNET_break (0); + break; + case GNUNET_BLOCK_TYPE_FS_IBLOCK: + GNUNET_break (0); + break; + default: + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _("Got result with unknown block type `%d', ignoring"), type); + break; + } +} + + +/** + * Shutdown any existing connection to the FS + * service and try to establish a fresh one + * (and then re-transmit our search request). + * + * @param sc the search to reconnec + */ +static void +try_reconnect (struct GNUNET_FS_SearchContext *sc); + + +/** + * Type of a function to call when we receive a message + * from the service. + * + * @param cls closure + * @param msg message received, NULL on timeout or fatal error + */ +static void +receive_results (void *cls, const struct GNUNET_MessageHeader *msg) +{ + struct GNUNET_FS_SearchContext *sc = cls; + const struct ClientPutMessage *cm; + uint16_t msize; + + if ((NULL == msg) || (ntohs (msg->type) != GNUNET_MESSAGE_TYPE_FS_PUT) || + (ntohs (msg->size) <= sizeof (struct ClientPutMessage))) + { + try_reconnect (sc); + return; + } + msize = ntohs (msg->size); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Receiving %u bytes of result from fs service\n", msize); + cm = (const struct ClientPutMessage *) msg; + process_result (sc, ntohl (cm->type), + GNUNET_TIME_absolute_ntoh (cm->expiration), &cm[1], + msize - sizeof (struct ClientPutMessage)); + /* continue receiving */ + GNUNET_CLIENT_receive (sc->client, &receive_results, sc, + GNUNET_TIME_UNIT_FOREVER_REL); +} + + +/** + * Schedule the transmission of the (next) search request + * to the service. + * + * @param sc context for the search + */ +static void +schedule_transmit_search_request (struct GNUNET_FS_SearchContext *sc); + + +/** + * Closure for 'build_result_set'. + */ +struct MessageBuilderContext +{ + /** + * How many entries can we store to xoff. + */ + unsigned int put_cnt; + + /** + * How many entries should we skip. + */ + unsigned int skip_cnt; + + /** + * Where to store the keys. + */ + GNUNET_HashCode *xoff; + + /** + * Search context we are iterating for. + */ + struct GNUNET_FS_SearchContext *sc; + + /** + * Keyword offset the search result must match (0 for SKS) + */ + unsigned int keyword_offset; +}; + + +/** + * Iterating over the known results, pick those matching the given + * result range and store their keys at 'xoff'. + * + * @param cls the 'struct MessageBuilderContext' + * @param key key for a result + * @param value the search result + * @return GNUNET_OK to continue iterating + */ +static int +build_result_set (void *cls, const GNUNET_HashCode * key, void *value) +{ + struct MessageBuilderContext *mbc = cls; + struct GNUNET_FS_SearchResult *sr = value; + + if ( (sr->keyword_bitmap != NULL) && + (0 == (sr->keyword_bitmap[mbc->keyword_offset / 8] & (1 << (mbc->keyword_offset % 8)))) ) + return GNUNET_OK; /* have no match for this keyword yet */ + if (mbc->skip_cnt > 0) + { + mbc->skip_cnt--; + return GNUNET_OK; + } + if (mbc->put_cnt == 0) + return GNUNET_SYSERR; + mbc->sc->search_request_map_offset++; + mbc->xoff[--mbc->put_cnt] = *key; + return GNUNET_OK; +} + + +/** + * Iterating over the known results, count those + * matching the given result range and increment + * put count for each. + * + * @param cls the 'struct MessageBuilderContext' + * @param key key for a result + * @param value the search result + * @return GNUNET_OK to continue iterating + */ +static int +find_result_set (void *cls, const GNUNET_HashCode * key, void *value) +{ + struct MessageBuilderContext *mbc = cls; + struct GNUNET_FS_SearchResult *sr = value; + + if ( (sr->keyword_bitmap != NULL) && + (0 == (sr->keyword_bitmap[mbc->keyword_offset / 8] & (1 << (mbc->keyword_offset % 8)))) ) + return GNUNET_OK; /* have no match for this keyword yet */ + mbc->put_cnt++; + return GNUNET_OK; +} + + +/** + * We're ready to transmit the search request to the + * file-sharing service. Do it. + * + * @param cls closure + * @param size number of bytes available in buf + * @param buf where the callee should write the message + * @return number of bytes written to buf + */ +static size_t +transmit_search_request (void *cls, size_t size, void *buf) +{ + struct GNUNET_FS_SearchContext *sc = cls; + struct MessageBuilderContext mbc; + size_t msize; + struct SearchMessage *sm; + const char *identifier; + GNUNET_HashCode key; + GNUNET_HashCode idh; + unsigned int sqms; + uint32_t options; + + if (NULL == buf) + { + try_reconnect (sc); + return 0; + } + mbc.sc = sc; + mbc.skip_cnt = sc->search_request_map_offset; + sm = buf; + sm->header.type = htons (GNUNET_MESSAGE_TYPE_FS_START_SEARCH); + mbc.xoff = (GNUNET_HashCode *) & sm[1]; + options = SEARCH_MESSAGE_OPTION_NONE; + if (0 != (sc->options & GNUNET_FS_SEARCH_OPTION_LOOPBACK_ONLY)) + options |= SEARCH_MESSAGE_OPTION_LOOPBACK_ONLY; + if (GNUNET_FS_uri_test_ksk (sc->uri)) + { + msize = sizeof (struct SearchMessage); + GNUNET_assert (size >= msize); + mbc.keyword_offset = sc->keyword_offset; + mbc.put_cnt = 0; + GNUNET_CONTAINER_multihashmap_iterate (sc->master_result_map, + &find_result_set, &mbc); + sqms = mbc.put_cnt; + mbc.put_cnt = (size - msize) / sizeof (GNUNET_HashCode); + mbc.put_cnt = GNUNET_MIN (mbc.put_cnt, sqms - mbc.skip_cnt); + if (sc->search_request_map_offset < sqms) + GNUNET_assert (mbc.put_cnt > 0); + + sm->header.size = htons (msize); + sm->type = htonl (GNUNET_BLOCK_TYPE_ANY); + sm->anonymity_level = htonl (sc->anonymity); + memset (&sm->target, 0, sizeof (GNUNET_HashCode)); + sm->query = sc->requests[sc->keyword_offset].query; + msize += sizeof (GNUNET_HashCode) * mbc.put_cnt; + GNUNET_CONTAINER_multihashmap_iterate (sc->master_result_map, + &build_result_set, &mbc); + sm->header.size = htons (msize); + GNUNET_assert (sqms >= sc->search_request_map_offset); + if (sqms != sc->search_request_map_offset) + { + /* more requesting to be done... */ + sm->options = htonl (options | SEARCH_MESSAGE_OPTION_CONTINUED); + schedule_transmit_search_request (sc); + return msize; + } + sm->options = htonl (options); + sc->keyword_offset++; + if (sc->uri->data.ksk.keywordCount != sc->keyword_offset) + { + /* more requesting to be done... */ + schedule_transmit_search_request (sc); + return msize; + } + } + else + { + GNUNET_assert (GNUNET_FS_uri_test_sks (sc->uri)); + msize = sizeof (struct SearchMessage); + GNUNET_assert (size >= msize); + sm->type = htonl (GNUNET_BLOCK_TYPE_FS_SBLOCK); + sm->anonymity_level = htonl (sc->anonymity); + sm->target = sc->uri->data.sks.namespace; + identifier = sc->uri->data.sks.identifier; + GNUNET_CRYPTO_hash (identifier, strlen (identifier), &key); + GNUNET_CRYPTO_hash (&key, sizeof (GNUNET_HashCode), &idh); + GNUNET_CRYPTO_hash_xor (&idh, &sm->target, &sm->query); + mbc.put_cnt = (size - msize) / sizeof (GNUNET_HashCode); + sqms = GNUNET_CONTAINER_multihashmap_size (sc->master_result_map); + mbc.put_cnt = GNUNET_MIN (mbc.put_cnt, sqms - mbc.skip_cnt); + mbc.keyword_offset = 0; + if (sc->search_request_map_offset < sqms) + GNUNET_assert (mbc.put_cnt > 0); + msize += sizeof (GNUNET_HashCode) * mbc.put_cnt; + GNUNET_CONTAINER_multihashmap_iterate (sc->master_result_map, + &build_result_set, &mbc); + sm->header.size = htons (msize); + GNUNET_assert (sqms >= sc->search_request_map_offset); + if (sqms != sc->search_request_map_offset) + { + /* more requesting to be done... */ + sm->options = htonl (options | SEARCH_MESSAGE_OPTION_CONTINUED); + schedule_transmit_search_request (sc); + return msize; + } + sm->options = htonl (options); + } + GNUNET_CLIENT_receive (sc->client, &receive_results, sc, + GNUNET_TIME_UNIT_FOREVER_REL); + return msize; +} + + +/** + * Schedule the transmission of the (next) search request + * to the service. + * + * @param sc context for the search + */ +static void +schedule_transmit_search_request (struct GNUNET_FS_SearchContext *sc) +{ + size_t size; + unsigned int sqms; + unsigned int fit; + + size = sizeof (struct SearchMessage); + sqms = + GNUNET_CONTAINER_multihashmap_size (sc->master_result_map) - + sc->search_request_map_offset; + fit = (GNUNET_SERVER_MAX_MESSAGE_SIZE - 1 - size) / sizeof (GNUNET_HashCode); + fit = GNUNET_MIN (fit, sqms); + size += sizeof (GNUNET_HashCode) * fit; + GNUNET_CLIENT_notify_transmit_ready (sc->client, size, + GNUNET_CONSTANTS_SERVICE_TIMEOUT, + GNUNET_NO, &transmit_search_request, sc); + +} + + +/** + * Reconnect to the FS service and transmit + * our queries NOW. + * + * @param cls our search context + * @param tc unused + */ +static void +do_reconnect (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct GNUNET_FS_SearchContext *sc = cls; + struct GNUNET_CLIENT_Connection *client; + + sc->task = GNUNET_SCHEDULER_NO_TASK; + client = GNUNET_CLIENT_connect ("fs", sc->h->cfg); + if (NULL == client) + { + try_reconnect (sc); + return; + } + sc->client = client; + sc->search_request_map_offset = 0; + sc->keyword_offset = 0; + schedule_transmit_search_request (sc); +} + + +/** + * Shutdown any existing connection to the FS + * service and try to establish a fresh one + * (and then re-transmit our search request). + * + * @param sc the search to reconnec + */ +static void +try_reconnect (struct GNUNET_FS_SearchContext *sc) +{ + if (NULL != sc->client) + { + GNUNET_CLIENT_disconnect (sc->client, GNUNET_NO); + sc->client = NULL; + } + sc->task = + GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_SECONDS, &do_reconnect, + sc); +} + + +/** + * Start search for content, internal API. + * + * @param h handle to the file sharing subsystem + * @param uri specifies the search parameters; can be + * a KSK URI or an SKS URI. + * @param anonymity desired level of anonymity + * @param options options for the search + * @param cctx initial value for the client context + * @param psearch parent search result (for namespace update searches) + * @return context that can be used to control the search + */ +static struct GNUNET_FS_SearchContext * +search_start (struct GNUNET_FS_Handle *h, const struct GNUNET_FS_Uri *uri, + uint32_t anonymity, enum GNUNET_FS_SearchOptions options, + void *cctx, struct GNUNET_FS_SearchResult *psearch) +{ + struct GNUNET_FS_SearchContext *sc; + struct GNUNET_FS_ProgressInfo pi; + + sc = GNUNET_malloc (sizeof (struct GNUNET_FS_SearchContext)); + sc->h = h; + sc->options = options; + sc->uri = GNUNET_FS_uri_dup (uri); + sc->anonymity = anonymity; + sc->start_time = GNUNET_TIME_absolute_get (); + if (psearch != NULL) + { + sc->psearch_result = psearch; + psearch->update_search = sc; + } + sc->master_result_map = GNUNET_CONTAINER_multihashmap_create (16); + sc->client_info = cctx; + if (GNUNET_OK != GNUNET_FS_search_start_searching_ (sc)) + { + GNUNET_FS_uri_destroy (sc->uri); + GNUNET_CONTAINER_multihashmap_destroy (sc->master_result_map); + GNUNET_free (sc); + return NULL; + } + GNUNET_FS_search_sync_ (sc); + pi.status = GNUNET_FS_STATUS_SEARCH_START; + sc->client_info = GNUNET_FS_search_make_status_ (&pi, sc); + return sc; +} + + +/** + * Build the request and actually initiate the search using the + * GNUnet FS service. + * + * @param sc search context + * @return GNUNET_OK on success, GNUNET_SYSERR on error + */ +int +GNUNET_FS_search_start_searching_ (struct GNUNET_FS_SearchContext *sc) +{ + unsigned int i; + const char *keyword; + GNUNET_HashCode hc; + struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded pub; + struct GNUNET_CRYPTO_RsaPrivateKey *pk; + + GNUNET_assert (NULL == sc->client); + if (GNUNET_FS_uri_test_ksk (sc->uri)) + { + GNUNET_assert (0 != sc->uri->data.ksk.keywordCount); + sc->requests = + GNUNET_malloc (sizeof (struct SearchRequestEntry) * + sc->uri->data.ksk.keywordCount); + for (i = 0; i < sc->uri->data.ksk.keywordCount; i++) + { + keyword = &sc->uri->data.ksk.keywords[i][1]; + GNUNET_CRYPTO_hash (keyword, strlen (keyword), &hc); + pk = GNUNET_CRYPTO_rsa_key_create_from_hash (&hc); + GNUNET_assert (pk != NULL); + GNUNET_CRYPTO_rsa_key_get_public (pk, &pub); + GNUNET_CRYPTO_rsa_key_free (pk); + GNUNET_CRYPTO_hash (&pub, + sizeof (struct + GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded), + &sc->requests[i].query); + sc->requests[i].mandatory = (sc->uri->data.ksk.keywords[i][0] == '+'); + if (sc->requests[i].mandatory) + sc->mandatory_count++; + sc->requests[i].results = GNUNET_CONTAINER_multihashmap_create (4); + GNUNET_CRYPTO_hash (keyword, strlen (keyword), &sc->requests[i].key); + } + } + sc->client = GNUNET_CLIENT_connect ("fs", sc->h->cfg); + if (NULL == sc->client) + return GNUNET_SYSERR; + schedule_transmit_search_request (sc); + return GNUNET_OK; +} + + +/** + * Freeze probes for the given search result. + * + * @param cls the global FS handle + * @param key the key for the search result (unused) + * @param value the search result to free + * @return GNUNET_OK + */ +static int +search_result_freeze_probes (void *cls, const GNUNET_HashCode * key, + void *value) +{ + struct GNUNET_FS_SearchResult *sr = value; + + if (sr->probe_ctx != NULL) + { + GNUNET_FS_download_stop (sr->probe_ctx, GNUNET_YES); + sr->probe_ctx = NULL; + } + if (sr->probe_cancel_task != GNUNET_SCHEDULER_NO_TASK) + { + GNUNET_SCHEDULER_cancel (sr->probe_cancel_task); + sr->probe_cancel_task = GNUNET_SCHEDULER_NO_TASK; + } + if (sr->update_search != NULL) + GNUNET_FS_search_pause (sr->update_search); + return GNUNET_OK; +} + + +/** + * Resume probes for the given search result. + * + * @param cls the global FS handle + * @param key the key for the search result (unused) + * @param value the search result to free + * @return GNUNET_OK + */ +static int +search_result_resume_probes (void *cls, const GNUNET_HashCode * key, + void *value) +{ + struct GNUNET_FS_SearchResult *sr = value; + + GNUNET_FS_search_start_probe_ (sr); + if (sr->update_search != NULL) + GNUNET_FS_search_continue (sr->update_search); + return GNUNET_OK; +} + + +/** + * Signal suspend and free the given search result. + * + * @param cls the global FS handle + * @param key the key for the search result (unused) + * @param value the search result to free + * @return GNUNET_OK + */ +static int +search_result_suspend (void *cls, const GNUNET_HashCode * key, void *value) +{ + struct GNUNET_FS_SearchContext *sc = cls; + struct GNUNET_FS_SearchResult *sr = value; + struct GNUNET_FS_ProgressInfo pi; + + if (sr->download != NULL) + GNUNET_FS_download_signal_suspend_ (sr->download); + if (sr->update_search != NULL) + GNUNET_FS_search_signal_suspend_ (sr->update_search); + pi.status = GNUNET_FS_STATUS_SEARCH_RESULT_SUSPEND; + pi.value.search.specifics.result_suspend.cctx = sr->client_info; + pi.value.search.specifics.result_suspend.meta = sr->meta; + pi.value.search.specifics.result_suspend.uri = sr->uri; + sr->client_info = GNUNET_FS_search_make_status_ (&pi, sc); + GNUNET_break (NULL == sr->client_info); + GNUNET_free_non_null (sr->serialization); + GNUNET_FS_uri_destroy (sr->uri); + GNUNET_CONTAINER_meta_data_destroy (sr->meta); + if (sr->probe_ctx != NULL) + GNUNET_FS_download_stop (sr->probe_ctx, GNUNET_YES); + if (sr->probe_cancel_task != GNUNET_SCHEDULER_NO_TASK) + GNUNET_SCHEDULER_cancel (sr->probe_cancel_task); + GNUNET_free_non_null (sr->keyword_bitmap); + GNUNET_free (sr); + return GNUNET_OK; +} + + +/** + * Create SUSPEND event for the given search operation + * and then clean up our state (without stop signal). + * + * @param cls the 'struct GNUNET_FS_SearchContext' to signal for + */ +void +GNUNET_FS_search_signal_suspend_ (void *cls) +{ + struct GNUNET_FS_SearchContext *sc = cls; + struct GNUNET_FS_ProgressInfo pi; + unsigned int i; + + GNUNET_FS_end_top (sc->h, sc->top); + GNUNET_CONTAINER_multihashmap_iterate (sc->master_result_map, + &search_result_suspend, sc); + pi.status = GNUNET_FS_STATUS_SEARCH_SUSPEND; + sc->client_info = GNUNET_FS_search_make_status_ (&pi, sc); + GNUNET_break (NULL == sc->client_info); + if (sc->task != GNUNET_SCHEDULER_NO_TASK) + GNUNET_SCHEDULER_cancel (sc->task); + if (NULL != sc->client) + GNUNET_CLIENT_disconnect (sc->client, GNUNET_NO); + GNUNET_CONTAINER_multihashmap_destroy (sc->master_result_map); + if (sc->requests != NULL) + { + GNUNET_assert (GNUNET_FS_uri_test_ksk (sc->uri)); + for (i = 0; i < sc->uri->data.ksk.keywordCount; i++) + GNUNET_CONTAINER_multihashmap_destroy (sc->requests[i].results); + } + GNUNET_free_non_null (sc->requests); + GNUNET_free_non_null (sc->emsg); + GNUNET_FS_uri_destroy (sc->uri); + GNUNET_free_non_null (sc->serialization); + GNUNET_free (sc); +} + + +/** + * Start search for content. + * + * @param h handle to the file sharing subsystem + * @param uri specifies the search parameters; can be + * a KSK URI or an SKS URI. + * @param anonymity desired level of anonymity + * @param options options for the search + * @param cctx initial value for the client context + * @return context that can be used to control the search + */ +struct GNUNET_FS_SearchContext * +GNUNET_FS_search_start (struct GNUNET_FS_Handle *h, + const struct GNUNET_FS_Uri *uri, uint32_t anonymity, + enum GNUNET_FS_SearchOptions options, void *cctx) +{ + struct GNUNET_FS_SearchContext *ret; + + ret = search_start (h, uri, anonymity, options, cctx, NULL); + if (ret == NULL) + return NULL; + ret->top = GNUNET_FS_make_top (h, &GNUNET_FS_search_signal_suspend_, ret); + return ret; +} + + +/** + * Pause search. + * + * @param sc context for the search that should be paused + */ +void +GNUNET_FS_search_pause (struct GNUNET_FS_SearchContext *sc) +{ + struct GNUNET_FS_ProgressInfo pi; + + if (sc->task != GNUNET_SCHEDULER_NO_TASK) + GNUNET_SCHEDULER_cancel (sc->task); + sc->task = GNUNET_SCHEDULER_NO_TASK; + if (NULL != sc->client) + GNUNET_CLIENT_disconnect (sc->client, GNUNET_NO); + sc->client = NULL; + GNUNET_FS_search_sync_ (sc); + GNUNET_CONTAINER_multihashmap_iterate (sc->master_result_map, + &search_result_freeze_probes, sc); + pi.status = GNUNET_FS_STATUS_SEARCH_PAUSED; + sc->client_info = GNUNET_FS_search_make_status_ (&pi, sc); +} + + +/** + * Continue paused search. + * + * @param sc context for the search that should be resumed + */ +void +GNUNET_FS_search_continue (struct GNUNET_FS_SearchContext *sc) +{ + struct GNUNET_FS_ProgressInfo pi; + + GNUNET_assert (sc->client == NULL); + GNUNET_assert (sc->task == GNUNET_SCHEDULER_NO_TASK); + do_reconnect (sc, NULL); + GNUNET_FS_search_sync_ (sc); + pi.status = GNUNET_FS_STATUS_SEARCH_CONTINUED; + sc->client_info = GNUNET_FS_search_make_status_ (&pi, sc); + GNUNET_CONTAINER_multihashmap_iterate (sc->master_result_map, + &search_result_resume_probes, sc); +} + + +/** + * Free the given search result. + * + * @param cls the global FS handle + * @param key the key for the search result (unused) + * @param value the search result to free + * @return GNUNET_OK + */ +static int +search_result_free (void *cls, const GNUNET_HashCode * key, void *value) +{ + struct GNUNET_FS_SearchContext *sc = cls; + struct GNUNET_FS_SearchResult *sr = value; + struct GNUNET_FS_ProgressInfo pi; + + if (NULL != sr->download) + { + sr->download->search = NULL; + sr->download->top = + GNUNET_FS_make_top (sr->download->h, + &GNUNET_FS_download_signal_suspend_, sr->download); + if (NULL != sr->download->serialization) + { + GNUNET_FS_remove_sync_file_ (sc->h, GNUNET_FS_SYNC_PATH_CHILD_DOWNLOAD, + sr->download->serialization); + GNUNET_free (sr->download->serialization); + sr->download->serialization = NULL; + } + pi.status = GNUNET_FS_STATUS_DOWNLOAD_LOST_PARENT; + GNUNET_FS_download_make_status_ (&pi, sr->download); + GNUNET_FS_download_sync_ (sr->download); + sr->download = NULL; + } + if (NULL != sr->update_search) + { + GNUNET_FS_search_stop (sr->update_search); + GNUNET_assert (sr->update_search == NULL); + } + pi.status = GNUNET_FS_STATUS_SEARCH_RESULT_STOPPED; + pi.value.search.specifics.result_stopped.cctx = sr->client_info; + pi.value.search.specifics.result_stopped.meta = sr->meta; + pi.value.search.specifics.result_stopped.uri = sr->uri; + sr->client_info = GNUNET_FS_search_make_status_ (&pi, sc); + GNUNET_break (NULL == sr->client_info); + GNUNET_free_non_null (sr->serialization); + GNUNET_FS_uri_destroy (sr->uri); + GNUNET_CONTAINER_meta_data_destroy (sr->meta); + if (sr->probe_ctx != NULL) + GNUNET_FS_download_stop (sr->probe_ctx, GNUNET_YES); + if (sr->probe_cancel_task != GNUNET_SCHEDULER_NO_TASK) + GNUNET_SCHEDULER_cancel (sr->probe_cancel_task); + GNUNET_free_non_null (sr->keyword_bitmap); + GNUNET_free (sr); + return GNUNET_OK; +} + + +/** + * Stop search for content. + * + * @param sc context for the search that should be stopped + */ +void +GNUNET_FS_search_stop (struct GNUNET_FS_SearchContext *sc) +{ + struct GNUNET_FS_ProgressInfo pi; + unsigned int i; + + if (sc->top != NULL) + GNUNET_FS_end_top (sc->h, sc->top); + if (sc->psearch_result != NULL) + sc->psearch_result->update_search = NULL; + GNUNET_CONTAINER_multihashmap_iterate (sc->master_result_map, + &search_result_free, sc); + if (sc->serialization != NULL) + { + GNUNET_FS_remove_sync_file_ (sc->h, + (sc->psearch_result != + NULL) ? GNUNET_FS_SYNC_PATH_CHILD_SEARCH : + GNUNET_FS_SYNC_PATH_MASTER_SEARCH, + sc->serialization); + GNUNET_FS_remove_sync_dir_ (sc->h, + (sc->psearch_result != + NULL) ? GNUNET_FS_SYNC_PATH_CHILD_SEARCH : + GNUNET_FS_SYNC_PATH_MASTER_SEARCH, + sc->serialization); + GNUNET_free (sc->serialization); + } + pi.status = GNUNET_FS_STATUS_SEARCH_STOPPED; + sc->client_info = GNUNET_FS_search_make_status_ (&pi, sc); + GNUNET_break (NULL == sc->client_info); + if (sc->task != GNUNET_SCHEDULER_NO_TASK) + GNUNET_SCHEDULER_cancel (sc->task); + if (NULL != sc->client) + GNUNET_CLIENT_disconnect (sc->client, GNUNET_NO); + GNUNET_CONTAINER_multihashmap_destroy (sc->master_result_map); + if (sc->requests != NULL) + { + GNUNET_assert (GNUNET_FS_uri_test_ksk (sc->uri)); + for (i = 0; i < sc->uri->data.ksk.keywordCount; i++) + GNUNET_CONTAINER_multihashmap_destroy (sc->requests[i].results); + } + GNUNET_free_non_null (sc->requests); + GNUNET_free_non_null (sc->emsg); + GNUNET_FS_uri_destroy (sc->uri); + GNUNET_free (sc); +} + +/* end of fs_search.c */ diff --git a/src/fs/fs_sharetree.c b/src/fs/fs_sharetree.c new file mode 100644 index 0000000..c929428 --- /dev/null +++ b/src/fs/fs_sharetree.c @@ -0,0 +1,452 @@ +/* + This file is part of GNUnet + (C) 2005-2012 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 2, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/fs_sharetree.c + * @brief code to manipulate the 'struct GNUNET_FS_ShareTreeItem' tree + * @author LRN + * @author Christian Grothoff + */ +#include "platform.h" +#include "gnunet_fs_service.h" +#include "gnunet_scheduler_lib.h" +#include <pthread.h> + + +/** + * Entry for each unique keyword to track how often + * it occured. Contains the keyword and the counter. + */ +struct KeywordCounter +{ + + /** + * This is a doubly-linked list + */ + struct KeywordCounter *prev; + + /** + * This is a doubly-linked list + */ + struct KeywordCounter *next; + + /** + * Keyword that was found. + */ + const char *value; + + /** + * How many files have this keyword? + */ + unsigned int count; + +}; + + +/** + * Aggregate information we keep for meta data in each directory. + */ +struct MetaCounter +{ + + /** + * This is a doubly-linked list + */ + struct MetaCounter *prev; + + /** + * This is a doubly-linked list + */ + struct MetaCounter *next; + + /** + * Name of the plugin that provided that piece of metadata + */ + const char *plugin_name; + + /** + * MIME-type of the metadata itself + */ + const char *data_mime_type; + + /** + * The actual meta data. + */ + const char *data; + + /** + * Number of bytes in 'data'. + */ + size_t data_size; + + /** + * Type of the data + */ + enum EXTRACTOR_MetaType type; + + /** + * Format of the data + */ + enum EXTRACTOR_MetaFormat format; + + /** + * How many files have meta entries matching this value? + * (type and format do not have to match). + */ + unsigned int count; + +}; + + +/** + * A structure that forms a singly-linked list that serves as a stack + * for metadata-processing function. + */ +struct TrimContext +{ + + /** + * Map from the hash over the keyword to an 'struct KeywordCounter *' + * counter that says how often this keyword was + * encountered in the current directory. + */ + struct GNUNET_CONTAINER_MultiHashMap *keywordcounter; + + /** + * Map from the hash over the metadata to an 'struct MetaCounter *' + * counter that says how often this metadata was + * encountered in the current directory. + */ + struct GNUNET_CONTAINER_MultiHashMap *metacounter; + + /** + * Position we are currently manipulating. + */ + struct GNUNET_FS_ShareTreeItem *pos; + + /** + * Number of times an item has to be found to be moved to the parent. + */ + unsigned int move_threshold; + +}; + + +/** + * Add the given keyword to the keyword statistics tracker. + * + * @param cls the multihashmap we store the keyword counters in + * @param keyword the keyword to count + * @param is_mandatory ignored + * @return always GNUNET_OK + */ +static int +add_to_keyword_counter (void *cls, const char *keyword, int is_mandatory) +{ + struct GNUNET_CONTAINER_MultiHashMap *mcm = cls; + struct KeywordCounter *cnt; + GNUNET_HashCode hc; + size_t klen; + + klen = strlen (keyword) + 1; + GNUNET_CRYPTO_hash (keyword, klen - 1, &hc); + cnt = GNUNET_CONTAINER_multihashmap_get (mcm, &hc); + if (cnt == NULL) + { + cnt = GNUNET_malloc (sizeof (struct KeywordCounter) + klen); + cnt->value = (const char *) &cnt[1]; + memcpy (&cnt[1], keyword, klen); + GNUNET_assert (GNUNET_OK == + GNUNET_CONTAINER_multihashmap_put (mcm, + &hc, cnt, + GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY)); + } + cnt->count++; + return GNUNET_OK; +} + + +/** + * Function called on each meta data item. Increments the + * respective counter. + * + * @param cls the container multihashmap to update + * @param plugin_name name of the plugin that produced this value; + * special values can be used (i.e. '<zlib>' for zlib being + * used in the main libextractor library and yielding + * meta data). + * @param type libextractor-type describing the meta data + * @param format basic format information about data + * @param data_mime_type mime-type of data (not of the original file); + * can be NULL (if mime-type is not known) + * @param data actual meta-data found + * @param data_len number of bytes in data + * @return GNUNET_OK to continue extracting / iterating + */ +static int +add_to_meta_counter (void *cls, const char *plugin_name, + enum EXTRACTOR_MetaType type, enum EXTRACTOR_MetaFormat format, + const char *data_mime_type, const char *data, size_t data_len) +{ + struct GNUNET_CONTAINER_MultiHashMap *map = cls; + GNUNET_HashCode key; + struct MetaCounter *cnt; + + GNUNET_CRYPTO_hash (data, data_len, &key); + cnt = GNUNET_CONTAINER_multihashmap_get (map, &key); + if (cnt == NULL) + { + cnt = GNUNET_malloc (sizeof (struct MetaCounter)); + cnt->data = data; + cnt->data_size = data_len; + cnt->plugin_name = plugin_name; + cnt->type = type; + cnt->format = format; + cnt->data_mime_type = data_mime_type; + GNUNET_assert (GNUNET_OK == + GNUNET_CONTAINER_multihashmap_put (map, + &key, cnt, + GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY)); + } + cnt->count++; + return 0; +} + + +/** + * Remove keywords above the threshold. + * + * @param cls the 'struct TrimContext' with the pos to remove the keywords from + * @param keyword the keyword to check + * @param is_mandatory ignored + * @return always GNUNET_OK + */ +static int +remove_high_frequency_keywords (void *cls, const char *keyword, int is_mandatory) +{ + struct TrimContext *tc = cls; + struct KeywordCounter *counter; + GNUNET_HashCode hc; + size_t klen; + + klen = strlen (keyword) + 1; + GNUNET_CRYPTO_hash (keyword, klen - 1, &hc); + counter = GNUNET_CONTAINER_multihashmap_get (tc->keywordcounter, &hc); + GNUNET_assert (NULL != counter); + if (counter->count < tc->move_threshold) + return GNUNET_OK; + GNUNET_FS_uri_ksk_remove_keyword (tc->pos->ksk_uri, + counter->value); + return GNUNET_OK; +} + + +/** + * Move "frequent" keywords over to the target ksk uri, free the + * counters. + * + * @param cls the 'struct TrimContext' + * @param key key of the entry + * @param value the 'struct KeywordCounter' + * @return GNUNET_YES (always) + */ +static int +migrate_and_drop_keywords (void *cls, const GNUNET_HashCode * key, void *value) +{ + struct TrimContext *tc = cls; + struct KeywordCounter *counter = value; + + if (counter->count >= tc->move_threshold) + { + if (NULL == tc->pos->ksk_uri) + tc->pos->ksk_uri = GNUNET_FS_uri_ksk_create_from_args (1, &counter->value); + else + GNUNET_FS_uri_ksk_add_keyword (tc->pos->ksk_uri, counter->value, GNUNET_NO); + } + GNUNET_assert (GNUNET_YES == + GNUNET_CONTAINER_multihashmap_remove (tc->keywordcounter, + key, + counter)); + GNUNET_free (counter); + return GNUNET_YES; +} + + +/** + * Copy "frequent" metadata items over to the + * target metadata container, free the counters. + * + * @param cls the 'struct TrimContext' + * @param key key of the entry + * @param value the 'struct KeywordCounter' + * @return GNUNET_YES (always) + */ +static int +migrate_and_drop_metadata (void *cls, const GNUNET_HashCode * key, void *value) +{ + struct TrimContext *tc = cls; + struct MetaCounter *counter = value; + + if (counter->count >= tc->move_threshold) + { + if (NULL == tc->pos->meta) + tc->pos->meta = GNUNET_CONTAINER_meta_data_create (); + GNUNET_CONTAINER_meta_data_insert (tc->pos->meta, + counter->plugin_name, + counter->type, + counter->format, + counter->data_mime_type, counter->data, + counter->data_size); + } + GNUNET_assert (GNUNET_YES == + GNUNET_CONTAINER_multihashmap_remove (tc->metacounter, + key, + counter)); + GNUNET_free (counter); + return GNUNET_YES; +} + + +/** + * Process a share item tree, moving frequent keywords up and + * copying frequent metadata up. + * + * @param tc trim context with hash maps to use + * @param tree tree to trim + */ +static void +share_tree_trim (struct TrimContext *tc, + struct GNUNET_FS_ShareTreeItem *tree) +{ + struct GNUNET_FS_ShareTreeItem *pos; + unsigned int num_children; + + /* first, trim all children */ + num_children = 0; + for (pos = tree->children_head; NULL != pos; pos = pos->next) + { + share_tree_trim (tc, pos); + num_children++; + } + + /* consider adding filename to directory meta data */ + if (tree->is_directory == GNUNET_YES) + { + const char *user = getenv ("USER"); + if ( (user == NULL) || + (0 != strncasecmp (user, tree->short_filename, strlen(user)))) + { + /* only use filename if it doesn't match $USER */ + if (NULL == tree->meta) + tree->meta = GNUNET_CONTAINER_meta_data_create (); + GNUNET_CONTAINER_meta_data_insert (tree->meta, "<libgnunetfs>", + EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", tree->short_filename, + strlen (tree->short_filename) + 1); + } + } + + if (1 >= num_children) + return; /* nothing to trim */ + + /* now, count keywords and meta data in children */ + for (pos = tree->children_head; NULL != pos; pos = pos->next) + { + if (NULL != pos->meta) + GNUNET_CONTAINER_meta_data_iterate (pos->meta, &add_to_meta_counter, tc->metacounter); + if (NULL != pos->ksk_uri) + GNUNET_FS_uri_ksk_get_keywords (pos->ksk_uri, &add_to_keyword_counter, tc->keywordcounter); + } + + /* calculate threshold for moving keywords / meta data */ + tc->move_threshold = 1 + (num_children / 2); + + /* remove high-frequency keywords from children */ + for (pos = tree->children_head; NULL != pos; pos = pos->next) + { + tc->pos = pos; + if (NULL != pos->ksk_uri) + { + struct GNUNET_FS_Uri *ksk_uri_copy = GNUNET_FS_uri_dup (pos->ksk_uri); + GNUNET_FS_uri_ksk_get_keywords (ksk_uri_copy, &remove_high_frequency_keywords, tc); + GNUNET_FS_uri_destroy (ksk_uri_copy); + } + } + + /* add high-frequency meta data and keywords to parent */ + tc->pos = tree; + GNUNET_CONTAINER_multihashmap_iterate (tc->keywordcounter, + &migrate_and_drop_keywords, + tc); + GNUNET_CONTAINER_multihashmap_iterate (tc->metacounter, + &migrate_and_drop_metadata, + tc); +} + + +/** + * Process a share item tree, moving frequent keywords up and + * copying frequent metadata up. + * + * @param toplevel toplevel directory in the tree, returned by the scanner + */ +void +GNUNET_FS_share_tree_trim (struct GNUNET_FS_ShareTreeItem *toplevel) +{ + struct TrimContext tc; + + if (toplevel == NULL) + return; + tc.keywordcounter = GNUNET_CONTAINER_multihashmap_create (1024); + tc.metacounter = GNUNET_CONTAINER_multihashmap_create (1024); + share_tree_trim (&tc, toplevel); + GNUNET_CONTAINER_multihashmap_destroy (tc.keywordcounter); + GNUNET_CONTAINER_multihashmap_destroy (tc.metacounter); +} + + +/** + * Release memory of a share item tree. + * + * @param toplevel toplevel of the tree to be freed + */ +void +GNUNET_FS_share_tree_free (struct GNUNET_FS_ShareTreeItem *toplevel) +{ + struct GNUNET_FS_ShareTreeItem *pos; + + while (NULL != (pos = toplevel->children_head)) + GNUNET_FS_share_tree_free (pos); + if (NULL != toplevel->parent) + GNUNET_CONTAINER_DLL_remove (toplevel->parent->children_head, + toplevel->parent->children_tail, + toplevel); + if (NULL != toplevel->meta) + GNUNET_CONTAINER_meta_data_destroy (toplevel->meta); + if (NULL != toplevel->ksk_uri) + GNUNET_FS_uri_destroy (toplevel->ksk_uri); + GNUNET_free_non_null (toplevel->filename); + GNUNET_free_non_null (toplevel->short_filename); + GNUNET_free (toplevel); +} + +/* end fs_sharetree.c */ + diff --git a/src/fs/fs_test_lib.c b/src/fs/fs_test_lib.c new file mode 100644 index 0000000..06ab01f --- /dev/null +++ b/src/fs/fs_test_lib.c @@ -0,0 +1,731 @@ +/* + This file is part of GNUnet. + (C) 2010, 2011 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/fs_test_lib.c + * @brief library routines for testing FS publishing and downloading + * with multiple peers; this code is limited to flat files + * and no keywords (those functions can be tested with + * single-peer setups; this is for testing routing). + * @author Christian Grothoff + */ +#include "platform.h" +#include "fs_api.h" +#include "fs_test_lib.h" +#include "gnunet_testing_lib.h" + +#define CONNECT_ATTEMPTS 4 + +#define CONTENT_LIFETIME GNUNET_TIME_UNIT_HOURS + +/** + * Handle for a daemon started for testing FS. + */ +struct GNUNET_FS_TestDaemon +{ + + /** + * Global configuration, only stored in first test daemon, + * otherwise NULL. + */ + struct GNUNET_CONFIGURATION_Handle *gcfg; + + /** + * Handle to the file sharing context using this daemon. + */ + struct GNUNET_FS_Handle *fs; + + /** + * Handle to the daemon via testing. + */ + struct GNUNET_TESTING_Daemon *daemon; + + /** + * Note that 'group' will be the same value for all of the + * daemons started jointly. + */ + struct GNUNET_TESTING_PeerGroup *group; + + /** + * Configuration for accessing this peer. + */ + struct GNUNET_CONFIGURATION_Handle *cfg; + + /** + * ID of this peer. + */ + struct GNUNET_PeerIdentity id; + + /** + * Function to call when upload is done. + */ + GNUNET_FS_TEST_UriContinuation publish_cont; + + /** + * Closure for publish_cont. + */ + void *publish_cont_cls; + + /** + * Task to abort publishing (timeout). + */ + GNUNET_SCHEDULER_TaskIdentifier publish_timeout_task; + + /** + * Seed for file generation. + */ + uint32_t publish_seed; + + /** + * Context for current publishing operation. + */ + struct GNUNET_FS_PublishContext *publish_context; + + /** + * Result URI. + */ + struct GNUNET_FS_Uri *publish_uri; + + /** + * Name of the temporary file used, or NULL for none. + */ + char *publish_tmp_file; + + /** + * Function to call when download is done. + */ + GNUNET_SCHEDULER_Task download_cont; + + /** + * Closure for download_cont. + */ + void *download_cont_cls; + + /** + * Seed for download verification. + */ + uint32_t download_seed; + + /** + * Task to abort downloading (timeout). + */ + GNUNET_SCHEDULER_TaskIdentifier download_timeout_task; + + /** + * Context for current download operation. + */ + struct GNUNET_FS_DownloadContext *download_context; + + /** + * Verbosity level of the current operation. + */ + int verbose; + + +}; + +/** + * Check whether peers successfully shut down. + */ +static void +shutdown_callback (void *cls, const char *emsg) +{ + struct GNUNET_CONFIGURATION_Handle *gcfg = cls; + + if (emsg != NULL) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Shutdown of peers failed: %s\n", + emsg); + } + else + { +#if VERBOSE + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "All peers successfully shut down!\n"); +#endif + } + if (gcfg != NULL) + GNUNET_CONFIGURATION_destroy (gcfg); +} + + +static void +report_uri (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct GNUNET_FS_TestDaemon *daemon = cls; + GNUNET_FS_TEST_UriContinuation cont; + struct GNUNET_FS_Uri *uri; + + GNUNET_FS_publish_stop (daemon->publish_context); + daemon->publish_context = NULL; + cont = daemon->publish_cont; + daemon->publish_cont = NULL; + uri = daemon->publish_uri; + cont (daemon->publish_cont_cls, uri); + GNUNET_FS_uri_destroy (uri); +} + + +static void +report_success (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct GNUNET_FS_TestDaemon *daemon = cls; + + GNUNET_FS_download_stop (daemon->download_context, GNUNET_YES); + daemon->download_context = NULL; + GNUNET_SCHEDULER_add_continuation (daemon->download_cont, + daemon->download_cont_cls, + GNUNET_SCHEDULER_REASON_PREREQ_DONE); + daemon->download_cont = NULL; +} + + +static void * +progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *info) +{ + struct GNUNET_FS_TestDaemon *daemon = cls; + + switch (info->status) + { + case GNUNET_FS_STATUS_PUBLISH_COMPLETED: + GNUNET_SCHEDULER_cancel (daemon->publish_timeout_task); + daemon->publish_timeout_task = GNUNET_SCHEDULER_NO_TASK; + daemon->publish_uri = + GNUNET_FS_uri_dup (info->value.publish.specifics.completed.chk_uri); + GNUNET_SCHEDULER_add_continuation (&report_uri, daemon, + GNUNET_SCHEDULER_REASON_PREREQ_DONE); + break; + case GNUNET_FS_STATUS_PUBLISH_PROGRESS: + if (daemon->verbose) + GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Publishing at %llu/%llu bytes\n", + (unsigned long long) info->value.publish.completed, + (unsigned long long) info->value.publish.size); + break; + case GNUNET_FS_STATUS_DOWNLOAD_PROGRESS: + if (daemon->verbose) + GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Download at %llu/%llu bytes\n", + (unsigned long long) info->value.download.completed, + (unsigned long long) info->value.download.size); + break; + case GNUNET_FS_STATUS_DOWNLOAD_COMPLETED: + GNUNET_SCHEDULER_cancel (daemon->download_timeout_task); + daemon->download_timeout_task = GNUNET_SCHEDULER_NO_TASK; + GNUNET_SCHEDULER_add_continuation (&report_success, daemon, + GNUNET_SCHEDULER_REASON_PREREQ_DONE); + break; + case GNUNET_FS_STATUS_DOWNLOAD_ACTIVE: + case GNUNET_FS_STATUS_DOWNLOAD_INACTIVE: + break; + /* FIXME: monitor data correctness during download progress */ + /* FIXME: do performance reports given sufficient verbosity */ + /* FIXME: advance timeout task to "immediate" on error */ + default: + break; + } + return NULL; +} + + +struct StartContext +{ + struct GNUNET_TIME_Relative timeout; + unsigned int total; + unsigned int have; + struct GNUNET_FS_TestDaemon **daemons; + GNUNET_SCHEDULER_Task cont; + void *cont_cls; + struct GNUNET_TESTING_PeerGroup *group; + struct GNUNET_CONFIGURATION_Handle *cfg; + GNUNET_SCHEDULER_TaskIdentifier timeout_task; +}; + + +static void +notify_running (void *cls, const struct GNUNET_PeerIdentity *id, + const struct GNUNET_CONFIGURATION_Handle *cfg, + struct GNUNET_TESTING_Daemon *d, const char *emsg) +{ + struct StartContext *sctx = cls; + unsigned int i; + + if (emsg != NULL) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, _("Failed to start daemon: %s\n"), + emsg); + return; + } + i = 0; + while (i < sctx->total) + { + if (GNUNET_TESTING_daemon_get (sctx->group, i) == d) + break; + i++; + } + GNUNET_assert (i < sctx->total); + GNUNET_assert (sctx->have < sctx->total); + GNUNET_assert (sctx->daemons[i]->cfg == NULL); + sctx->daemons[i]->cfg = GNUNET_CONFIGURATION_dup (cfg); + sctx->daemons[i]->group = sctx->group; + sctx->daemons[i]->daemon = d; + sctx->daemons[i]->id = *id; + sctx->have++; + if (sctx->have == sctx->total) + { + GNUNET_SCHEDULER_add_continuation (sctx->cont, sctx->cont_cls, + GNUNET_SCHEDULER_REASON_PREREQ_DONE); + sctx->daemons[0]->gcfg = sctx->cfg; + GNUNET_SCHEDULER_cancel (sctx->timeout_task); + for (i = 0; i < sctx->total; i++) + { + sctx->daemons[i]->fs = + GNUNET_FS_start (sctx->daemons[i]->cfg, "<tester>", &progress_cb, + sctx->daemons[i], GNUNET_FS_FLAGS_NONE, + GNUNET_FS_OPTIONS_END); + } + GNUNET_free (sctx); + } +} + + +static void +start_timeout (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct StartContext *sctx = cls; + unsigned int i; + + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Timeout while trying to start daemons\n"); + GNUNET_TESTING_daemons_stop (sctx->group, + GNUNET_TIME_relative_multiply + (GNUNET_TIME_UNIT_SECONDS, 30), + &shutdown_callback, NULL); + for (i = 0; i < sctx->total; i++) + { + if (i < sctx->have) + GNUNET_CONFIGURATION_destroy (sctx->daemons[i]->cfg); + GNUNET_free (sctx->daemons[i]); + sctx->daemons[i] = NULL; + } + GNUNET_CONFIGURATION_destroy (sctx->cfg); + GNUNET_SCHEDULER_add_continuation (sctx->cont, sctx->cont_cls, + GNUNET_SCHEDULER_REASON_TIMEOUT); + GNUNET_free (sctx); +} + + +/** + * Start daemons for testing. + * + * @param template_cfg_file configuration template to use + * @param timeout if this operation cannot be completed within the + * given period, call the continuation with an error code + * @param total number of daemons to start + * @param daemons array of 'total' entries to be initialized + * (array must already be allocated, will be filled) + * @param cont function to call when done + * @param cont_cls closure for cont + */ +void +GNUNET_FS_TEST_daemons_start (const char *template_cfg_file, + struct GNUNET_TIME_Relative timeout, + unsigned int total, + struct GNUNET_FS_TestDaemon **daemons, + GNUNET_SCHEDULER_Task cont, void *cont_cls) +{ + struct StartContext *sctx; + unsigned int i; + + GNUNET_assert (total > 0); + sctx = GNUNET_malloc (sizeof (struct StartContext)); + sctx->daemons = daemons; + sctx->total = total; + sctx->cont = cont; + sctx->cont_cls = cont_cls; + sctx->cfg = GNUNET_CONFIGURATION_create (); + if (GNUNET_OK != GNUNET_CONFIGURATION_load (sctx->cfg, template_cfg_file)) + { + GNUNET_break (0); + GNUNET_CONFIGURATION_destroy (sctx->cfg); + GNUNET_free (sctx); + GNUNET_SCHEDULER_add_continuation (cont, cont_cls, + GNUNET_SCHEDULER_REASON_TIMEOUT); + return; + } + for (i = 0; i < total; i++) + daemons[i] = GNUNET_malloc (sizeof (struct GNUNET_FS_TestDaemon)); + sctx->group = GNUNET_TESTING_daemons_start (sctx->cfg, total, total, /* Outstanding connections */ + total, /* Outstanding ssh connections */ + timeout, NULL, NULL, + ¬ify_running, sctx, NULL, NULL, + NULL); + sctx->timeout_task = + GNUNET_SCHEDULER_add_delayed (timeout, &start_timeout, sctx); +} + + +struct GNUNET_FS_TEST_ConnectContext +{ + GNUNET_SCHEDULER_Task cont; + void *cont_cls; + struct GNUNET_TESTING_ConnectContext *cc; +}; + + +/** + * Prototype of a function that will be called whenever + * two daemons are connected by the testing library. + * + * @param cls closure + * @param first peer id for first daemon + * @param second peer id for the second daemon + * @param distance distance between the connected peers + * @param first_cfg config for the first daemon + * @param second_cfg config for the second daemon + * @param first_daemon handle for the first daemon + * @param second_daemon handle for the second daemon + * @param emsg error message (NULL on success) + */ +static void +notify_connection (void *cls, const struct GNUNET_PeerIdentity *first, + const struct GNUNET_PeerIdentity *second, uint32_t distance, + const struct GNUNET_CONFIGURATION_Handle *first_cfg, + const struct GNUNET_CONFIGURATION_Handle *second_cfg, + struct GNUNET_TESTING_Daemon *first_daemon, + struct GNUNET_TESTING_Daemon *second_daemon, + const char *emsg) +{ + struct GNUNET_FS_TEST_ConnectContext *cc = cls; + + cc->cc = NULL; + if (emsg != NULL) + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Failed to connect peers: %s\n", + emsg); + GNUNET_SCHEDULER_add_continuation (cc->cont, cc->cont_cls, + (emsg != + NULL) ? GNUNET_SCHEDULER_REASON_TIMEOUT : + GNUNET_SCHEDULER_REASON_PREREQ_DONE); + GNUNET_free (cc); +} + + +/** + * Connect two daemons for testing. + * + * @param daemon1 first daemon to connect + * @param daemon2 second first daemon to connect + * @param timeout if this operation cannot be completed within the + * given period, call the continuation with an error code + * @param cont function to call when done + * @param cont_cls closure for cont + */ +struct GNUNET_FS_TEST_ConnectContext * +GNUNET_FS_TEST_daemons_connect (struct GNUNET_FS_TestDaemon *daemon1, + struct GNUNET_FS_TestDaemon *daemon2, + struct GNUNET_TIME_Relative timeout, + GNUNET_SCHEDULER_Task cont, void *cont_cls) +{ + struct GNUNET_FS_TEST_ConnectContext *ncc; + + ncc = GNUNET_malloc (sizeof (struct GNUNET_FS_TEST_ConnectContext)); + ncc->cont = cont; + ncc->cont_cls = cont_cls; + ncc->cc = + GNUNET_TESTING_daemons_connect (daemon1->daemon, daemon2->daemon, timeout, + CONNECT_ATTEMPTS, GNUNET_YES, + ¬ify_connection, ncc); + return ncc; +} + + +/** + * Cancel connect operation. + * + * @param cc operation to cancel + */ +void +GNUNET_FS_TEST_daemons_connect_cancel (struct GNUNET_FS_TEST_ConnectContext *cc) +{ + GNUNET_TESTING_daemons_connect_cancel (cc->cc); + GNUNET_free (cc); +} + + +/** + * Obtain peer configuration used for testing. + * + * @param daemons array with the daemons + * @param off which configuration to get + * @return peer configuration + */ +const struct GNUNET_CONFIGURATION_Handle * +GNUNET_FS_TEST_get_configuration (struct GNUNET_FS_TestDaemon **daemons, + unsigned int off) +{ + return daemons[off]->cfg; +} + +/** + * Obtain peer group used for testing. + * + * @param daemons array with the daemons (must contain at least one) + * @return peer group + */ +struct GNUNET_TESTING_PeerGroup * +GNUNET_FS_TEST_get_group (struct GNUNET_FS_TestDaemon **daemons) +{ + return daemons[0]->group; +} + + +/** + * Stop daemons used for testing. + * + * @param total number of daemons to stop + * @param daemons array with the daemons (values will be clobbered) + */ +void +GNUNET_FS_TEST_daemons_stop (unsigned int total, + struct GNUNET_FS_TestDaemon **daemons) +{ + unsigned int i; + struct GNUNET_TESTING_PeerGroup *pg; + struct GNUNET_CONFIGURATION_Handle *gcfg; + struct GNUNET_FS_TestDaemon *daemon; + + GNUNET_assert (total > 0); + GNUNET_assert (daemons[0] != NULL); + pg = daemons[0]->group; + gcfg = daemons[0]->gcfg; + for (i = 0; i < total; i++) + { + daemon = daemons[i]; + if (daemon->download_timeout_task != GNUNET_SCHEDULER_NO_TASK) + { + GNUNET_SCHEDULER_cancel (daemon->download_timeout_task); + daemon->download_timeout_task = GNUNET_SCHEDULER_NO_TASK; + } + if (daemon->publish_timeout_task != GNUNET_SCHEDULER_NO_TASK) + { + GNUNET_SCHEDULER_cancel (daemon->publish_timeout_task); + daemon->publish_timeout_task = GNUNET_SCHEDULER_NO_TASK; + } + if (NULL != daemon->download_context) + { + GNUNET_FS_download_stop (daemon->download_context, GNUNET_YES); + daemon->download_context = NULL; + } + if (daemon->fs != NULL) + GNUNET_FS_stop (daemon->fs); + if (daemon->cfg != NULL) + GNUNET_CONFIGURATION_destroy (daemon->cfg); + if (NULL != daemon->publish_tmp_file) + { + GNUNET_break (GNUNET_OK == + GNUNET_DISK_directory_remove (daemon->publish_tmp_file)); + GNUNET_free (daemon->publish_tmp_file); + daemon->publish_tmp_file = NULL; + } + GNUNET_free (daemon); + daemons[i] = NULL; + } + GNUNET_TESTING_daemons_stop (pg, + GNUNET_TIME_relative_multiply + (GNUNET_TIME_UNIT_SECONDS, 30), + &shutdown_callback, gcfg); +} + + +static void +publish_timeout (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct GNUNET_FS_TestDaemon *daemon = cls; + GNUNET_FS_TEST_UriContinuation cont; + + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Timeout while trying to publish data\n"); + cont = daemon->publish_cont; + daemon->publish_timeout_task = GNUNET_SCHEDULER_NO_TASK; + daemon->publish_cont = NULL; + GNUNET_FS_publish_stop (daemon->publish_context); + daemon->publish_context = NULL; + cont (daemon->publish_cont_cls, NULL); +} + + +static size_t +file_generator (void *cls, uint64_t offset, size_t max, void *buf, char **emsg) +{ + struct GNUNET_FS_TestDaemon *daemon = cls; + uint64_t pos; + uint8_t *cbuf = buf; + int mod; + + if (emsg != NULL) + *emsg = NULL; + if (buf == NULL) + return 0; + for (pos = 0; pos < 8; pos++) + cbuf[pos] = (uint8_t) (offset >> pos * 8); + for (pos = 8; pos < max; pos++) + { + mod = (255 - (offset / 1024 / 32)); + if (mod == 0) + mod = 1; + cbuf[pos] = (uint8_t) ((offset * daemon->publish_seed) % mod); + } + return max; +} + + + +/** + * Publish a file at the given daemon. + * + * @param daemon where to publish + * @param timeout if this operation cannot be completed within the + * given period, call the continuation with an error code + * @param anonymity option for publication + * @param do_index GNUNET_YES for index, GNUNET_NO for insertion, + * GNUNET_SYSERR for simulation + * @param size size of the file to publish + * @param seed seed to use for file generation + * @param verbose how verbose to be in reporting + * @param cont function to call when done + * @param cont_cls closure for cont + */ +void +GNUNET_FS_TEST_publish (struct GNUNET_FS_TestDaemon *daemon, + struct GNUNET_TIME_Relative timeout, uint32_t anonymity, + int do_index, uint64_t size, uint32_t seed, + unsigned int verbose, + GNUNET_FS_TEST_UriContinuation cont, void *cont_cls) +{ + struct GNUNET_FS_FileInformation *fi; + struct GNUNET_DISK_FileHandle *fh; + char *emsg; + uint64_t off; + char buf[DBLOCK_SIZE]; + size_t bsize; + struct GNUNET_FS_BlockOptions bo; + + GNUNET_assert (daemon->publish_cont == NULL); + daemon->publish_cont = cont; + daemon->publish_cont_cls = cont_cls; + daemon->publish_seed = seed; + daemon->verbose = verbose; + bo.expiration_time = GNUNET_TIME_relative_to_absolute (CONTENT_LIFETIME); + bo.anonymity_level = anonymity; + bo.content_priority = 42; + bo.replication_level = 1; + if (GNUNET_YES == do_index) + { + GNUNET_assert (daemon->publish_tmp_file == NULL); + daemon->publish_tmp_file = GNUNET_DISK_mktemp ("fs-test-publish-index"); + GNUNET_assert (daemon->publish_tmp_file != NULL); + fh = GNUNET_DISK_file_open (daemon->publish_tmp_file, + GNUNET_DISK_OPEN_WRITE | + GNUNET_DISK_OPEN_CREATE, + GNUNET_DISK_PERM_USER_READ | + GNUNET_DISK_PERM_USER_WRITE); + GNUNET_assert (NULL != fh); + off = 0; + while (off < size) + { + bsize = GNUNET_MIN (sizeof (buf), size - off); + emsg = NULL; + GNUNET_assert (bsize == file_generator (daemon, off, bsize, buf, &emsg)); + GNUNET_assert (emsg == NULL); + GNUNET_assert (bsize == GNUNET_DISK_file_write (fh, buf, bsize)); + off += bsize; + } + GNUNET_assert (GNUNET_OK == GNUNET_DISK_file_close (fh)); + fi = GNUNET_FS_file_information_create_from_file (daemon->fs, daemon, + daemon->publish_tmp_file, + NULL, NULL, do_index, + &bo); + } + else + { + fi = GNUNET_FS_file_information_create_from_reader (daemon->fs, daemon, + size, &file_generator, + daemon, NULL, NULL, + do_index, &bo); + } + daemon->publish_context = + GNUNET_FS_publish_start (daemon->fs, fi, NULL, NULL, NULL, + GNUNET_FS_PUBLISH_OPTION_NONE); + daemon->publish_timeout_task = + GNUNET_SCHEDULER_add_delayed (timeout, &publish_timeout, daemon); +} + + +static void +download_timeout (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct GNUNET_FS_TestDaemon *daemon = cls; + + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Timeout while trying to download file\n"); + daemon->download_timeout_task = GNUNET_SCHEDULER_NO_TASK; + GNUNET_FS_download_stop (daemon->download_context, GNUNET_YES); + daemon->download_context = NULL; + GNUNET_SCHEDULER_add_continuation (daemon->download_cont, + daemon->download_cont_cls, + GNUNET_SCHEDULER_REASON_TIMEOUT); + daemon->download_cont = NULL; +} + + +/** + * Perform test download. + * + * @param daemon which peer to download from + * @param timeout if this operation cannot be completed within the + * given period, call the continuation with an error code + * @param anonymity option for download + * @param seed used for file validation + * @param uri URI of file to download (CHK/LOC only) + * @param verbose how verbose to be in reporting + * @param cont function to call when done + * @param cont_cls closure for cont + */ +void +GNUNET_FS_TEST_download (struct GNUNET_FS_TestDaemon *daemon, + struct GNUNET_TIME_Relative timeout, + uint32_t anonymity, uint32_t seed, + const struct GNUNET_FS_Uri *uri, unsigned int verbose, + GNUNET_SCHEDULER_Task cont, void *cont_cls) +{ + uint64_t size; + + GNUNET_assert (daemon->download_cont == NULL); + size = GNUNET_FS_uri_chk_get_file_size (uri); + daemon->verbose = verbose; + daemon->download_cont = cont; + daemon->download_cont_cls = cont_cls; + daemon->download_seed = seed; + daemon->download_context = + GNUNET_FS_download_start (daemon->fs, uri, NULL, NULL, NULL, 0, size, + anonymity, GNUNET_FS_DOWNLOAD_OPTION_NONE, NULL, + NULL); + daemon->download_timeout_task = + GNUNET_SCHEDULER_add_delayed (timeout, &download_timeout, daemon); +} + +/* end of test_fs_lib.c */ diff --git a/src/fs/fs_test_lib.h b/src/fs/fs_test_lib.h new file mode 100644 index 0000000..81125ca --- /dev/null +++ b/src/fs/fs_test_lib.h @@ -0,0 +1,183 @@ +/* + This file is part of GNUnet. + (C) 2010 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/fs_test_lib.h + * @brief library routines for testing FS publishing and downloading + * with multiple peers; this code is limited to flat files + * and no keywords (those functions can be tested with + * single-peer setups; this is for testing routing). + * @author Christian Grothoff + */ +#ifndef FS_TEST_LIB_H +#define FS_TEST_LIB_H + +#include "gnunet_util_lib.h" +#include "gnunet_fs_service.h" + +/** + * Handle for a daemon started for testing FS. + */ +struct GNUNET_FS_TestDaemon; + + +/** + * Start daemons for testing. + * + * @param template_cfg_file configuration template to use + * @param timeout if this operation cannot be completed within the + * given period, call the continuation with an error code + * @param total number of daemons to start + * @param daemons array of 'total' entries to be initialized + * (array must already be allocated, will be filled) + * @param cont function to call when done; note that if 'cont' + * is called with reason "TIMEOUT", then starting the + * daemons has failed and the client MUST NOT call + * 'GNUNET_FS_TEST_daemons_stop'! + * @param cont_cls closure for cont + */ +void +GNUNET_FS_TEST_daemons_start (const char *template_cfg_file, + struct GNUNET_TIME_Relative timeout, + unsigned int total, + struct GNUNET_FS_TestDaemon **daemons, + GNUNET_SCHEDULER_Task cont, void *cont_cls); + + +struct GNUNET_FS_TEST_ConnectContext; + + +/** + * Connect two daemons for testing. + * + * @param daemon1 first daemon to connect + * @param daemon2 second first daemon to connect + * @param timeout if this operation cannot be completed within the + * given period, call the continuation with an error code + * @param cont function to call when done + * @param cont_cls closure for cont + */ +struct GNUNET_FS_TEST_ConnectContext * +GNUNET_FS_TEST_daemons_connect (struct GNUNET_FS_TestDaemon *daemon1, + struct GNUNET_FS_TestDaemon *daemon2, + struct GNUNET_TIME_Relative timeout, + GNUNET_SCHEDULER_Task cont, void *cont_cls); + + +/** + * Cancel connect operation. + * + * @param cc operation to cancel + */ +void +GNUNET_FS_TEST_daemons_connect_cancel (struct GNUNET_FS_TEST_ConnectContext + *cc); + + +/** + * Obtain peer group used for testing. + * + * @param daemons array with the daemons (must contain at least one) + * @return peer group + */ +struct GNUNET_TESTING_PeerGroup * +GNUNET_FS_TEST_get_group (struct GNUNET_FS_TestDaemon **daemons); + + + +/** + * Obtain peer configuration used for testing. + * + * @param daemons array with the daemons + * @param off which configuration to get + * @return peer configuration + */ +const struct GNUNET_CONFIGURATION_Handle * +GNUNET_FS_TEST_get_configuration (struct GNUNET_FS_TestDaemon **daemons, + unsigned int off); + +/** + * Stop daemons used for testing. + * + * @param total number of daemons to stop + * @param daemons array with the daemons (values will be clobbered) + */ +void +GNUNET_FS_TEST_daemons_stop (unsigned int total, + struct GNUNET_FS_TestDaemon **daemons); + + +/** + * Function signature. + * + * @param cls closure (user defined) + * @param uri a URI, NULL for errors + */ +typedef void (*GNUNET_FS_TEST_UriContinuation) (void *cls, + const struct GNUNET_FS_Uri * + uri); + + +/** + * Publish a file at the given daemon. + * + * @param daemon where to publish + * @param timeout if this operation cannot be completed within the + * given period, call the continuation with an error code + * @param anonymity option for publication + * @param do_index GNUNET_YES for index, GNUNET_NO for insertion, + * GNUNET_SYSERR for simulation + * @param size size of the file to publish + * @param seed seed to use for file generation + * @param verbose how verbose to be in reporting + * @param cont function to call when done + * @param cont_cls closure for cont + */ +void +GNUNET_FS_TEST_publish (struct GNUNET_FS_TestDaemon *daemon, + struct GNUNET_TIME_Relative timeout, uint32_t anonymity, + int do_index, uint64_t size, uint32_t seed, + unsigned int verbose, + GNUNET_FS_TEST_UriContinuation cont, void *cont_cls); + + +/** + * Perform test download. + * + * @param daemon which peer to download from + * @param timeout if this operation cannot be completed within the + * given period, call the continuation with an error code + * @param anonymity option for download + * @param seed used for file validation + * @param uri URI of file to download (CHK/LOC only) + * @param verbose how verbose to be in reporting + * @param cont function to call when done + * @param cont_cls closure for cont + */ +void +GNUNET_FS_TEST_download (struct GNUNET_FS_TestDaemon *daemon, + struct GNUNET_TIME_Relative timeout, + uint32_t anonymity, uint32_t seed, + const struct GNUNET_FS_Uri *uri, unsigned int verbose, + GNUNET_SCHEDULER_Task cont, void *cont_cls); + + + +#endif diff --git a/src/fs/fs_test_lib_data.conf b/src/fs/fs_test_lib_data.conf new file mode 100644 index 0000000..e6c2abd --- /dev/null +++ b/src/fs/fs_test_lib_data.conf @@ -0,0 +1,11 @@ +@INLINE@ test_fs_defaults.conf +[PATHS] +SERVICEHOME = /tmp/gnunet-fs-test-lib/ + +[ats] +WAN_QUOTA_IN = 3932160 +WAN_QUOTA_OUT = 3932160 + +[datastore] +QUOTA = 2 GB + diff --git a/src/fs/fs_tree.c b/src/fs/fs_tree.c new file mode 100644 index 0000000..b3bbdc7 --- /dev/null +++ b/src/fs/fs_tree.c @@ -0,0 +1,441 @@ +/* + This file is part of GNUnet. + (C) 2009-2011 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ +/** + * @file fs/fs_tree.c + * @brief Merkle-tree-ish-CHK file encoding for GNUnet + * @see http://gnunet.org/encoding.php3 + * @author Krista Bennett + * @author Christian Grothoff + */ +#include "platform.h" +#include "fs_tree.h" + + +/** + * Context for an ECRS-based file encoder that computes + * the Merkle-ish-CHK tree. + */ +struct GNUNET_FS_TreeEncoder +{ + + /** + * Global FS context. + */ + struct GNUNET_FS_Handle *h; + + /** + * Closure for all callbacks. + */ + void *cls; + + /** + * Function to call on encrypted blocks. + */ + GNUNET_FS_TreeBlockProcessor proc; + + /** + * Function to call with progress information. + */ + GNUNET_FS_TreeProgressCallback progress; + + /** + * Function to call to receive input data. + */ + GNUNET_FS_DataReader reader; + + /** + * Function to call once we're done with processing. + */ + GNUNET_SCHEDULER_Task cont; + + /** + * Set to an error message (if we had an error). + */ + char *emsg; + + /** + * Set to the URI (upon successful completion) + */ + struct GNUNET_FS_Uri *uri; + + /** + * Overall file size. + */ + uint64_t size; + + /** + * How far are we? + */ + uint64_t publish_offset; + + /** + * How deep are we? Depth 0 is for the DBLOCKs. + */ + unsigned int current_depth; + + /** + * How deep is the tree? Always > 0. + */ + unsigned int chk_tree_depth; + + /** + * In-memory cache of the current CHK tree. + * This struct will contain the CHK values + * from the root to the currently processed + * node in the tree as identified by + * "current_depth" and "publish_offset". + * The "chktree" will be initially NULL, + * then allocated to a sufficient number of + * entries for the size of the file and + * finally freed once the upload is complete. + */ + struct ContentHashKey *chk_tree; + + /** + * Are we currently in 'GNUNET_FS_tree_encoder_next'? + * Flag used to prevent recursion. + */ + int in_next; +}; + + +/** + * Compute the depth of the CHK tree. + * + * @param flen file length for which to compute the depth + * @return depth of the tree, always > 0. A depth of 1 means only a DBLOCK. + */ +unsigned int +GNUNET_FS_compute_depth (uint64_t flen) +{ + unsigned int treeDepth; + uint64_t fl; + + treeDepth = 1; + fl = DBLOCK_SIZE; + while (fl < flen) + { + treeDepth++; + if (fl * CHK_PER_INODE < fl) + { + /* integer overflow, this is a HUGE file... */ + return treeDepth; + } + fl = fl * CHK_PER_INODE; + } + return treeDepth; +} + + +/** + * Calculate how many bytes of payload a block tree of the given + * depth MAY correspond to at most (this function ignores the fact that + * some blocks will only be present partially due to the total file + * size cutting some blocks off at the end). + * + * @param depth depth of the block. depth==0 is a DBLOCK. + * @return number of bytes of payload a subtree of this depth may correspond to + */ +uint64_t +GNUNET_FS_tree_compute_tree_size (unsigned int depth) +{ + uint64_t rsize; + unsigned int i; + + rsize = DBLOCK_SIZE; + for (i = 0; i < depth; i++) + rsize *= CHK_PER_INODE; + return rsize; +} + + +/** + * Compute the size of the current IBLOCK. The encoder is + * triggering the calculation of the size of an IBLOCK at the + * *end* (hence end_offset) of its construction. The IBLOCK + * maybe a full or a partial IBLOCK, and this function is to + * calculate how long it should be. + * + * @param depth depth of the IBlock in the tree, 0 would be a DBLOCK, + * must be > 0 (this function is for IBLOCKs only!) + * @param end_offset current offset in the payload (!) of the overall file, + * must be > 0 (since this function is called at the + * end of a block). + * @return size of the corresponding IBlock + */ +static uint16_t +GNUNET_FS_tree_compute_iblock_size (unsigned int depth, uint64_t end_offset) +{ + unsigned int ret; + uint64_t mod; + uint64_t bds; + + GNUNET_assert (depth > 0); + GNUNET_assert (end_offset > 0); + bds = GNUNET_FS_tree_compute_tree_size (depth); + mod = end_offset % bds; + if (0 == mod) + { + /* we were triggered at the end of a full block */ + ret = CHK_PER_INODE; + } + else + { + /* we were triggered at the end of the file */ + bds /= CHK_PER_INODE; + ret = mod / bds; + if (0 != mod % bds) + ret++; + } + return (uint16_t) (ret * sizeof (struct ContentHashKey)); +} + + +/** + * Compute how many bytes of data should be stored in + * the specified block. + * + * @param fsize overall file size, must be > 0. + * @param offset offset in the original data corresponding + * to the beginning of the tree induced by the block; + * must be <= fsize + * @param depth depth of the node in the tree, 0 for DBLOCK + * @return number of bytes stored in this node + */ +size_t +GNUNET_FS_tree_calculate_block_size (uint64_t fsize, uint64_t offset, + unsigned int depth) +{ + size_t ret; + uint64_t rsize; + uint64_t epos; + unsigned int chks; + + GNUNET_assert (fsize > 0); + GNUNET_assert (offset <= fsize); + if (depth == 0) + { + ret = DBLOCK_SIZE; + if ((offset + ret > fsize) || (offset + ret < offset)) + ret = (size_t) (fsize - offset); + return ret; + } + + rsize = GNUNET_FS_tree_compute_tree_size (depth - 1); + epos = offset + rsize * CHK_PER_INODE; + if ((epos < offset) || (epos > fsize)) + epos = fsize; + /* round up when computing #CHKs in our IBlock */ + chks = (epos - offset + rsize - 1) / rsize; + GNUNET_assert (chks <= CHK_PER_INODE); + return chks * sizeof (struct ContentHashKey); +} + + +/** + * Initialize a tree encoder. This function will call "proc" and + * "progress" on each block in the tree. Once all blocks have been + * processed, "cont" will be scheduled. The "reader" will be called + * to obtain the (plaintext) blocks for the file. Note that this + * function will not actually call "proc". The client must + * call "GNUNET_FS_tree_encoder_next" to trigger encryption (and + * calling of "proc") for the each block. + * + * @param h the global FS context + * @param size overall size of the file to encode + * @param cls closure for reader, proc, progress and cont + * @param reader function to call to read plaintext data + * @param proc function to call on each encrypted block + * @param progress function to call with progress information + * @param cont function to call when done + */ +struct GNUNET_FS_TreeEncoder * +GNUNET_FS_tree_encoder_create (struct GNUNET_FS_Handle *h, uint64_t size, + void *cls, GNUNET_FS_DataReader reader, + GNUNET_FS_TreeBlockProcessor proc, + GNUNET_FS_TreeProgressCallback progress, + GNUNET_SCHEDULER_Task cont) +{ + struct GNUNET_FS_TreeEncoder *te; + + te = GNUNET_malloc (sizeof (struct GNUNET_FS_TreeEncoder)); + te->h = h; + te->size = size; + te->cls = cls; + te->reader = reader; + te->proc = proc; + te->progress = progress; + te->cont = cont; + te->chk_tree_depth = GNUNET_FS_compute_depth (size); + te->chk_tree = + GNUNET_malloc (te->chk_tree_depth * CHK_PER_INODE * + sizeof (struct ContentHashKey)); + return te; +} + + +/** + * Compute the offset of the CHK for the + * current block in the IBlock above. + * + * @param depth depth of the IBlock in the tree (aka overall + * number of tree levels minus depth); 0 == DBlock + * @param end_offset current offset in the overall file, + * at the *beginning* of the block for DBLOCKs (depth==0), + * otherwise at the *end* of the block (exclusive) + * @return (array of CHKs') offset in the above IBlock + */ +static unsigned int +compute_chk_offset (unsigned int depth, uint64_t end_offset) +{ + uint64_t bds; + unsigned int ret; + + bds = GNUNET_FS_tree_compute_tree_size (depth); + if (depth > 0) + end_offset--; /* round down since for depth > 0 offset is at the END of the block */ + ret = end_offset / bds; + return ret % CHK_PER_INODE; +} + + +/** + * Encrypt the next block of the file (and call proc and progress + * accordingly; or of course "cont" if we have already completed + * encoding of the entire file). + * + * @param te tree encoder to use + */ +void +GNUNET_FS_tree_encoder_next (struct GNUNET_FS_TreeEncoder *te) +{ + struct ContentHashKey *mychk; + const void *pt_block; + uint16_t pt_size; + char iob[DBLOCK_SIZE]; + char enc[DBLOCK_SIZE]; + struct GNUNET_CRYPTO_AesSessionKey sk; + struct GNUNET_CRYPTO_AesInitializationVector iv; + unsigned int off; + + GNUNET_assert (GNUNET_NO == te->in_next); + te->in_next = GNUNET_YES; + if (te->chk_tree_depth == te->current_depth) + { + off = CHK_PER_INODE * (te->chk_tree_depth - 1); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "TE done, reading CHK `%s' from %u\n", + GNUNET_h2s (&te->chk_tree[off].query), off); + te->uri = GNUNET_malloc (sizeof (struct GNUNET_FS_Uri)); + te->uri->type = chk; + te->uri->data.chk.chk = te->chk_tree[off]; + te->uri->data.chk.file_length = GNUNET_htonll (te->size); + te->in_next = GNUNET_NO; + te->cont (te->cls, NULL); + return; + } + if (0 == te->current_depth) + { + /* read DBLOCK */ + pt_size = GNUNET_MIN (DBLOCK_SIZE, te->size - te->publish_offset); + if (pt_size != + te->reader (te->cls, te->publish_offset, pt_size, iob, &te->emsg)) + { + te->cont (te->cls, NULL); + te->in_next = GNUNET_NO; + return; + } + pt_block = iob; + } + else + { + pt_size = + GNUNET_FS_tree_compute_iblock_size (te->current_depth, + te->publish_offset); + pt_block = &te->chk_tree[(te->current_depth - 1) * CHK_PER_INODE]; + } + off = compute_chk_offset (te->current_depth, te->publish_offset); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "TE is at offset %llu and depth %u with block size %u and target-CHK-offset %u\n", + (unsigned long long) te->publish_offset, te->current_depth, + (unsigned int) pt_size, (unsigned int) off); + mychk = &te->chk_tree[te->current_depth * CHK_PER_INODE + off]; + GNUNET_CRYPTO_hash (pt_block, pt_size, &mychk->key); + GNUNET_CRYPTO_hash_to_aes_key (&mychk->key, &sk, &iv); + GNUNET_CRYPTO_aes_encrypt (pt_block, pt_size, &sk, &iv, enc); + GNUNET_CRYPTO_hash (enc, pt_size, &mychk->query); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "TE calculates query to be `%s', stored at %u\n", + GNUNET_h2s (&mychk->query), + te->current_depth * CHK_PER_INODE + off); + if (NULL != te->proc) + te->proc (te->cls, mychk, te->publish_offset, te->current_depth, + (0 == + te->current_depth) ? GNUNET_BLOCK_TYPE_FS_DBLOCK : + GNUNET_BLOCK_TYPE_FS_IBLOCK, enc, pt_size); + if (NULL != te->progress) + te->progress (te->cls, te->publish_offset, pt_block, pt_size, + te->current_depth); + if (0 == te->current_depth) + { + te->publish_offset += pt_size; + if ((te->publish_offset == te->size) || + (0 == te->publish_offset % (CHK_PER_INODE * DBLOCK_SIZE))) + te->current_depth++; + } + else + { + if ((off == CHK_PER_INODE) || (te->publish_offset == te->size)) + te->current_depth++; + else + te->current_depth = 0; + } + te->in_next = GNUNET_NO; +} + + +/** + * Clean up a tree encoder and return information + * about the resulting URI or an error message. + * + * @param te the tree encoder to clean up + * @param uri set to the resulting URI (if encoding finished) + * @param emsg set to an error message (if an error occured + * within the tree encoder; if this function is called + * prior to completion and prior to an internal error, + * both "*uri" and "*emsg" will be set to NULL). + */ +void +GNUNET_FS_tree_encoder_finish (struct GNUNET_FS_TreeEncoder *te, + struct GNUNET_FS_Uri **uri, char **emsg) +{ + GNUNET_assert (GNUNET_NO == te->in_next); + if (uri != NULL) + *uri = te->uri; + else if (NULL != te->uri) + GNUNET_FS_uri_destroy (te->uri); + if (emsg != NULL) + *emsg = te->emsg; + else + GNUNET_free_non_null (te->emsg); + GNUNET_free (te->chk_tree); + GNUNET_free (te); +} + +/* end of fs_tree.c */ diff --git a/src/fs/fs_tree.h b/src/fs/fs_tree.h new file mode 100644 index 0000000..5b1c202 --- /dev/null +++ b/src/fs/fs_tree.h @@ -0,0 +1,207 @@ +/* + This file is part of GNUnet. + (C) 2009 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/fs_tree.h + * @brief Merkle-tree-ish-CHK file encoding for GNUnet + * @see https://gnunet.org/encoding + * @author Krista Bennett + * @author Christian Grothoff + * + * TODO: + * - decide if this API should be made public (gnunet_fs_service.h) + * or remain "internal" (but with exported symbols?) + */ +#ifndef GNUNET_FS_TREE_H +#define GNUNET_FS_TREE_H + +#include "fs_api.h" + +/** + * Compute the depth of the CHK tree. + * + * @param flen file length for which to compute the depth + * @return depth of the tree, always > 0. A depth of 1 means only a DBLOCK. + */ +unsigned int +GNUNET_FS_compute_depth (uint64_t flen); + + +/** + * Calculate how many bytes of payload a block tree of the given + * depth MAY correspond to at most (this function ignores the fact that + * some blocks will only be present partially due to the total file + * size cutting some blocks off at the end). + * + * @param depth depth of the block. depth==0 is a DBLOCK. + * @return number of bytes of payload a subtree of this depth may correspond to + */ +uint64_t +GNUNET_FS_tree_compute_tree_size (unsigned int depth); + + +/** + * Compute how many bytes of data should be stored in + * the specified block. + * + * @param fsize overall file size, must be > 0. + * @param offset offset in the original data corresponding + * to the beginning of the tree induced by the block; + * must be < fsize + * @param depth depth of the node in the tree, 0 for DBLOCK + * @return number of bytes stored in this node + */ +size_t +GNUNET_FS_tree_calculate_block_size (uint64_t fsize, uint64_t offset, + unsigned int depth); + + +/** + * Context for an ECRS-based file encoder that computes + * the Merkle-ish-CHK tree. + */ +struct GNUNET_FS_TreeEncoder; + + +/** + * Function called asking for the current (encoded) + * block to be processed. After processing the + * client should either call "GNUNET_FS_tree_encode_next" + * or (on error) "GNUNET_FS_tree_encode_finish". + * + * @param cls closure + * @param chk content hash key for the block + * @param offset offset of the block + * @param depth depth of the block, 0 for DBLOCKs + * @param type type of the block (IBLOCK or DBLOCK) + * @param block the (encrypted) block + * @param block_size size of block (in bytes) + */ +typedef void (*GNUNET_FS_TreeBlockProcessor) (void *cls, + const struct ContentHashKey * chk, + uint64_t offset, + unsigned int depth, + enum GNUNET_BLOCK_Type type, + const void *block, + uint16_t block_size); + + +/** + * Function called with information about our + * progress in computing the tree encoding. + * + * @param cls closure + * @param offset where are we in the file + * @param pt_block plaintext of the currently processed block + * @param pt_size size of pt_block + * @param depth depth of the block in the tree, 0 for DBLOCKS + */ +typedef void (*GNUNET_FS_TreeProgressCallback) (void *cls, uint64_t offset, + const void *pt_block, + size_t pt_size, + unsigned int depth); + + +/** + * Initialize a tree encoder. This function will call "proc" and + * "progress" on each block in the tree. Once all blocks have been + * processed, "cont" will be scheduled. The "reader" will be called + * to obtain the (plaintext) blocks for the file. Note that this + * function will actually never call "proc"; the "proc" function must + * be triggered by calling "GNUNET_FS_tree_encoder_next" to trigger + * encryption (and calling of "proc") for each block. + * + * @param h the global FS context + * @param size overall size of the file to encode + * @param cls closure for reader, proc, progress and cont + * @param reader function to call to read plaintext data + * @param proc function to call on each encrypted block + * @param progress function to call with progress information + * @param cont function to call when done + * @return tree encoder context + */ +struct GNUNET_FS_TreeEncoder * +GNUNET_FS_tree_encoder_create (struct GNUNET_FS_Handle *h, uint64_t size, + void *cls, GNUNET_FS_DataReader reader, + GNUNET_FS_TreeBlockProcessor proc, + GNUNET_FS_TreeProgressCallback progress, + GNUNET_SCHEDULER_Task cont); + + +/** + * Encrypt the next block of the file (and + * call proc and progress accordingly; or + * of course "cont" if we have already completed + * encoding of the entire file). + * + * @param te tree encoder to use + */ +void +GNUNET_FS_tree_encoder_next (struct GNUNET_FS_TreeEncoder *te); + + +/** + * Clean up a tree encoder and return information + * about the resulting URI or an error message. + * + * @param te the tree encoder to clean up + * @param uri set to the resulting URI (if encoding finished) + * @param emsg set to an error message (if an error occured + * within the tree encoder; if this function is called + * prior to completion and prior to an internal error, + * both "*uri" and "*emsg" will be set to NULL). + */ +void +GNUNET_FS_tree_encoder_finish (struct GNUNET_FS_TreeEncoder *te, + struct GNUNET_FS_Uri **uri, char **emsg); + + +#if 0 +/* the functions below will be needed for persistence + but are not yet implemented -- FIXME... */ +/** + * Get data that would be needed to resume + * the encoding later. + * + * @param te encoding to resume + * @param data set to the resume data + * @param size set to the size of the resume data + */ +void +GNUNET_FS_tree_encoder_resume_get_data (const struct GNUNET_FS_TreeEncoder *te, + void **data, size_t * size); + + +/** + * Reset tree encoder to point previously + * obtained for resuming. + * + * @param te encoding to resume + * @param data the resume data + * @param size the size of the resume data + */ +void +GNUNET_FS_tree_encoder_resume (struct GNUNET_FS_TreeEncoder *te, + const void *data, size_t size); +#endif + +#endif + +/* end of fs_tree.h */ diff --git a/src/fs/fs_unindex.c b/src/fs/fs_unindex.c new file mode 100644 index 0000000..ff1996a --- /dev/null +++ b/src/fs/fs_unindex.c @@ -0,0 +1,524 @@ +/* + This file is part of GNUnet. + (C) 2003, 2004, 2006, 2009 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/fs_unindex.c + * @author Krista Grothoff + * @author Christian Grothoff + * @brief Unindex file. + */ +#include "platform.h" +#include "gnunet_constants.h" +#include "gnunet_fs_service.h" +#include "gnunet_protocols.h" +#include "fs_api.h" +#include "fs_tree.h" + + +/** + * Function called by the tree encoder to obtain + * a block of plaintext data (for the lowest level + * of the tree). + * + * @param cls our publishing context + * @param offset identifies which block to get + * @param max (maximum) number of bytes to get; returning + * fewer will also cause errors + * @param buf where to copy the plaintext buffer + * @param emsg location to store an error message (on error) + * @return number of bytes copied to buf, 0 on error + */ +static size_t +unindex_reader (void *cls, uint64_t offset, size_t max, void *buf, char **emsg) +{ + struct GNUNET_FS_UnindexContext *uc = cls; + size_t pt_size; + + pt_size = GNUNET_MIN (max, uc->file_size - offset); + if (offset != GNUNET_DISK_file_seek (uc->fh, offset, GNUNET_DISK_SEEK_SET)) + { + *emsg = GNUNET_strdup (_("Failed to find given position in file")); + return 0; + } + if (pt_size != GNUNET_DISK_file_read (uc->fh, buf, pt_size)) + { + *emsg = GNUNET_strdup (_("Failed to read file")); + return 0; + } + return pt_size; +} + + +/** + * Fill in all of the generic fields for + * an unindex event and call the callback. + * + * @param pi structure to fill in + * @param uc overall unindex context + * @param offset where we are in the file (for progress) + */ +void +GNUNET_FS_unindex_make_status_ (struct GNUNET_FS_ProgressInfo *pi, + struct GNUNET_FS_UnindexContext *uc, + uint64_t offset) +{ + pi->value.unindex.uc = uc; + pi->value.unindex.cctx = uc->client_info; + pi->value.unindex.filename = uc->filename; + pi->value.unindex.size = uc->file_size; + pi->value.unindex.eta = + GNUNET_TIME_calculate_eta (uc->start_time, offset, uc->file_size); + pi->value.unindex.duration = + GNUNET_TIME_absolute_get_duration (uc->start_time); + pi->value.unindex.completed = offset; + uc->client_info = uc->h->upcb (uc->h->upcb_cls, pi); + +} + + +/** + * Function called with information about our + * progress in computing the tree encoding. + * + * @param cls closure + * @param offset where are we in the file + * @param pt_block plaintext of the currently processed block + * @param pt_size size of pt_block + * @param depth depth of the block in the tree, 0 for DBLOCK + */ +static void +unindex_progress (void *cls, uint64_t offset, const void *pt_block, + size_t pt_size, unsigned int depth) +{ + struct GNUNET_FS_UnindexContext *uc = cls; + struct GNUNET_FS_ProgressInfo pi; + + pi.status = GNUNET_FS_STATUS_UNINDEX_PROGRESS; + pi.value.unindex.specifics.progress.data = pt_block; + pi.value.unindex.specifics.progress.offset = offset; + pi.value.unindex.specifics.progress.data_len = pt_size; + pi.value.unindex.specifics.progress.depth = depth; + GNUNET_FS_unindex_make_status_ (&pi, uc, offset); +} + + +/** + * We've encountered an error during + * unindexing. Signal the client. + * + * @param uc context for the failed unindexing operation + */ +static void +signal_unindex_error (struct GNUNET_FS_UnindexContext *uc) +{ + struct GNUNET_FS_ProgressInfo pi; + + pi.status = GNUNET_FS_STATUS_UNINDEX_ERROR; + pi.value.unindex.eta = GNUNET_TIME_UNIT_FOREVER_REL; + pi.value.unindex.specifics.error.message = uc->emsg; + GNUNET_FS_unindex_make_status_ (&pi, uc, 0); +} + + +/** + * Continuation called to notify client about result of the + * datastore removal operation. + * + * @param cls closure + * @param success GNUNET_SYSERR on failure + * @param min_expiration minimum expiration time required for content to be stored + * @param msg NULL on success, otherwise an error message + */ +static void +process_cont (void *cls, int success, struct GNUNET_TIME_Absolute min_expiration, const char *msg) +{ + struct GNUNET_FS_UnindexContext *uc = cls; + + if (success == GNUNET_SYSERR) + { + uc->emsg = GNUNET_strdup (msg); + signal_unindex_error (uc); + return; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Datastore REMOVE operation succeeded\n"); + GNUNET_FS_tree_encoder_next (uc->tc); +} + + +/** + * Function called asking for the current (encoded) + * block to be processed. After processing the + * client should either call "GNUNET_FS_tree_encode_next" + * or (on error) "GNUNET_FS_tree_encode_finish". + * + * @param cls closure + * @param chk content hash key for the block (key for lookup in the datastore) + * @param offset offset of the block + * @param depth depth of the block, 0 for DBLOCK + * @param type type of the block (IBLOCK or DBLOCK) + * @param block the (encrypted) block + * @param block_size size of block (in bytes) + */ +static void +unindex_process (void *cls, const struct ContentHashKey *chk, uint64_t offset, + unsigned int depth, enum GNUNET_BLOCK_Type type, + const void *block, uint16_t block_size) +{ + struct GNUNET_FS_UnindexContext *uc = cls; + uint32_t size; + const void *data; + struct OnDemandBlock odb; + + if (type != GNUNET_BLOCK_TYPE_FS_DBLOCK) + { + size = block_size; + data = block; + } + else /* on-demand encoded DBLOCK */ + { + size = sizeof (struct OnDemandBlock); + odb.offset = GNUNET_htonll (offset); + odb.file_id = uc->file_id; + data = &odb; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Sending REMOVE request to DATASTORE service\n"); + GNUNET_DATASTORE_remove (uc->dsh, &chk->query, size, data, -2, 1, + GNUNET_CONSTANTS_SERVICE_TIMEOUT, &process_cont, uc); +} + + +/** + * Function called with the response from the + * FS service to our unindexing request. + * + * @param cls closure, unindex context + * @param msg NULL on timeout, otherwise the response + */ +static void +process_fs_response (void *cls, const struct GNUNET_MessageHeader *msg) +{ + struct GNUNET_FS_UnindexContext *uc = cls; + struct GNUNET_FS_ProgressInfo pi; + + if (uc->client != NULL) + { + GNUNET_CLIENT_disconnect (uc->client, GNUNET_NO); + uc->client = NULL; + } + if (uc->state != UNINDEX_STATE_FS_NOTIFY) + { + uc->state = UNINDEX_STATE_ERROR; + uc->emsg = + GNUNET_strdup (_("Unexpected time for a response from `fs' service.")); + GNUNET_FS_unindex_sync_ (uc); + signal_unindex_error (uc); + return; + } + if (NULL == msg) + { + uc->state = UNINDEX_STATE_ERROR; + uc->emsg = GNUNET_strdup (_("Timeout waiting for `fs' service.")); + GNUNET_FS_unindex_sync_ (uc); + signal_unindex_error (uc); + return; + } + if (ntohs (msg->type) != GNUNET_MESSAGE_TYPE_FS_UNINDEX_OK) + { + uc->state = UNINDEX_STATE_ERROR; + uc->emsg = GNUNET_strdup (_("Invalid response from `fs' service.")); + GNUNET_FS_unindex_sync_ (uc); + signal_unindex_error (uc); + return; + } + uc->state = UNINDEX_STATE_COMPLETE; + pi.status = GNUNET_FS_STATUS_UNINDEX_COMPLETED; + pi.value.unindex.eta = GNUNET_TIME_UNIT_ZERO; + GNUNET_FS_unindex_sync_ (uc); + GNUNET_FS_unindex_make_status_ (&pi, uc, uc->file_size); +} + + +/** + * Function called when the tree encoder has + * processed all blocks. Clean up. + * + * @param cls our unindexing context + * @param tc not used + */ +static void +unindex_finish (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct GNUNET_FS_UnindexContext *uc = cls; + char *emsg; + struct GNUNET_FS_Uri *uri; + struct UnindexMessage req; + + /* generate final progress message */ + unindex_progress (uc, uc->file_size, NULL, 0, 0); + GNUNET_FS_tree_encoder_finish (uc->tc, &uri, &emsg); + uc->tc = NULL; + if (uri != NULL) + GNUNET_FS_uri_destroy (uri); + GNUNET_DISK_file_close (uc->fh); + uc->fh = NULL; + GNUNET_DATASTORE_disconnect (uc->dsh, GNUNET_NO); + uc->dsh = NULL; + uc->state = UNINDEX_STATE_FS_NOTIFY; + GNUNET_FS_unindex_sync_ (uc); + uc->client = GNUNET_CLIENT_connect ("fs", uc->h->cfg); + if (uc->client == NULL) + { + uc->state = UNINDEX_STATE_ERROR; + uc->emsg = + GNUNET_strdup (_("Failed to connect to FS service for unindexing.")); + GNUNET_FS_unindex_sync_ (uc); + signal_unindex_error (uc); + return; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Sending UNINDEX message to FS service\n"); + req.header.size = htons (sizeof (struct UnindexMessage)); + req.header.type = htons (GNUNET_MESSAGE_TYPE_FS_UNINDEX); + req.reserved = 0; + req.file_id = uc->file_id; + GNUNET_break (GNUNET_OK == + GNUNET_CLIENT_transmit_and_get_response (uc->client, + &req.header, + GNUNET_CONSTANTS_SERVICE_TIMEOUT, + GNUNET_YES, + &process_fs_response, + uc)); +} + + +/** + * Connect to the datastore and remove the blocks. + * + * @param uc context for the unindex operation. + */ +void +GNUNET_FS_unindex_do_remove_ (struct GNUNET_FS_UnindexContext *uc) +{ + uc->dsh = GNUNET_DATASTORE_connect (uc->h->cfg); + if (NULL == uc->dsh) + { + uc->state = UNINDEX_STATE_ERROR; + uc->emsg = GNUNET_strdup (_("Failed to connect to `datastore' service.")); + GNUNET_FS_unindex_sync_ (uc); + signal_unindex_error (uc); + return; + } + uc->fh = + GNUNET_DISK_file_open (uc->filename, GNUNET_DISK_OPEN_READ, + GNUNET_DISK_PERM_NONE); + if (NULL == uc->fh) + { + GNUNET_DATASTORE_disconnect (uc->dsh, GNUNET_NO); + uc->dsh = NULL; + uc->state = UNINDEX_STATE_ERROR; + uc->emsg = GNUNET_strdup (_("Failed to open file for unindexing.")); + GNUNET_FS_unindex_sync_ (uc); + signal_unindex_error (uc); + return; + } + uc->tc = + GNUNET_FS_tree_encoder_create (uc->h, uc->file_size, uc, &unindex_reader, + &unindex_process, &unindex_progress, + &unindex_finish); + GNUNET_FS_tree_encoder_next (uc->tc); +} + + +/** + * Function called once the hash of the file + * that is being unindexed has been computed. + * + * @param cls closure, unindex context + * @param file_id computed hash, NULL on error + */ +void +GNUNET_FS_unindex_process_hash_ (void *cls, const GNUNET_HashCode * file_id) +{ + struct GNUNET_FS_UnindexContext *uc = cls; + + uc->fhc = NULL; + if (uc->state != UNINDEX_STATE_HASHING) + { + GNUNET_FS_unindex_stop (uc); + return; + } + if (file_id == NULL) + { + uc->state = UNINDEX_STATE_ERROR; + uc->emsg = GNUNET_strdup (_("Failed to compute hash of file.")); + GNUNET_FS_unindex_sync_ (uc); + signal_unindex_error (uc); + return; + } + uc->file_id = *file_id; + uc->state = UNINDEX_STATE_DS_REMOVE; + GNUNET_FS_unindex_sync_ (uc); + GNUNET_FS_unindex_do_remove_ (uc); +} + + +/** + * Create SUSPEND event for the given unindex operation + * and then clean up our state (without stop signal). + * + * @param cls the 'struct GNUNET_FS_UnindexContext' to signal for + */ +void +GNUNET_FS_unindex_signal_suspend_ (void *cls) +{ + struct GNUNET_FS_UnindexContext *uc = cls; + struct GNUNET_FS_ProgressInfo pi; + + if (uc->fhc != NULL) + { + GNUNET_CRYPTO_hash_file_cancel (uc->fhc); + uc->fhc = NULL; + } + if (uc->client != NULL) + { + GNUNET_CLIENT_disconnect (uc->client, GNUNET_NO); + uc->client = NULL; + } + if (NULL != uc->dsh) + { + GNUNET_DATASTORE_disconnect (uc->dsh, GNUNET_NO); + uc->dsh = NULL; + } + if (NULL != uc->tc) + { + GNUNET_FS_tree_encoder_finish (uc->tc, NULL, NULL); + uc->tc = NULL; + } + if (uc->fh != NULL) + { + GNUNET_DISK_file_close (uc->fh); + uc->fh = NULL; + } + GNUNET_FS_end_top (uc->h, uc->top); + pi.status = GNUNET_FS_STATUS_UNINDEX_SUSPEND; + GNUNET_FS_unindex_make_status_ (&pi, uc, + (uc->state == + UNINDEX_STATE_COMPLETE) ? uc->file_size : 0); + GNUNET_break (NULL == uc->client_info); + GNUNET_free (uc->filename); + GNUNET_free_non_null (uc->serialization); + GNUNET_free_non_null (uc->emsg); + GNUNET_free (uc); +} + + +/** + * Unindex a file. + * + * @param h handle to the file sharing subsystem + * @param filename file to unindex + * @param cctx initial value for the client context + * @return NULL on error, otherwise handle + */ +struct GNUNET_FS_UnindexContext * +GNUNET_FS_unindex_start (struct GNUNET_FS_Handle *h, const char *filename, + void *cctx) +{ + struct GNUNET_FS_UnindexContext *ret; + struct GNUNET_FS_ProgressInfo pi; + uint64_t size; + + if (GNUNET_OK != GNUNET_DISK_file_size (filename, &size, GNUNET_YES)) + return NULL; + ret = GNUNET_malloc (sizeof (struct GNUNET_FS_UnindexContext)); + ret->h = h; + ret->filename = GNUNET_strdup (filename); + ret->start_time = GNUNET_TIME_absolute_get (); + ret->file_size = size; + ret->client_info = cctx; + GNUNET_FS_unindex_sync_ (ret); + pi.status = GNUNET_FS_STATUS_UNINDEX_START; + pi.value.unindex.eta = GNUNET_TIME_UNIT_FOREVER_REL; + GNUNET_FS_unindex_make_status_ (&pi, ret, 0); + ret->fhc = + GNUNET_CRYPTO_hash_file (GNUNET_SCHEDULER_PRIORITY_IDLE, filename, + HASHING_BLOCKSIZE, + &GNUNET_FS_unindex_process_hash_, ret); + ret->top = GNUNET_FS_make_top (h, &GNUNET_FS_unindex_signal_suspend_, ret); + return ret; +} + + +/** + * Clean up after completion of an unindex operation. + * + * @param uc handle + */ +void +GNUNET_FS_unindex_stop (struct GNUNET_FS_UnindexContext *uc) +{ + struct GNUNET_FS_ProgressInfo pi; + + if (uc->fhc != NULL) + { + GNUNET_CRYPTO_hash_file_cancel (uc->fhc); + uc->fhc = NULL; + } + if (uc->client != NULL) + { + GNUNET_CLIENT_disconnect (uc->client, GNUNET_NO); + uc->client = NULL; + } + if (NULL != uc->dsh) + { + GNUNET_DATASTORE_disconnect (uc->dsh, GNUNET_NO); + uc->dsh = NULL; + } + if (NULL != uc->tc) + { + GNUNET_FS_tree_encoder_finish (uc->tc, NULL, NULL); + uc->tc = NULL; + } + if (uc->fh != NULL) + { + GNUNET_DISK_file_close (uc->fh); + uc->fh = NULL; + } + GNUNET_FS_end_top (uc->h, uc->top); + if (uc->serialization != NULL) + { + GNUNET_FS_remove_sync_file_ (uc->h, GNUNET_FS_SYNC_PATH_MASTER_UNINDEX, + uc->serialization); + GNUNET_free (uc->serialization); + uc->serialization = NULL; + } + pi.status = GNUNET_FS_STATUS_UNINDEX_STOPPED; + pi.value.unindex.eta = GNUNET_TIME_UNIT_ZERO; + GNUNET_FS_unindex_make_status_ (&pi, uc, + (uc->state == + UNINDEX_STATE_COMPLETE) ? uc->file_size : 0); + GNUNET_break (NULL == uc->client_info); + GNUNET_free (uc->filename); + GNUNET_free (uc); +} + +/* end of fs_unindex.c */ diff --git a/src/fs/fs_uri.c b/src/fs/fs_uri.c new file mode 100644 index 0000000..5dfdcb5 --- /dev/null +++ b/src/fs/fs_uri.c @@ -0,0 +1,2084 @@ +/* + This file is part of GNUnet. + (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/fs_uri.c + * @brief Parses and produces uri strings. + * @author Igor Wronsky, Christian Grothoff + * + * GNUnet URIs are of the general form "gnunet://MODULE/IDENTIFIER". + * The specific structure of "IDENTIFIER" depends on the module and + * maybe differenciated into additional subcategories if applicable. + * This module only deals with fs identifiers (MODULE = "fs"). + * <p> + * + * This module only parses URIs for the AFS module. The FS URIs fall + * into four categories, "chk", "sks", "ksk" and "loc". The first three + * categories were named in analogy (!) to Freenet, but they do NOT + * work in exactly the same way. They are very similar from the user's + * point of view (unique file identifier, subspace, keyword), but the + * implementation is rather different in pretty much every detail. + * The concrete URI formats are: + * + * <ul><li> + * + * First, there are URIs that identify a file. They have the format + * "gnunet://fs/chk/HEX1.HEX2.SIZE". These URIs can be used to + * download the file. The description, filename, mime-type and other + * meta-data is NOT part of the file-URI since a URI uniquely + * identifies a resource (and the contents of the file would be the + * same even if it had a different description). + * + * </li><li> + * + * The second category identifies entries in a namespace. The format + * is "gnunet://fs/sks/NAMESPACE/IDENTIFIER" where the namespace + * should be given in HEX. Applications may allow using a nickname + * for the namespace if the nickname is not ambiguous. The identifier + * can be either an ASCII sequence or a HEX-encoding. If the + * identifier is in ASCII but the format is ambiguous and could denote + * a HEX-string a "/" is appended to indicate ASCII encoding. + * + * </li> <li> + * + * The third category identifies ordinary searches. The format is + * "gnunet://fs/ksk/KEYWORD[+KEYWORD]*". Using the "+" syntax + * it is possible to encode searches with the boolean "AND" operator. + * "+" is used since it indicates a commutative 'and' operation and + * is unlikely to be used in a keyword by itself. + * + * </li><li> + * + * The last category identifies a datum on a specific machine. The + * format is "gnunet://fs/loc/HEX1.HEX2.SIZE.PEER.SIG.EXPTIME". PEER is + * the BinName of the public key of the peer storing the datum. The + * signature (SIG) certifies that this peer has this content. + * HEX1, HEX2 and SIZE correspond to a 'chk' URI. + * + * </li></ul> + * + * The encoding for hexadecimal values is defined in the hashing.c + * module in the gnunetutil library and discussed there. + * <p> + */ +#include "platform.h" +#include "gnunet_fs_service.h" +#include "gnunet_signatures.h" +#include "fs_api.h" +#include <unitypes.h> +#include <unicase.h> +#include <uniconv.h> +#include <unistr.h> +#include <unistdio.h> + + + +/** + * Get a unique key from a URI. This is for putting URIs + * into HashMaps. The key may change between FS implementations. + * + * @param uri uri to convert to a unique key + * @param key wherer to store the unique key + */ +void +GNUNET_FS_uri_to_key (const struct GNUNET_FS_Uri *uri, GNUNET_HashCode * key) +{ + switch (uri->type) + { + case chk: + *key = uri->data.chk.chk.query; + return; + case sks: + GNUNET_CRYPTO_hash (uri->data.sks.identifier, + strlen (uri->data.sks.identifier), key); + break; + case ksk: + if (uri->data.ksk.keywordCount > 0) + GNUNET_CRYPTO_hash (uri->data.ksk.keywords[0], + strlen (uri->data.ksk.keywords[0]), key); + break; + case loc: + GNUNET_CRYPTO_hash (&uri->data.loc.fi, + sizeof (struct FileIdentifier) + + sizeof (struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded), + key); + break; + default: + memset (key, 0, sizeof (GNUNET_HashCode)); + break; + } +} + + +/** + * Convert keyword URI to a human readable format + * (i.e. the search query that was used in the first place) + * + * @param uri ksk uri to convert to a string + * @return string with the keywords + */ +char * +GNUNET_FS_uri_ksk_to_string_fancy (const struct GNUNET_FS_Uri *uri) +{ + size_t n; + char *ret; + unsigned int i; + const char *keyword; + char **keywords; + unsigned int keywordCount; + + if ((uri == NULL) || (uri->type != ksk)) + { + GNUNET_break (0); + return NULL; + } + keywords = uri->data.ksk.keywords; + keywordCount = uri->data.ksk.keywordCount; + n = keywordCount + 1; + for (i = 0; i < keywordCount; i++) + { + keyword = keywords[i]; + n += strlen (keyword) - 1; + if (NULL != strstr (&keyword[1], " ")) + n += 2; + if (keyword[0] == '+') + n++; + } + ret = GNUNET_malloc (n); + strcpy (ret, ""); + for (i = 0; i < keywordCount; i++) + { + keyword = keywords[i]; + if (NULL != strstr (&keyword[1], " ")) + { + strcat (ret, "\""); + if (keyword[0] == '+') + strcat (ret, keyword); + else + strcat (ret, &keyword[1]); + strcat (ret, "\""); + } + else + { + if (keyword[0] == '+') + strcat (ret, keyword); + else + strcat (ret, &keyword[1]); + } + strcat (ret, " "); + } + return ret; +} + + +/** + * Given a keyword with %-encoding (and possibly quotes to protect + * spaces), return a copy of the keyword without %-encoding and + * without double-quotes (%22). Also, add a space at the beginning + * if there is not a '+'. + * + * @param in string with %-encoding + * @param emsg where to store the parser error message (if any) + * @return decodded string with leading space (or preserved plus) + */ +static char * +percent_decode_keyword (const char *in, char **emsg) +{ + char *out; + char *ret; + unsigned int rpos; + unsigned int wpos; + unsigned int hx; + + out = GNUNET_strdup (in); + rpos = 0; + wpos = 0; + while (out[rpos] != '\0') + { + if (out[rpos] == '%') + { + if (1 != sscanf (&out[rpos + 1], "%2X", &hx)) + { + GNUNET_free (out); + *emsg = GNUNET_strdup (_("`%' must be followed by HEX number")); + return NULL; + } + rpos += 3; + if (hx == '"') + continue; /* skip double quote */ + out[wpos++] = (char) hx; + } + else + { + out[wpos++] = out[rpos++]; + } + } + out[wpos] = '\0'; + if (out[0] == '+') + { + ret = GNUNET_strdup (out); + } + else + { + /* need to prefix with space */ + ret = GNUNET_malloc (strlen (out) + 2); + strcpy (ret, " "); + strcat (ret, out); + } + GNUNET_free (out); + return ret; +} + +#define GNUNET_FS_URI_KSK_PREFIX GNUNET_FS_URI_PREFIX GNUNET_FS_URI_KSK_INFIX + +/** + * Parse a KSK URI. + * + * @param s an uri string + * @param emsg where to store the parser error message (if any) + * @return NULL on error, otherwise the KSK URI + */ +static struct GNUNET_FS_Uri * +uri_ksk_parse (const char *s, char **emsg) +{ + struct GNUNET_FS_Uri *ret; + char **keywords; + unsigned int pos; + int max; + int iret; + int i; + size_t slen; + char *dup; + int saw_quote; + + GNUNET_assert (s != NULL); + slen = strlen (s); + pos = strlen (GNUNET_FS_URI_KSK_PREFIX); + if ((slen <= pos) || (0 != strncmp (s, GNUNET_FS_URI_KSK_PREFIX, pos))) + return NULL; /* not KSK URI */ + if ((s[slen - 1] == '+') || (s[pos] == '+')) + { + *emsg = + GNUNET_strdup (_("Malformed KSK URI (must not begin or end with `+')")); + return NULL; + } + max = 1; + saw_quote = 0; + for (i = pos; i < slen; i++) + { + if ((s[i] == '%') && (&s[i] == strstr (&s[i], "%22"))) + { + saw_quote = (saw_quote + 1) % 2; + i += 3; + continue; + } + if ((s[i] == '+') && (saw_quote == 0)) + { + max++; + if (s[i - 1] == '+') + { + *emsg = GNUNET_strdup (_("`++' not allowed in KSK URI")); + return NULL; + } + } + } + if (saw_quote == 1) + { + *emsg = GNUNET_strdup (_("Quotes not balanced in KSK URI")); + return NULL; + } + iret = max; + dup = GNUNET_strdup (s); + keywords = GNUNET_malloc (max * sizeof (char *)); + for (i = slen - 1; i >= pos; i--) + { + if ((s[i] == '%') && (&s[i] == strstr (&s[i], "%22"))) + { + saw_quote = (saw_quote + 1) % 2; + i += 3; + continue; + } + if ((dup[i] == '+') && (saw_quote == 0)) + { + keywords[--max] = percent_decode_keyword (&dup[i + 1], emsg); + if (NULL == keywords[max]) + goto CLEANUP; + dup[i] = '\0'; + } + } + keywords[--max] = percent_decode_keyword (&dup[pos], emsg); + if (NULL == keywords[max]) + goto CLEANUP; + GNUNET_assert (max == 0); + GNUNET_free (dup); + ret = GNUNET_malloc (sizeof (struct GNUNET_FS_Uri)); + ret->type = ksk; + ret->data.ksk.keywordCount = iret; + ret->data.ksk.keywords = keywords; + return ret; +CLEANUP: + for (i = 0; i < max; i++) + GNUNET_free_non_null (keywords[i]); + GNUNET_free (keywords); + GNUNET_free (dup); + return NULL; +} + + +#define GNUNET_FS_URI_SKS_PREFIX GNUNET_FS_URI_PREFIX GNUNET_FS_URI_SKS_INFIX + +/** + * Parse an SKS URI. + * + * @param s an uri string + * @param emsg where to store the parser error message (if any) + * @return NULL on error, SKS URI otherwise + */ +static struct GNUNET_FS_Uri * +uri_sks_parse (const char *s, char **emsg) +{ + struct GNUNET_FS_Uri *ret; + GNUNET_HashCode namespace; + char *identifier; + unsigned int pos; + size_t slen; + char enc[sizeof (struct GNUNET_CRYPTO_HashAsciiEncoded)]; + + GNUNET_assert (s != NULL); + slen = strlen (s); + pos = strlen (GNUNET_FS_URI_SKS_PREFIX); + if ((slen <= pos) || (0 != strncmp (s, GNUNET_FS_URI_SKS_PREFIX, pos))) + return NULL; /* not an SKS URI */ + if ((slen < pos + sizeof (struct GNUNET_CRYPTO_HashAsciiEncoded)) || + (s[pos + sizeof (struct GNUNET_CRYPTO_HashAsciiEncoded) - 1] != '/')) + { + *emsg = GNUNET_strdup (_("Malformed SKS URI")); + return NULL; + } + memcpy (enc, &s[pos], sizeof (struct GNUNET_CRYPTO_HashAsciiEncoded)); + enc[sizeof (struct GNUNET_CRYPTO_HashAsciiEncoded) - 1] = '\0'; + if (GNUNET_OK != GNUNET_CRYPTO_hash_from_string (enc, &namespace)) + { + *emsg = GNUNET_strdup (_("Malformed SKS URI")); + return NULL; + } + identifier = + GNUNET_strdup (&s[pos + sizeof (struct GNUNET_CRYPTO_HashAsciiEncoded)]); + ret = GNUNET_malloc (sizeof (struct GNUNET_FS_Uri)); + ret->type = sks; + ret->data.sks.namespace = namespace; + ret->data.sks.identifier = identifier; + return ret; +} + +#define GNUNET_FS_URI_CHK_PREFIX GNUNET_FS_URI_PREFIX GNUNET_FS_URI_CHK_INFIX + + +/** + * Parse a CHK URI. + * + * @param s an uri string + * @param emsg where to store the parser error message (if any) + * @return NULL on error, CHK URI otherwise + */ +static struct GNUNET_FS_Uri * +uri_chk_parse (const char *s, char **emsg) +{ + struct GNUNET_FS_Uri *ret; + struct FileIdentifier fi; + unsigned int pos; + unsigned long long flen; + size_t slen; + char h1[sizeof (struct GNUNET_CRYPTO_HashAsciiEncoded)]; + char h2[sizeof (struct GNUNET_CRYPTO_HashAsciiEncoded)]; + + if (NULL == s) + return NULL; + GNUNET_assert (s != NULL); + slen = strlen (s); + pos = strlen (GNUNET_FS_URI_CHK_PREFIX); + if ((slen < pos + 2 * sizeof (struct GNUNET_CRYPTO_HashAsciiEncoded) + 1) || + (0 != strncmp (s, GNUNET_FS_URI_CHK_PREFIX, pos))) + return NULL; /* not a CHK URI */ + if ((s[pos + sizeof (struct GNUNET_CRYPTO_HashAsciiEncoded) - 1] != '.') || + (s[pos + sizeof (struct GNUNET_CRYPTO_HashAsciiEncoded) * 2 - 1] != '.')) + { + *emsg = GNUNET_strdup (_("Malformed CHK URI")); + return NULL; + } + memcpy (h1, &s[pos], sizeof (struct GNUNET_CRYPTO_HashAsciiEncoded)); + h1[sizeof (struct GNUNET_CRYPTO_HashAsciiEncoded) - 1] = '\0'; + memcpy (h2, &s[pos + sizeof (struct GNUNET_CRYPTO_HashAsciiEncoded)], + sizeof (struct GNUNET_CRYPTO_HashAsciiEncoded)); + h2[sizeof (struct GNUNET_CRYPTO_HashAsciiEncoded) - 1] = '\0'; + + if ((GNUNET_OK != GNUNET_CRYPTO_hash_from_string (h1, &fi.chk.key)) || + (GNUNET_OK != GNUNET_CRYPTO_hash_from_string (h2, &fi.chk.query)) || + (1 != + SSCANF (&s[pos + sizeof (struct GNUNET_CRYPTO_HashAsciiEncoded) * 2], + "%llu", &flen))) + { + *emsg = GNUNET_strdup (_("Malformed CHK URI")); + return NULL; + } + fi.file_length = GNUNET_htonll (flen); + ret = GNUNET_malloc (sizeof (struct GNUNET_FS_Uri)); + ret->type = chk; + ret->data.chk = fi; + return ret; +} + + +/** + * Convert a character back to the binary value + * that it represents (given base64-encoding). + * + * @param a character to convert + * @return offset in the "tbl" array + */ +static unsigned int +c2v (unsigned char a) +{ + if ((a >= '0') && (a <= '9')) + return a - '0'; + if ((a >= 'A') && (a <= 'Z')) + return (a - 'A' + 10); + if ((a >= 'a') && (a <= 'z')) + return (a - 'a' + 36); + if (a == '_') + return 62; + if (a == '=') + return 63; + return -1; +} + + +/** + * Convert string back to binary data. + * + * @param input '\\0'-terminated string + * @param data where to write binary data + * @param size how much data should be converted + * @return number of characters processed from input, + * -1 on error + */ +static int +enc2bin (const char *input, void *data, size_t size) +{ + size_t len; + size_t pos; + unsigned int bits; + unsigned int hbits; + + len = size * 8 / 6; + if (((size * 8) % 6) != 0) + len++; + if (strlen (input) < len) + return -1; /* error! */ + bits = 0; + hbits = 0; + len = 0; + for (pos = 0; pos < size; pos++) + { + while (hbits < 8) + { + bits |= (c2v (input[len++]) << hbits); + hbits += 6; + } + (((unsigned char *) data)[pos]) = (unsigned char) bits; + bits >>= 8; + hbits -= 8; + } + return len; +} + + +/** + * Structure that defines how the + * contents of a location URI must be + * assembled in memory to create or + * verify the signature of a location + * URI. + */ +struct LocUriAssembly +{ + struct GNUNET_CRYPTO_RsaSignaturePurpose purpose; + + struct GNUNET_TIME_AbsoluteNBO exptime; + + struct FileIdentifier fi; + + struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded peer; + +}; + + +#define GNUNET_FS_URI_LOC_PREFIX GNUNET_FS_URI_PREFIX GNUNET_FS_URI_LOC_INFIX + +/** + * Parse a LOC URI. + * Also verifies validity of the location URI. + * + * @param s an uri string + * @param emsg where to store the parser error message (if any) + * @return NULL on error, valid LOC URI otherwise + */ +static struct GNUNET_FS_Uri * +uri_loc_parse (const char *s, char **emsg) +{ + struct GNUNET_FS_Uri *uri; + char h1[sizeof (struct GNUNET_CRYPTO_HashAsciiEncoded)]; + char h2[sizeof (struct GNUNET_CRYPTO_HashAsciiEncoded)]; + unsigned int pos; + unsigned int npos; + unsigned long long exptime; + unsigned long long flen; + struct GNUNET_TIME_Absolute et; + struct GNUNET_CRYPTO_RsaSignature sig; + struct LocUriAssembly ass; + int ret; + size_t slen; + + GNUNET_assert (s != NULL); + slen = strlen (s); + pos = strlen (GNUNET_FS_URI_LOC_PREFIX); + if ((slen < pos + 2 * sizeof (struct GNUNET_CRYPTO_HashAsciiEncoded) + 1) || + (0 != strncmp (s, GNUNET_FS_URI_LOC_PREFIX, pos))) + return NULL; /* not an SKS URI */ + if ((s[pos + sizeof (struct GNUNET_CRYPTO_HashAsciiEncoded) - 1] != '.') || + (s[pos + sizeof (struct GNUNET_CRYPTO_HashAsciiEncoded) * 2 - 1] != '.')) + { + *emsg = GNUNET_strdup (_("SKS URI malformed")); + return NULL; + } + memcpy (h1, &s[pos], sizeof (struct GNUNET_CRYPTO_HashAsciiEncoded)); + h1[sizeof (struct GNUNET_CRYPTO_HashAsciiEncoded) - 1] = '\0'; + memcpy (h2, &s[pos + sizeof (struct GNUNET_CRYPTO_HashAsciiEncoded)], + sizeof (struct GNUNET_CRYPTO_HashAsciiEncoded)); + h2[sizeof (struct GNUNET_CRYPTO_HashAsciiEncoded) - 1] = '\0'; + + if ((GNUNET_OK != GNUNET_CRYPTO_hash_from_string (h1, &ass.fi.chk.key)) || + (GNUNET_OK != GNUNET_CRYPTO_hash_from_string (h2, &ass.fi.chk.query)) || + (1 != + SSCANF (&s[pos + sizeof (struct GNUNET_CRYPTO_HashAsciiEncoded) * 2], + "%llu", &flen))) + { + *emsg = GNUNET_strdup (_("SKS URI malformed")); + return NULL; + } + ass.fi.file_length = GNUNET_htonll (flen); + + npos = pos + sizeof (struct GNUNET_CRYPTO_HashAsciiEncoded) * 2; + while ((s[npos] != '\0') && (s[npos] != '.')) + npos++; + if (s[npos] == '\0') + { + *emsg = GNUNET_strdup (_("SKS URI malformed")); + goto ERR; + } + npos++; + ret = + enc2bin (&s[npos], &ass.peer, + sizeof (struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded)); + if (ret == -1) + { + *emsg = + GNUNET_strdup (_("SKS URI malformed (could not decode public key)")); + goto ERR; + } + npos += ret; + if (s[npos++] != '.') + { + *emsg = GNUNET_strdup (_("SKS URI malformed (could not find signature)")); + goto ERR; + } + ret = enc2bin (&s[npos], &sig, sizeof (struct GNUNET_CRYPTO_RsaSignature)); + if (ret == -1) + { + *emsg = GNUNET_strdup (_("SKS URI malformed (could not decode signature)")); + goto ERR; + } + npos += ret; + if (s[npos++] != '.') + { + *emsg = GNUNET_strdup (_("SKS URI malformed")); + goto ERR; + } + if (1 != SSCANF (&s[npos], "%llu", &exptime)) + { + *emsg = + GNUNET_strdup (_ + ("SKS URI malformed (could not parse expiration time)")); + goto ERR; + } + ass.purpose.size = htonl (sizeof (struct LocUriAssembly)); + ass.purpose.purpose = htonl (GNUNET_SIGNATURE_PURPOSE_PEER_PLACEMENT); + et.abs_value = exptime; + ass.exptime = GNUNET_TIME_absolute_hton (et); + if (GNUNET_OK != + GNUNET_CRYPTO_rsa_verify (GNUNET_SIGNATURE_PURPOSE_PEER_PLACEMENT, + &ass.purpose, &sig, &ass.peer)) + { + *emsg = + GNUNET_strdup (_("SKS URI malformed (signature failed validation)")); + goto ERR; + } + uri = GNUNET_malloc (sizeof (struct GNUNET_FS_Uri)); + uri->type = loc; + uri->data.loc.fi = ass.fi; + uri->data.loc.peer = ass.peer; + uri->data.loc.expirationTime = et; + uri->data.loc.contentSignature = sig; + + return uri; +ERR: + return NULL; +} + + +/** + * Convert a UTF-8 String to a URI. + * + * @param uri string to parse + * @param emsg where to store the parser error message (if any) + * @return NULL on error + */ +struct GNUNET_FS_Uri * +GNUNET_FS_uri_parse (const char *uri, char **emsg) +{ + struct GNUNET_FS_Uri *ret; + char *msg; + + if (NULL == emsg) + emsg = &msg; + *emsg = NULL; + if ((NULL != (ret = uri_chk_parse (uri, emsg))) || + (NULL != (ret = uri_ksk_parse (uri, emsg))) || + (NULL != (ret = uri_sks_parse (uri, emsg))) || + (NULL != (ret = uri_loc_parse (uri, emsg)))) + return ret; + if (NULL == *emsg) + *emsg = GNUNET_strdup (_("Unrecognized URI type")); + if (emsg == &msg) + GNUNET_free (msg); + return NULL; +} + + +/** + * Free URI. + * + * @param uri uri to free + */ +void +GNUNET_FS_uri_destroy (struct GNUNET_FS_Uri *uri) +{ + unsigned int i; + + GNUNET_assert (uri != NULL); + switch (uri->type) + { + case ksk: + for (i = 0; i < uri->data.ksk.keywordCount; i++) + GNUNET_free (uri->data.ksk.keywords[i]); + GNUNET_array_grow (uri->data.ksk.keywords, uri->data.ksk.keywordCount, 0); + break; + case sks: + GNUNET_free (uri->data.sks.identifier); + break; + case loc: + break; + default: + /* do nothing */ + break; + } + GNUNET_free (uri); +} + +/** + * How many keywords are ANDed in this keyword URI? + * + * @param uri ksk uri to get the number of keywords from + * @return 0 if this is not a keyword URI + */ +unsigned int +GNUNET_FS_uri_ksk_get_keyword_count (const struct GNUNET_FS_Uri *uri) +{ + if (uri->type != ksk) + return 0; + return uri->data.ksk.keywordCount; +} + + +/** + * Iterate over all keywords in this keyword URI. + * + * @param uri ksk uri to get the keywords from + * @param iterator function to call on each keyword + * @param iterator_cls closure for iterator + * @return -1 if this is not a keyword URI, otherwise number of + * keywords iterated over until iterator aborted + */ +int +GNUNET_FS_uri_ksk_get_keywords (const struct GNUNET_FS_Uri *uri, + GNUNET_FS_KeywordIterator iterator, + void *iterator_cls) +{ + unsigned int i; + char *keyword; + + if (uri->type != ksk) + return -1; + if (iterator == NULL) + return uri->data.ksk.keywordCount; + for (i = 0; i < uri->data.ksk.keywordCount; i++) + { + keyword = uri->data.ksk.keywords[i]; + /* first character of keyword indicates + * if it is mandatory or not */ + if (GNUNET_OK != iterator (iterator_cls, &keyword[1], keyword[0] == '+')) + return i; + } + return i; +} + + +/** + * Add the given keyword to the set of keywords represented by the URI. + * Does nothing if the keyword is already present. + * + * @param uri ksk uri to modify + * @param keyword keyword to add + * @param is_mandatory is this keyword mandatory? + */ +void +GNUNET_FS_uri_ksk_add_keyword (struct GNUNET_FS_Uri *uri, const char *keyword, + int is_mandatory) +{ + unsigned int i; + const char *old; + char *n; + + GNUNET_assert (uri->type == ksk); + for (i = 0; i < uri->data.ksk.keywordCount; i++) + { + old = uri->data.ksk.keywords[i]; + if (0 == strcmp (&old[1], keyword)) + return; + } + GNUNET_asprintf (&n, is_mandatory ? "+%s" : " %s", keyword); + GNUNET_array_append (uri->data.ksk.keywords, uri->data.ksk.keywordCount, n); +} + + +/** + * Remove the given keyword from the set of keywords represented by the URI. + * Does nothing if the keyword is not present. + * + * @param uri ksk uri to modify + * @param keyword keyword to add + */ +void +GNUNET_FS_uri_ksk_remove_keyword (struct GNUNET_FS_Uri *uri, + const char *keyword) +{ + unsigned int i; + char *old; + + GNUNET_assert (uri->type == ksk); + for (i = 0; i < uri->data.ksk.keywordCount; i++) + { + old = uri->data.ksk.keywords[i]; + if (0 == strcmp (&old[1], keyword)) + { + uri->data.ksk.keywords[i] = + uri->data.ksk.keywords[uri->data.ksk.keywordCount - 1]; + GNUNET_array_grow (uri->data.ksk.keywords, uri->data.ksk.keywordCount, + uri->data.ksk.keywordCount - 1); + GNUNET_free (old); + return; + } + } +} + + +/** + * Obtain the identity of the peer offering the data + * + * @param uri the location URI to inspect + * @param peer where to store the identify of the peer (presumably) offering the content + * @return GNUNET_SYSERR if this is not a location URI, otherwise GNUNET_OK + */ +int +GNUNET_FS_uri_loc_get_peer_identity (const struct GNUNET_FS_Uri *uri, + struct GNUNET_PeerIdentity *peer) +{ + if (uri->type != loc) + return GNUNET_SYSERR; + GNUNET_CRYPTO_hash (&uri->data.loc.peer, + sizeof (struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded), + &peer->hashPubKey); + return GNUNET_OK; +} + + +/** + * Obtain the expiration of the LOC URI. + * + * @param uri location URI to get the expiration from + * @return expiration time of the URI + */ +struct GNUNET_TIME_Absolute +GNUNET_FS_uri_loc_get_expiration (const struct GNUNET_FS_Uri *uri) +{ + GNUNET_assert (uri->type == loc); + return uri->data.loc.expirationTime; +} + + + +/** + * Obtain the URI of the content itself. + * + * @param uri location URI to get the content URI from + * @return NULL if argument is not a location URI + */ +struct GNUNET_FS_Uri * +GNUNET_FS_uri_loc_get_uri (const struct GNUNET_FS_Uri *uri) +{ + struct GNUNET_FS_Uri *ret; + + if (uri->type != loc) + return NULL; + ret = GNUNET_malloc (sizeof (struct GNUNET_FS_Uri)); + ret->type = chk; + ret->data.chk = uri->data.loc.fi; + return ret; +} + + +/** + * Construct a location URI (this peer will be used for the location). + * + * @param baseUri content offered by the sender + * @param cfg configuration information (used to find our hostkey) + * @param expiration_time how long will the content be offered? + * @return the location URI, NULL on error + */ +struct GNUNET_FS_Uri * +GNUNET_FS_uri_loc_create (const struct GNUNET_FS_Uri *baseUri, + const struct GNUNET_CONFIGURATION_Handle *cfg, + struct GNUNET_TIME_Absolute expiration_time) +{ + struct GNUNET_FS_Uri *uri; + struct GNUNET_CRYPTO_RsaPrivateKey *my_private_key; + struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded my_public_key; + char *keyfile; + struct LocUriAssembly ass; + + if (baseUri->type != chk) + return NULL; + if (GNUNET_OK != + GNUNET_CONFIGURATION_get_value_filename (cfg, "GNUNETD", "HOSTKEY", + &keyfile)) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + _("Lacking key configuration settings.\n")); + return NULL; + } + my_private_key = GNUNET_CRYPTO_rsa_key_create_from_file (keyfile); + if (my_private_key == NULL) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + _("Could not access hostkey file `%s'.\n"), keyfile); + GNUNET_free (keyfile); + return NULL; + } + GNUNET_free (keyfile); + GNUNET_CRYPTO_rsa_key_get_public (my_private_key, &my_public_key); + ass.purpose.size = htonl (sizeof (struct LocUriAssembly)); + ass.purpose.purpose = htonl (GNUNET_SIGNATURE_PURPOSE_PEER_PLACEMENT); + ass.exptime = GNUNET_TIME_absolute_hton (expiration_time); + ass.fi = baseUri->data.chk; + ass.peer = my_public_key; + uri = GNUNET_malloc (sizeof (struct GNUNET_FS_Uri)); + uri->type = loc; + uri->data.loc.fi = baseUri->data.chk; + uri->data.loc.expirationTime = expiration_time; + uri->data.loc.peer = my_public_key; + GNUNET_assert (GNUNET_OK == + GNUNET_CRYPTO_rsa_sign (my_private_key, &ass.purpose, + &uri->data.loc.contentSignature)); + GNUNET_CRYPTO_rsa_key_free (my_private_key); + return uri; +} + + +/** + * Create an SKS URI from a namespace and an identifier. + * + * @param ns namespace + * @param id identifier + * @param emsg where to store an error message + * @return an FS URI for the given namespace and identifier + */ +struct GNUNET_FS_Uri * +GNUNET_FS_uri_sks_create (struct GNUNET_FS_Namespace *ns, const char *id, + char **emsg) +{ + struct GNUNET_FS_Uri *ns_uri; + struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded pk; + + ns_uri = GNUNET_malloc (sizeof (struct GNUNET_FS_Uri)); + ns_uri->type = sks; + GNUNET_CRYPTO_rsa_key_get_public (ns->key, &pk); + GNUNET_CRYPTO_hash (&pk, sizeof (pk), &ns_uri->data.sks.namespace); + ns_uri->data.sks.identifier = GNUNET_strdup (id); + return ns_uri; +} + + +/** + * Create an SKS URI from a namespace ID and an identifier. + * + * @param nsid namespace ID + * @param id identifier + * @return an FS URI for the given namespace and identifier + */ +struct GNUNET_FS_Uri * +GNUNET_FS_uri_sks_create_from_nsid (GNUNET_HashCode * nsid, const char *id) +{ + struct GNUNET_FS_Uri *ns_uri; + + ns_uri = GNUNET_malloc (sizeof (struct GNUNET_FS_Uri)); + ns_uri->type = sks; + ns_uri->data.sks.namespace = *nsid; + ns_uri->data.sks.identifier = GNUNET_strdup (id); + return ns_uri; +} + + +/** + * Merge the sets of keywords from two KSK URIs. + * (useful for merging the canonicalized keywords with + * the original keywords for sharing). + * + * @param u1 first uri + * @param u2 second uri + * @return merged URI, NULL on error + */ +struct GNUNET_FS_Uri * +GNUNET_FS_uri_ksk_merge (const struct GNUNET_FS_Uri *u1, + const struct GNUNET_FS_Uri *u2) +{ + struct GNUNET_FS_Uri *ret; + unsigned int kc; + unsigned int i; + unsigned int j; + int found; + const char *kp; + char **kl; + + if ((u1 == NULL) && (u2 == NULL)) + return NULL; + if (u1 == NULL) + return GNUNET_FS_uri_dup (u2); + if (u2 == NULL) + return GNUNET_FS_uri_dup (u1); + if ((u1->type != ksk) || (u2->type != ksk)) + { + GNUNET_break (0); + return NULL; + } + kc = u1->data.ksk.keywordCount; + kl = GNUNET_malloc ((kc + u2->data.ksk.keywordCount) * sizeof (char *)); + for (i = 0; i < u1->data.ksk.keywordCount; i++) + kl[i] = GNUNET_strdup (u1->data.ksk.keywords[i]); + for (i = 0; i < u2->data.ksk.keywordCount; i++) + { + kp = u2->data.ksk.keywords[i]; + found = 0; + for (j = 0; j < u1->data.ksk.keywordCount; j++) + if (0 == strcmp (kp + 1, kl[j] + 1)) + { + found = 1; + if (kp[0] == '+') + kl[j][0] = '+'; + break; + } + if (0 == found) + kl[kc++] = GNUNET_strdup (kp); + } + ret = GNUNET_malloc (sizeof (struct GNUNET_FS_Uri)); + ret->type = ksk; + ret->data.ksk.keywordCount = kc; + ret->data.ksk.keywords = kl; + return ret; +} + + +/** + * Duplicate URI. + * + * @param uri the URI to duplicate + * @return copy of the URI + */ +struct GNUNET_FS_Uri * +GNUNET_FS_uri_dup (const struct GNUNET_FS_Uri *uri) +{ + struct GNUNET_FS_Uri *ret; + unsigned int i; + + if (uri == NULL) + return NULL; + ret = GNUNET_malloc (sizeof (struct GNUNET_FS_Uri)); + memcpy (ret, uri, sizeof (struct GNUNET_FS_Uri)); + switch (ret->type) + { + case ksk: + if (ret->data.ksk.keywordCount >= + GNUNET_MAX_MALLOC_CHECKED / sizeof (char *)) + { + GNUNET_break (0); + GNUNET_free (ret); + return NULL; + } + if (ret->data.ksk.keywordCount > 0) + { + ret->data.ksk.keywords = + GNUNET_malloc (ret->data.ksk.keywordCount * sizeof (char *)); + for (i = 0; i < ret->data.ksk.keywordCount; i++) + ret->data.ksk.keywords[i] = GNUNET_strdup (uri->data.ksk.keywords[i]); + } + else + ret->data.ksk.keywords = NULL; /* just to be sure */ + break; + case sks: + ret->data.sks.identifier = GNUNET_strdup (uri->data.sks.identifier); + break; + case loc: + break; + default: + break; + } + return ret; +} + + +/** + * Create an FS URI from a single user-supplied string of keywords. + * The string is broken up at spaces into individual keywords. + * Keywords that start with "+" are mandatory. Double-quotes can + * be used to prevent breaking up strings at spaces (and also + * to specify non-mandatory keywords starting with "+"). + * + * Keywords must contain a balanced number of double quotes and + * double quotes can not be used in the actual keywords (for + * example, the string '""foo bar""' will be turned into two + * "OR"ed keywords 'foo' and 'bar', not into '"foo bar"'. + * + * @param keywords the keyword string + * @param emsg where to store an error message + * @return an FS URI for the given keywords, NULL + * if keywords is not legal (i.e. empty). + */ +struct GNUNET_FS_Uri * +GNUNET_FS_uri_ksk_create (const char *keywords, char **emsg) +{ + char **keywordarr; + unsigned int num_Words; + int inWord; + char *pos; + struct GNUNET_FS_Uri *uri; + char *searchString; + int saw_quote; + + if (keywords == NULL) + { + *emsg = GNUNET_strdup (_("No keywords specified!\n")); + GNUNET_break (0); + return NULL; + } + searchString = GNUNET_strdup (keywords); + num_Words = 0; + inWord = 0; + saw_quote = 0; + pos = searchString; + while ('\0' != *pos) + { + if ((saw_quote == 0) && (isspace ((unsigned char) *pos))) + { + inWord = 0; + } + else if (0 == inWord) + { + inWord = 1; + ++num_Words; + } + if ('"' == *pos) + saw_quote = (saw_quote + 1) % 2; + pos++; + } + if (num_Words == 0) + { + GNUNET_free (searchString); + *emsg = GNUNET_strdup (_("No keywords specified!\n")); + return NULL; + } + if (saw_quote != 0) + { + GNUNET_free (searchString); + *emsg = GNUNET_strdup (_("Number of double-quotes not balanced!\n")); + return NULL; + } + keywordarr = GNUNET_malloc (num_Words * sizeof (char *)); + num_Words = 0; + inWord = 0; + pos = searchString; + while ('\0' != *pos) + { + if ((saw_quote == 0) && (isspace ((unsigned char) *pos))) + { + inWord = 0; + *pos = '\0'; + } + else if (0 == inWord) + { + keywordarr[num_Words] = pos; + inWord = 1; + ++num_Words; + } + if ('"' == *pos) + saw_quote = (saw_quote + 1) % 2; + pos++; + } + uri = + GNUNET_FS_uri_ksk_create_from_args (num_Words, + (const char **) keywordarr); + GNUNET_free (keywordarr); + GNUNET_free (searchString); + return uri; +} + + +/** + * Create an FS URI from a user-supplied command line of keywords. + * Arguments should start with "+" to indicate mandatory + * keywords. + * + * @param argc number of keywords + * @param argv keywords (double quotes are not required for + * keywords containing spaces; however, double + * quotes are required for keywords starting with + * "+"); there is no mechanism for having double + * quotes in the actual keywords (if the user + * did specifically specify double quotes, the + * caller should convert each double quote + * into two single quotes). + * @return an FS URI for the given keywords, NULL + * if keywords is not legal (i.e. empty). + */ +struct GNUNET_FS_Uri * +GNUNET_FS_uri_ksk_create_from_args (unsigned int argc, const char **argv) +{ + unsigned int i; + struct GNUNET_FS_Uri *uri; + const char *keyword; + char *val; + const char *r; + char *w; + char *emsg; + + if (argc == 0) + return NULL; + /* allow URI to be given as one and only keyword and + * handle accordingly */ + emsg = NULL; + if ((argc == 1) && (strlen (argv[0]) > strlen (GNUNET_FS_URI_PREFIX)) && + (0 == + strncmp (argv[0], GNUNET_FS_URI_PREFIX, strlen (GNUNET_FS_URI_PREFIX))) + && (NULL != (uri = GNUNET_FS_uri_parse (argv[0], &emsg)))) + return uri; + GNUNET_free_non_null (emsg); + uri = GNUNET_malloc (sizeof (struct GNUNET_FS_Uri)); + uri->type = ksk; + uri->data.ksk.keywordCount = argc; + uri->data.ksk.keywords = GNUNET_malloc (argc * sizeof (char *)); + for (i = 0; i < argc; i++) + { + keyword = argv[i]; + if (keyword[0] == '+') + val = GNUNET_strdup (keyword); + else + GNUNET_asprintf (&val, " %s", keyword); + r = val; + w = val; + while ('\0' != *r) + { + if ('"' == *r) + r++; + else + *(w++) = *(r++); + } + *w = '\0'; + uri->data.ksk.keywords[i] = val; + } + return uri; +} + + +/** + * Test if two URIs are equal. + * + * @param u1 one of the URIs + * @param u2 the other URI + * @return GNUNET_YES if the URIs are equal + */ +int +GNUNET_FS_uri_test_equal (const struct GNUNET_FS_Uri *u1, + const struct GNUNET_FS_Uri *u2) +{ + int ret; + unsigned int i; + unsigned int j; + + GNUNET_assert (u1 != NULL); + GNUNET_assert (u2 != NULL); + if (u1->type != u2->type) + return GNUNET_NO; + switch (u1->type) + { + case chk: + if (0 == + memcmp (&u1->data.chk, &u2->data.chk, sizeof (struct FileIdentifier))) + return GNUNET_YES; + return GNUNET_NO; + case sks: + if ((0 == + memcmp (&u1->data.sks.namespace, &u2->data.sks.namespace, + sizeof (GNUNET_HashCode))) && + (0 == strcmp (u1->data.sks.identifier, u2->data.sks.identifier))) + + return GNUNET_YES; + return GNUNET_NO; + case ksk: + if (u1->data.ksk.keywordCount != u2->data.ksk.keywordCount) + return GNUNET_NO; + for (i = 0; i < u1->data.ksk.keywordCount; i++) + { + ret = GNUNET_NO; + for (j = 0; j < u2->data.ksk.keywordCount; j++) + { + if (0 == strcmp (u1->data.ksk.keywords[i], u2->data.ksk.keywords[j])) + { + ret = GNUNET_YES; + break; + } + } + if (ret == GNUNET_NO) + return GNUNET_NO; + } + return GNUNET_YES; + case loc: + if (memcmp + (&u1->data.loc, &u2->data.loc, + sizeof (struct FileIdentifier) + + sizeof (struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded) + + sizeof (struct GNUNET_TIME_Absolute) + sizeof (unsigned short) + + sizeof (unsigned short)) != 0) + return GNUNET_NO; + return GNUNET_YES; + default: + return GNUNET_NO; + } +} + + +/** + * Is this a namespace URI? + * + * @param uri the uri to check + * @return GNUNET_YES if this is an SKS uri + */ +int +GNUNET_FS_uri_test_sks (const struct GNUNET_FS_Uri *uri) +{ + return uri->type == sks; +} + + +/** + * Get the ID of a namespace from the given + * namespace URI. + * + * @param uri the uri to get the namespace ID from + * @param nsid where to store the ID of the namespace + * @return GNUNET_OK on success + */ +int +GNUNET_FS_uri_sks_get_namespace (const struct GNUNET_FS_Uri *uri, + GNUNET_HashCode * nsid) +{ + if (!GNUNET_FS_uri_test_sks (uri)) + { + GNUNET_break (0); + return GNUNET_SYSERR; + } + *nsid = uri->data.sks.namespace; + return GNUNET_OK; +} + + +/** + * Get the content identifier of an SKS URI. + * + * @param uri the sks uri + * @return NULL on error (not a valid SKS URI) + */ +char * +GNUNET_FS_uri_sks_get_content_id (const struct GNUNET_FS_Uri *uri) +{ + if (!GNUNET_FS_uri_test_sks (uri)) + { + GNUNET_break (0); + return NULL; + } + return GNUNET_strdup (uri->data.sks.identifier); +} + + +/** + * Convert namespace URI to a human readable format + * (using the namespace description, if available). + * + * @param cfg configuration to use + * @param uri SKS uri to convert + * @return NULL on error (not an SKS URI) + */ +char * +GNUNET_FS_uri_sks_to_string_fancy (struct GNUNET_CONFIGURATION_Handle *cfg, + const struct GNUNET_FS_Uri *uri) +{ + char *ret; + char *name; + + if (uri->type != sks) + return NULL; + name = GNUNET_PSEUDONYM_id_to_name (cfg, &uri->data.sks.namespace); + if (name == NULL) + return GNUNET_FS_uri_to_string (uri); + GNUNET_asprintf (&ret, "%s: %s", name, uri->data.sks.identifier); + GNUNET_free (name); + return ret; +} + + +/** + * Is this a keyword URI? + * + * @param uri the uri + * @return GNUNET_YES if this is a KSK uri + */ +int +GNUNET_FS_uri_test_ksk (const struct GNUNET_FS_Uri *uri) +{ +#if EXTRA_CHECKS + unsigned int i; + + if (uri->type == ksk) + { + for (i=0;i < uri->data.ksk.keywordCount; i++) + GNUNET_assert (uri->data.ksk.keywords[i] != NULL); + } +#endif + return uri->type == ksk; +} + + +/** + * Is this a file (or directory) URI? + * + * @param uri the uri to check + * @return GNUNET_YES if this is a CHK uri + */ +int +GNUNET_FS_uri_test_chk (const struct GNUNET_FS_Uri *uri) +{ + return uri->type == chk; +} + + +/** + * What is the size of the file that this URI + * refers to? + * + * @param uri the CHK URI to inspect + * @return size of the file as specified in the CHK URI + */ +uint64_t +GNUNET_FS_uri_chk_get_file_size (const struct GNUNET_FS_Uri * uri) +{ + switch (uri->type) + { + case chk: + return GNUNET_ntohll (uri->data.chk.file_length); + case loc: + return GNUNET_ntohll (uri->data.loc.fi.file_length); + default: + GNUNET_assert (0); + } + return 0; /* unreachable */ +} + + +/** + * Is this a location URI? + * + * @param uri the uri to check + * @return GNUNET_YES if this is a LOC uri + */ +int +GNUNET_FS_uri_test_loc (const struct GNUNET_FS_Uri *uri) +{ + return uri->type == loc; +} + + +/** + * Add a keyword as non-mandatory (with ' '-prefix) to the + * given keyword list at offset 'index'. The array is + * guaranteed to be long enough. + * + * @param s keyword to add + * @param array array to add the keyword to + * @param index offset where to add the keyword + */ +static void +insert_non_mandatory_keyword (const char *s, char **array, int index) +{ + char *nkword; + GNUNET_asprintf (&nkword, " %s", /* space to mark as 'non mandatory' */ s); + array[index] = nkword; +} + + +/** + * Test if the given keyword 's' is already present in the + * given array, ignoring the '+'-mandatory prefix in the array. + * + * @param s keyword to test + * @param array keywords to test against, with ' ' or '+' prefix to ignore + * @param array_length length of the array + * @return GNUNET_YES if the keyword exists, GNUNET_NO if not + */ +static int +find_duplicate (const char *s, const char **array, int array_length) +{ + int j; + + for (j = array_length - 1; j >= 0; j--) + if (0 == strcmp (&array[j][1], s)) + return GNUNET_YES; + return GNUNET_NO; +} + +static char * +normalize_metadata (enum EXTRACTOR_MetaFormat format, const char *data, + size_t data_len) +{ + uint8_t *free_str = NULL; + uint8_t *str_to_normalize = (uint8_t *) data; + uint8_t *normalized; + size_t r_len; + if (str_to_normalize == NULL) + return NULL; + /* Don't trust libextractor */ + if (format == EXTRACTOR_METAFORMAT_UTF8) + { + free_str = (uint8_t *) u8_check ((const uint8_t *) data, data_len); + if (free_str == NULL) + free_str = NULL; + else + format = EXTRACTOR_METAFORMAT_C_STRING; + } + if (format == EXTRACTOR_METAFORMAT_C_STRING) + { + free_str = u8_strconv_from_encoding (data, locale_charset (), iconveh_escape_sequence); + if (free_str == NULL) + return NULL; + } + + normalized = u8_tolower (str_to_normalize, strlen ((char *) str_to_normalize), NULL, UNINORM_NFD, NULL, &r_len); + /* free_str is allocated by libunistring internally, use free() */ + if (free_str != NULL) + free (free_str); + if (normalized != NULL) + { + /* u8_tolower allocates a non-NULL-terminated string! */ + free_str = GNUNET_malloc (r_len + 1); + memcpy (free_str, normalized, r_len); + free_str[r_len] = '\0'; + free (normalized); + normalized = free_str; + } + return (char *) normalized; +} + +/** + * Counts the number of UTF-8 characters (not bytes) in the string, + * returns that count. + */ +static size_t +u8_strcount (const uint8_t *s) +{ + size_t count; + ucs4_t c; + GNUNET_assert (s != NULL); + if (s[0] == 0) + return 0; + for (count = 0; s != NULL; count++) + s = u8_next (&c, s); + return count - 1; +} + + +/** + * Break the filename up by matching [], () and {} pairs to make + * keywords. In case of nesting parentheses only the inner pair counts. + * You can't escape parentheses to scan something like "[blah\{foo]" to + * make a "blah{foo" keyword, this function is only a heuristic! + * + * @param s string to break down. + * @param array array to fill with enclosed tokens. If NULL, then tokens + * are only counted. + * @param index index at which to start filling the array (entries prior + * to it are used to check for duplicates). ignored if array == NULL. + * @return number of tokens counted (including duplicates), or number of + * tokens extracted (excluding duplicates). 0 if there are no + * matching parens in the string (when counting), or when all tokens + * were duplicates (when extracting). + */ +static int +get_keywords_from_parens (const char *s, char **array, int index) +{ + int count = 0; + char *open_paren; + char *close_paren; + char *ss; + char tmp; + + if (NULL == s) + return 0; + ss = GNUNET_strdup (s); + open_paren = ss - 1; + while (NULL != (open_paren = strpbrk (open_paren + 1, "[{("))) + { + int match = 0; + + close_paren = strpbrk (open_paren + 1, "]})"); + if (NULL == close_paren) + continue; + switch (open_paren[0]) + { + case '[': + if (']' == close_paren[0]) + match = 1; + break; + case '{': + if ('}' == close_paren[0]) + match = 1; + break; + case '(': + if (')' == close_paren[0]) + match = 1; + break; + default: + break; + } + if (match && (close_paren - open_paren > 1)) + { + tmp = close_paren[0]; + close_paren[0] = '\0'; + /* Keywords must be at least 3 characters long */ + if (u8_strcount ((const uint8_t *) &open_paren[1]) <= 2) + { + close_paren[0] = tmp; + continue; + } + if (NULL != array) + { + char *normalized; + if (GNUNET_NO == find_duplicate ((const char *) &open_paren[1], + (const char **) array, index + count)) + { + insert_non_mandatory_keyword ((const char *) &open_paren[1], array, + index + count); + count++; + } + normalized = normalize_metadata (EXTRACTOR_METAFORMAT_UTF8, + &open_paren[1], close_paren - &open_paren[1]); + if (normalized != NULL) + { + if (GNUNET_NO == find_duplicate ((const char *) normalized, + (const char **) array, index + count)) + { + insert_non_mandatory_keyword ((const char *) normalized, array, + index + count); + count++; + } + GNUNET_free (normalized); + } + } + else + count++; + close_paren[0] = tmp; + } + } + GNUNET_free (ss); + return count; +} + + +/** + * Where to break up keywords + */ +#define TOKENS "_. /-!?#&+@\"\'\\;:," + +/** + * Break the filename up by TOKENS to make + * keywords. + * + * @param s string to break down. + * @param array array to fill with tokens. If NULL, then tokens are only + * counted. + * @param index index at which to start filling the array (entries prior + * to it are used to check for duplicates). ignored if array == NULL. + * @return number of tokens (>1) counted (including duplicates), or number of + * tokens extracted (excluding duplicates). 0 if there are no + * separators in the string (when counting), or when all tokens were + * duplicates (when extracting). + */ +static int +get_keywords_from_tokens (const char *s, char **array, int index) +{ + char *p; + char *ss; + int seps = 0; + + ss = GNUNET_strdup (s); + for (p = strtok (ss, TOKENS); p != NULL; p = strtok (NULL, TOKENS)) + { + /* Keywords must be at least 3 characters long */ + if (u8_strcount ((const uint8_t *) p) <= 2) + continue; + if (NULL != array) + { + char *normalized; + if (GNUNET_NO == find_duplicate (p, (const char **) array, index + seps)) + { + insert_non_mandatory_keyword (p, array, + index + seps); + seps++; + } + normalized = normalize_metadata (EXTRACTOR_METAFORMAT_UTF8, + p, strlen (p)); + if (normalized != NULL) + { + if (GNUNET_NO == find_duplicate ((const char *) normalized, + (const char **) array, index + seps)) + { + insert_non_mandatory_keyword ((const char *) normalized, array, + index + seps); + seps++; + } + GNUNET_free (normalized); + } + } + else + seps++; + } + GNUNET_free (ss); + return seps; +} +#undef TOKENS + +/** + * Function called on each value in the meta data. + * Adds it to the URI. + * + * @param cls URI to update + * @param plugin_name name of the plugin that produced this value; + * special values can be used (i.e. '<zlib>' for zlib being + * used in the main libextractor library and yielding + * meta data). + * @param type libextractor-type describing the meta data + * @param format basic format information about data + * @param data_mime_type mime-type of data (not of the original file); + * can be NULL (if mime-type is not known) + * @param data actual meta-data found + * @param data_len number of bytes in data + * @return 0 (always) + */ +static int +gather_uri_data (void *cls, const char *plugin_name, + enum EXTRACTOR_MetaType type, enum EXTRACTOR_MetaFormat format, + const char *data_mime_type, const char *data, size_t data_len) +{ + struct GNUNET_FS_Uri *uri = cls; + char *normalized_data; + + if ((format != EXTRACTOR_METAFORMAT_UTF8) && + (format != EXTRACTOR_METAFORMAT_C_STRING)) + return 0; + /* Keywords must be at least 3 characters long + * If given non-utf8 string it will, most likely, find it to be invalid, + * and will return the length of its valid part, skipping the keyword. + * If it does - fix the extractor, not this check! + */ + if (u8_strcount ((const uint8_t *) data) <= 2) + { + return 0; + } + normalized_data = normalize_metadata (format, data, data_len); + if (!find_duplicate (data, (const char **) uri->data.ksk.keywords, uri->data.ksk.keywordCount)) + { + insert_non_mandatory_keyword (data, + uri->data.ksk.keywords, uri->data.ksk.keywordCount); + uri->data.ksk.keywordCount++; + } + if (normalized_data != NULL) + { + if (!find_duplicate (normalized_data, (const char **) uri->data.ksk.keywords, uri->data.ksk.keywordCount)) + { + insert_non_mandatory_keyword (normalized_data, + uri->data.ksk.keywords, uri->data.ksk.keywordCount); + uri->data.ksk.keywordCount++; + } + GNUNET_free (normalized_data); + } + return 0; +} + + +/** + * Construct a keyword-URI from meta-data (take all entries + * in the meta-data and construct one large keyword URI + * that lists all keywords that can be found in the meta-data). + * + * @param md metadata to use + * @return NULL on error, otherwise a KSK URI + */ +struct GNUNET_FS_Uri * +GNUNET_FS_uri_ksk_create_from_meta_data (const struct GNUNET_CONTAINER_MetaData + *md) +{ + struct GNUNET_FS_Uri *ret; + char *filename; + char *full_name = NULL; + char *ss; + int ent; + int tok_keywords = 0; + int paren_keywords = 0; + + if (md == NULL) + return NULL; + ret = GNUNET_malloc (sizeof (struct GNUNET_FS_Uri)); + ret->type = ksk; + ent = GNUNET_CONTAINER_meta_data_iterate (md, NULL, NULL); + if (ent > 0) + { + full_name = GNUNET_CONTAINER_meta_data_get_first_by_types (md, + EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME, -1); + if (NULL != full_name) + { + filename = full_name; + while (NULL != (ss = strstr (filename, DIR_SEPARATOR_STR))) + filename = ss + 1; + tok_keywords = get_keywords_from_tokens (filename, NULL, 0); + paren_keywords = get_keywords_from_parens (filename, NULL, 0); + } + /* x2 because there might be a normalized variant of every keyword */ + ret->data.ksk.keywords = GNUNET_malloc (sizeof (char *) * (ent + + tok_keywords + paren_keywords) * 2); + GNUNET_CONTAINER_meta_data_iterate (md, &gather_uri_data, ret); + } + if (tok_keywords > 0) + ret->data.ksk.keywordCount += get_keywords_from_tokens (filename, + ret->data.ksk.keywords, + ret->data.ksk.keywordCount); + if (paren_keywords > 0) + ret->data.ksk.keywordCount += get_keywords_from_parens (filename, + ret->data.ksk.keywords, + ret->data.ksk.keywordCount); + if (ent > 0) + GNUNET_free_non_null (full_name); + return ret; +} + + +/** + * In URI-encoding, does the given character + * need to be encoded using %-encoding? + */ +static int +needs_percent (char c) +{ + return (! + ((isalnum ((unsigned char) c)) || (c == '-') || (c == '_') || + (c == '.') || (c == '~'))); +} + + +/** + * Convert a KSK URI to a string. + * + * @param uri the URI to convert + * @return NULL on error (i.e. keywordCount == 0) + */ +static char * +uri_ksk_to_string (const struct GNUNET_FS_Uri *uri) +{ + char **keywords; + unsigned int keywordCount; + size_t n; + char *ret; + unsigned int i; + unsigned int j; + unsigned int wpos; + size_t slen; + const char *keyword; + + if (uri->type != ksk) + return NULL; + keywords = uri->data.ksk.keywords; + keywordCount = uri->data.ksk.keywordCount; + n = keywordCount + strlen (GNUNET_FS_URI_PREFIX) + + strlen (GNUNET_FS_URI_KSK_INFIX) + 1; + for (i = 0; i < keywordCount; i++) + { + keyword = keywords[i]; + slen = strlen (keyword); + n += slen; + for (j = 0; j < slen; j++) + { + if ((j == 0) && (keyword[j] == ' ')) + { + n--; + continue; /* skip leading space */ + } + if (needs_percent (keyword[j])) + n += 2; /* will use %-encoding */ + } + } + ret = GNUNET_malloc (n); + strcpy (ret, GNUNET_FS_URI_PREFIX); + strcat (ret, GNUNET_FS_URI_KSK_INFIX); + wpos = strlen (ret); + for (i = 0; i < keywordCount; i++) + { + keyword = keywords[i]; + slen = strlen (keyword); + for (j = 0; j < slen; j++) + { + if ((j == 0) && (keyword[j] == ' ')) + continue; /* skip leading space */ + if (needs_percent (keyword[j])) + { + sprintf (&ret[wpos], "%%%02X", keyword[j]); + wpos += 3; + } + else + { + ret[wpos++] = keyword[j]; + } + } + if (i != keywordCount - 1) + ret[wpos++] = '+'; + } + return ret; +} + + +/** + * Convert SKS URI to a string. + * + * @param uri sks uri to convert + * @return NULL on error + */ +static char * +uri_sks_to_string (const struct GNUNET_FS_Uri *uri) +{ + const GNUNET_HashCode *namespace; + const char *identifier; + char *ret; + struct GNUNET_CRYPTO_HashAsciiEncoded ns; + + if (uri->type != sks) + return NULL; + namespace = &uri->data.sks.namespace; + identifier = uri->data.sks.identifier; + GNUNET_CRYPTO_hash_to_enc (namespace, &ns); + GNUNET_asprintf (&ret, "%s%s%s/%s", GNUNET_FS_URI_PREFIX, + GNUNET_FS_URI_SKS_INFIX, (const char *) &ns, identifier); + return ret; +} + + +/** + * Convert a CHK URI to a string. + * + * @param uri chk uri to convert + * @return NULL on error + */ +static char * +uri_chk_to_string (const struct GNUNET_FS_Uri *uri) +{ + const struct FileIdentifier *fi; + char *ret; + struct GNUNET_CRYPTO_HashAsciiEncoded keyhash; + struct GNUNET_CRYPTO_HashAsciiEncoded queryhash; + + if (uri->type != chk) + return NULL; + fi = &uri->data.chk; + GNUNET_CRYPTO_hash_to_enc (&fi->chk.key, &keyhash); + GNUNET_CRYPTO_hash_to_enc (&fi->chk.query, &queryhash); + + GNUNET_asprintf (&ret, "%s%s%s.%s.%llu", GNUNET_FS_URI_PREFIX, + GNUNET_FS_URI_CHK_INFIX, (const char *) &keyhash, + (const char *) &queryhash, GNUNET_ntohll (fi->file_length)); + return ret; +} + +/** + * Convert binary data to a string. + * + * @param data binary data to convert + * @param size number of bytes in data + * @return converted data + */ +static char * +bin2enc (const void *data, size_t size) +{ + /** + * 64 characters for encoding, 6 bits per character + */ + static char *tbl = + "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz_="; + + size_t len; + size_t pos; + unsigned int bits; + unsigned int hbits; + char *ret; + + GNUNET_assert (strlen (tbl) == 64); + len = size * 8 / 6; + if (((size * 8) % 6) != 0) + len++; + ret = GNUNET_malloc (len + 1); + ret[len] = '\0'; + len = 0; + bits = 0; + hbits = 0; + for (pos = 0; pos < size; pos++) + { + bits |= ((((const unsigned char *) data)[pos]) << hbits); + hbits += 8; + while (hbits >= 6) + { + ret[len++] = tbl[bits & 63]; + bits >>= 6; + hbits -= 6; + } + } + if (hbits > 0) + ret[len] = tbl[bits & 63]; + return ret; +} + + +/** + * Convert a LOC URI to a string. + * + * @param uri loc uri to convert + * @return NULL on error + */ +static char * +uri_loc_to_string (const struct GNUNET_FS_Uri *uri) +{ + char *ret; + struct GNUNET_CRYPTO_HashAsciiEncoded keyhash; + struct GNUNET_CRYPTO_HashAsciiEncoded queryhash; + char *peerId; + char *peerSig; + + GNUNET_CRYPTO_hash_to_enc (&uri->data.loc.fi.chk.key, &keyhash); + GNUNET_CRYPTO_hash_to_enc (&uri->data.loc.fi.chk.query, &queryhash); + peerId = + bin2enc (&uri->data.loc.peer, + sizeof (struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded)); + peerSig = + bin2enc (&uri->data.loc.contentSignature, + sizeof (struct GNUNET_CRYPTO_RsaSignature)); + GNUNET_asprintf (&ret, "%s%s%s.%s.%llu.%s.%s.%llu", GNUNET_FS_URI_PREFIX, + GNUNET_FS_URI_LOC_INFIX, (const char *) &keyhash, + (const char *) &queryhash, + (unsigned long long) GNUNET_ntohll (uri->data.loc. + fi.file_length), peerId, + peerSig, + (unsigned long long) uri->data.loc.expirationTime.abs_value); + GNUNET_free (peerSig); + GNUNET_free (peerId); + return ret; +} + + +/** + * Convert a URI to a UTF-8 String. + * + * @param uri uri to convert to a string + * @return the UTF-8 string + */ +char * +GNUNET_FS_uri_to_string (const struct GNUNET_FS_Uri *uri) +{ + if (uri == NULL) + { + GNUNET_break (0); + return NULL; + } + switch (uri->type) + { + case ksk: + return uri_ksk_to_string (uri); + case sks: + return uri_sks_to_string (uri); + case chk: + return uri_chk_to_string (uri); + case loc: + return uri_loc_to_string (uri); + default: + GNUNET_break (0); + return NULL; + } +} + +/* end of fs_uri.c */ diff --git a/src/fs/gnunet-directory.c b/src/fs/gnunet-directory.c new file mode 100644 index 0000000..0721ea9 --- /dev/null +++ b/src/fs/gnunet-directory.c @@ -0,0 +1,183 @@ +/* + This file is part of GNUnet. + (C) 2001, 2002, 2004, 2005, 2006, 2007, 2009 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ +/** + * @file fs/gnunet-directory.c + * @brief display content of GNUnet directories + * @author Christian Grothoff + */ +#include "platform.h" +#include "gnunet_fs_service.h" + +static int ret; + +/** + * Print a meta data entry. + * + * @param cls closure (unused) + * @param plugin_name name of the plugin that generated the meta data + * @param type type of the keyword + * @param format format of data + * @param data_mime_type mime type of data + * @param data value of the meta data + * @param data_size number of bytes in data + * @return always 0 (to continue iterating) + */ +static int +item_printer (void *cls, const char *plugin_name, enum EXTRACTOR_MetaType type, + enum EXTRACTOR_MetaFormat format, const char *data_mime_type, + const char *data, size_t data_size) +{ + if (type == EXTRACTOR_METATYPE_GNUNET_FULL_DATA) + { + printf (_("\t<original file embedded in %u bytes of meta data>\n"), + (unsigned int) data_size); + return 0; + } + if ((format != EXTRACTOR_METAFORMAT_UTF8) && + (format != EXTRACTOR_METAFORMAT_C_STRING)) + return 0; + if (type == EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME) + return 0; + printf ("\t%20s: %s\n", + dgettext (LIBEXTRACTOR_GETTEXT_DOMAIN, + EXTRACTOR_metatype_to_string (type)), data); + return 0; +} + + + +/** + * Print an entry in a directory. + * + * @param cls closure (not used) + * @param filename name of the file in the directory + * @param uri URI of the file + * @param meta metadata for the file; metadata for + * the directory if everything else is NULL/zero + * @param length length of the available data for the file + * (of type size_t since data must certainly fit + * into memory; if files are larger than size_t + * permits, then they will certainly not be + * embedded with the directory itself). + * @param data data available for the file (length bytes) + */ +static void +print_entry (void *cls, const char *filename, const struct GNUNET_FS_Uri *uri, + const struct GNUNET_CONTAINER_MetaData *meta, size_t length, + const void *data) +{ + char *string; + char *name; + + name = + GNUNET_CONTAINER_meta_data_get_by_type (meta, + EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME); + if (uri == NULL) + { + printf (_("Directory `%s' meta data:\n"), name); + GNUNET_CONTAINER_meta_data_iterate (meta, &item_printer, NULL); + printf ("\n"); + printf (_("Directory `%s' contents:\n"), name); + GNUNET_free (name); + return; + } + string = GNUNET_FS_uri_to_string (uri); + printf ("%s (%s):\n", name, string); + GNUNET_free (string); + GNUNET_CONTAINER_meta_data_iterate (meta, &item_printer, NULL); + printf ("\n"); + GNUNET_free (name); +} + + +/** + * Main function that will be run by the scheduler. + * + * @param cls closure + * @param args remaining command-line arguments + * @param cfgfile name of the configuration file used (for saving, can be NULL!) + * @param cfg configuration + */ +static void +run (void *cls, char *const *args, const char *cfgfile, + const struct GNUNET_CONFIGURATION_Handle *cfg) +{ + struct GNUNET_DISK_MapHandle *map; + struct GNUNET_DISK_FileHandle *h; + void *data; + size_t len; + uint64_t size; + const char *filename; + int i; + + if (NULL == args[0]) + { + FPRINTF (stderr, "%s", _("You must specify a filename to inspect.\n")); + ret = 1; + return; + } + i = 0; + while (NULL != (filename = args[i++])) + { + if ((GNUNET_OK != GNUNET_DISK_file_size (filename, &size, GNUNET_YES)) || + (NULL == + (h = + GNUNET_DISK_file_open (filename, GNUNET_DISK_OPEN_READ, + GNUNET_DISK_PERM_NONE)))) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, _("Failed to read directory `%s'\n"), + filename); + ret = 1; + continue; + } + len = (size_t) size; + data = GNUNET_DISK_file_map (h, &map, GNUNET_DISK_MAP_TYPE_READ, len); + GNUNET_assert (NULL != data); + if (GNUNET_OK != GNUNET_FS_directory_list_contents (len, data, 0, &print_entry, NULL)) + fprintf (stdout, _("`%s' is not a GNUnet directory\n"), + filename); + else + printf ("\n"); + GNUNET_DISK_file_unmap (map); + GNUNET_DISK_file_close (h); + } +} + +/** + * The main function to inspect GNUnet directories. + * + * @param argc number of arguments from the command line + * @param argv command line arguments + * @return 0 ok, 1 on error + */ +int +main (int argc, char *const *argv) +{ + static struct GNUNET_GETOPT_CommandLineOption options[] = { + GNUNET_GETOPT_OPTION_END + }; + return (GNUNET_OK == + GNUNET_PROGRAM_run (argc, argv, "gnunet-directory [OPTIONS] FILENAME", + gettext_noop + ("Display contents of a GNUnet directory"), + options, &run, NULL)) ? ret : 1; +} + +/* end of gnunet-directory.c */ diff --git a/src/fs/gnunet-download-manager.scm b/src/fs/gnunet-download-manager.scm new file mode 100755 index 0000000..80d04fa --- /dev/null +++ b/src/fs/gnunet-download-manager.scm @@ -0,0 +1,407 @@ +#!/bin/sh +exec guile -e main -s "$0" "$@" +!# + +;;; gnunet-download-manager -- Manage GNUnet downloads. +;;; Copyright (C) 2004 Ludovic Courtès +;;; +;;; This program is free software; you can redistribute it and/or +;;; modify it under the terms of the GNU General Public License +;;; as published by the Free Software Foundation; either version 2 +;;; of the License, or (at your option) any later version. +;;; +;;; This program is distributed in the hope that it will be useful, +;;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;; GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with this program; if not, write to the Free Software +;;; Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + +;;; Remember ongoing GNUnet downloads so as to be able to resume them +;;; later. Typical usage is to define the following alias in your +;;; favorite shell: +;;; +;;; alias gnunet-download='gnunet-download-manager.scm download' +;;; +;;; You may have a ~/.gnunet-download-manager.scm Scheme configuration +;;; file. In particular, if you would like to be notified of +;;; completed downloads, you may want to add the following line to +;;; your configuration file: +;;; +;;; (add-hook! *completed-download-hook* +;;; completed-download-notification-hook) +;;; +;;; This script works fine with GNU Guile 1.6.4, and doesn't run with +;;; Guile 1.4.x. +;;; +;;; Enjoy! +;;; Ludovic Courtès <ludo@chbouib.org> + +(use-modules (ice-9 format) + (ice-9 optargs) + (ice-9 regex) + (ice-9 and-let-star) + (ice-9 pretty-print) + (ice-9 documentation)) + +;; Overall user settings +(define *debug?* #f) +(define *rc-file* (string-append (getenv "HOME") + "/.gnunet-download-manager.scm")) +(define *status-directory* (string-append (getenv "HOME") "/" + ".gnunet-download-manager")) +(define *gnunet-download* "gnunet-download") + +;; Helper macros +(define-macro (gnunet-info fmt . args) + `(format #t (string-append *program-name* ": " ,fmt "~%") + ,@args)) + +(define-macro (gnunet-debug fmt . args) + (if *debug?* + (cons 'gnunet-info (cons fmt args)) + #t)) + +(define-macro (gnunet-error fmt . args) + `(and ,(cons 'gnunet-info (cons fmt args)) + (exit 1))) + +(define (exception-string key args) + "Describe an error, using the format from @var{args}, if available." + (if (< (length args) 4) + (format #f "Scheme exception: ~S" key) + (string-append + (if (string? (car args)) + (string-append "In " (car args)) + "Scheme exception") + ": " + (apply format `(#f ,(cadr args) ,@(caddr args)))))) + + +;; Regexps matching GNUnet URIs +(define *uri-base* + "([[:alnum:]]+)\.([[:alnum:]]+)\.([[:alnum:]]+)\.([0-9]+)") +(define *uri-re* + (make-regexp (string-append "^gnunet://afs/" *uri-base* "$") + regexp/extended)) +(define *uri-status-file-re* + (make-regexp (string-append "^" *uri-base* "$") + regexp/extended)) + + +(define (uri-status-file-name directory uri) + "Return the name of the status file for URI @var{uri}." + (let ((match (regexp-exec *uri-re* uri))) + (if (not match) + (and (gnunet-info "~a: Invalid URI" uri) #f) + (let ((start (match:start match 1)) + (end (match:end match 4))) + (string-append directory "/" + (substring uri start end)))))) + +(define (uri-status directory uri) + "Load the current status alist for URI @var{uri} from @var{directory}." + (gnunet-debug "uri-status") + (let ((filename (uri-status-file-name directory uri))) + (catch 'system-error + (lambda () + (let* ((file (open-input-file filename)) + (status (read file))) + (begin + (close-port file) + status))) + (lambda (key . args) + (and (gnunet-debug (exception-string key args)) + '()))))) + +(define (process-exists? pid) + (false-if-exception (begin (kill pid 0) #t))) + +(define (fork-and-exec directory program . args) + "Launch @var{program} and return its PID." + (gnunet-debug "fork-and-exec: ~a ~a" program args) + (let ((pid (primitive-fork))) + (if (= 0 pid) + (begin + (if directory (chdir directory)) + (apply execlp (cons program (cons program args)))) + pid))) + +(define* (start-downloader downloader uri options + #:key (directory #f)) + "Start the GNUnet downloader for URI @var{uri} with options +@var{options}. Return an alist describing the download status." + (catch 'system-error + (lambda () + (let* ((pid (apply fork-and-exec + `(,(if directory directory (getcwd)) + ,downloader + ,@options)))) + (gnunet-info "Launched process ~a" pid) + `((uri . ,uri) + (working-directory . ,(if directory directory (getcwd))) + (options . ,options) + (pid . ,(getpid)) + (downloader-pid . ,pid)))) + (lambda (key . args) + (gnunet-error (exception-string key args))))) + +(define (download-process-alive? uri-status) + "Return true if the download whose status is that described by +@var{uri-status} is still alive." + (let ((pid (assoc-ref uri-status 'pid)) + (downloader-pid (assoc-ref uri-status 'downloader-pid))) + (and (process-exists? pid) + (process-exists? downloader-pid)))) + +(define (start-file-download downloader status-dir uri options) + "Dowload the file located at @var{uri}, with options @var{options} +and return an updated status alist." + (gnunet-debug "start-file-download") + (let ((uri-status (uri-status status-dir uri))) + (if (null? uri-status) + (acons 'start-date (current-time) + (start-downloader downloader uri options)) + (if (download-process-alive? uri-status) + (and (gnunet-info "~a already being downloaded by process ~a" + uri (assoc-ref uri-status 'pid)) + #f) + (and (gnunet-info "Resuming download") + (let ((start-date (assoc-ref uri-status 'start-date)) + (dir (assoc-ref uri-status 'working-directory)) + (options (assoc-ref uri-status 'options))) + (acons 'start-date start-date + (start-downloader downloader uri options + #:directory dir)))))))) + +(define *completed-download-hook* (make-hook 1)) + +(define (download-file downloader status-dir uri options) + "Start downloading file located at URI @var{uri}, with options +@var{options}, resuming it if it's already started." + (catch 'system-error + (lambda () + (and-let* ((status (start-file-download downloader + status-dir + uri options)) + (pid (assoc-ref status 'downloader-pid)) + (filename (uri-status-file-name status-dir + uri)) + (file (open-file filename "w"))) + + ;; Write down the status + (pretty-print status file) + (close-port file) + + ;; Wait for `gnunet-download' + (gnunet-info "Waiting for process ~a" pid) + (let* ((process-status (waitpid pid)) + (exit-val (status:exit-val (cdr process-status))) + (term-sig (status:term-sig (cdr process-status)))) + + ;; Terminate + (delete-file filename) + (gnunet-info + "Download completed (PID ~a, exit code ~a)" + pid exit-val) + (let ((ret `((end-date . ,(current-time)) + (exit-code . ,exit-val) + (terminating-signal . ,term-sig) + ,@status))) + (run-hook *completed-download-hook* ret) + ret)))) + (lambda (key . args) + (gnunet-error (exception-string key args))))) + +(define (uri-status-files directory) + "Return the list of URI status files in @var{directory}." + (catch 'system-error + (lambda () + (let ((dir (opendir directory))) + (let loop ((filename (readdir dir)) + (file-list '())) + (if (eof-object? filename) + file-list + (if (regexp-exec *uri-status-file-re* filename) + (loop (readdir dir) + (cons filename file-list)) + (loop (readdir dir) file-list)))))) + (lambda (key . args) + (gnunet-error (exception-string key args))))) + +(define (output-file-option option-list) + "Return the output file specified in @var{option-list}, false if +anavailable." + (if (null? option-list) + #f + (let ((rest (cdr option-list)) + (opt (car option-list))) + (if (null? rest) + #f + (if (or (string=? opt "-o") + (string=? opt "--output")) + (car rest) + (output-file-option rest)))))) + +(define (download-command . args) + "Start downloading a file using the given `gnunet-download' +arguments." + (gnunet-debug "download-command") + (let* ((argc (length args)) + ;; FIXME: We're assuming the URI is the last argument + (uri (car (list-tail args (- argc 1)))) + (options args)) + (download-file *gnunet-download* *status-directory* uri options))) + +(define (status-command . args) + "Print status info about files being downloaded." + (for-each (lambda (status) + (format #t "~a: ~a~% ~a~% ~a~% ~a~%" + (assoc-ref status 'uri) + (if (download-process-alive? status) + (string-append "running (PID " + (number->string (assoc-ref status + 'pid)) + ")") + "not running") + (string-append "Started on " + (strftime "%c" + (localtime (assoc-ref + status + 'start-date)))) + (string-append "Directory: " + (assoc-ref status + 'working-directory)) + (string-append "Output file: " + (or (output-file-option (assoc-ref + status + 'options)) + "<unknown>")))) + (map (lambda (file) + (uri-status *status-directory* + (string-append "gnunet://afs/" file))) + (uri-status-files *status-directory*)))) + +(define (resume-command . args) + "Resume stopped downloads." + (for-each (lambda (status) + (if (not (download-process-alive? status)) + (if (= 0 (primitive-fork)) + (let* ((ret (download-file *gnunet-download* + *status-directory* + (assoc-ref status 'uri) + (assoc-ref status 'options))) + (code (assoc-ref ret 'exit-code))) + (exit code))))) + (map (lambda (file) + (uri-status *status-directory* + (string-append "gnunet://afs/" file))) + (uri-status-files *status-directory*)))) + +(define (killall-command . args) + "Stop all running downloads." + (for-each (lambda (status) + (if (download-process-alive? status) + (let ((pid (assoc-ref status 'pid)) + (dl-pid (assoc-ref status 'downloader-pid))) + (and (gnunet-info "Stopping processes ~a and ~a" + pid dl-pid) + (kill pid 15) + (kill dl-pid 15))))) + (map (lambda (file) + (uri-status *status-directory* + (string-append "gnunet://afs/" file))) + (uri-status-files *status-directory*)))) + + +(define (help-command . args) + "Show this help message." + (format #t "Usage: ~a <command> [options]~%" *program-name*) + (format #t "Where <command> may be one of the following:~%~%") + (for-each (lambda (command) + (if (not (eq? (cdr command) help-command)) + (format #t (string-append " " (car command) ": " + (object-documentation + (cdr command)) + "~%")))) + *commands*) + (format #t "~%")) + +(define (settings-command . args) + "Dump the current settings." + (format #t "Current settings:~%~%") + (module-for-each (lambda (symbol variable) + (if (string-match "^\\*.*\\*$" (symbol->string symbol)) + (format #t " ~a: ~a~%" + symbol (variable-ref variable)))) + (current-module)) + (format #t "~%")) + +(define (version-command . args) + "Show version information." + (format #t "~a ~a.~a (~a)~%" + *program-name* *version-major* *version-minor* *version-date*)) + +;; This hook may be added to *completed-download-hook*. +(define (completed-download-notification-hook status) + "Notifies of the completion of a file download." + (let ((msg (string-append "GNUnet download of " + (output-file-option + (assoc-ref status 'options)) + " in " + (assoc-ref status + 'working-directory) + " complete!"))) + (if (getenv "DISPLAY") + (waitpid (fork-and-exec #f "xmessage" msg)) + (waitpid (fork-and-exec #f "write" + (cuserid) msg))))) + +;; Available user commands +(define *commands* + `(("download" . ,download-command) + ("status" . ,status-command) + ("resume" . ,resume-command) + ("killall" . ,killall-command) + ("settings" . ,settings-command) + ("version" . ,version-command) + ("help" . ,help-command) + ("--help" . ,help-command) + ("-h" . ,help-command))) + +(define *program-name* "gnunet-download-manager") +(define *version-major* 0) +(define *version-minor* 1) +(define *version-date* "april 2004") + +(define (main args) + (set! *program-name* (basename (car args))) + + ;; Load the user's configuration file + (if (file-exists? *rc-file*) + (load *rc-file*)) + + ;; Check whether the status directory already exists + (if (not (file-exists? *status-directory*)) + (begin + (gnunet-info "Creating status directory ~a..." *status-directory*) + (catch 'system-error + (lambda () + (mkdir *status-directory*)) + (lambda (key . args) + (and (gnunet-error (exception-string key args)) + (exit 1)))))) + + ;; Go ahead + (if (< (length args) 2) + (and (format #t "Usage: ~a <command> [options]~%" + *program-name*) + (exit 1)) + (let* ((command-name (cadr args)) + (command (assoc-ref *commands* command-name))) + (if command + (apply command (cddr args)) + (and (gnunet-info "~a command not found" command-name) + (exit 1))))))
\ No newline at end of file diff --git a/src/fs/gnunet-download.c b/src/fs/gnunet-download.c new file mode 100644 index 0000000..ff10c39 --- /dev/null +++ b/src/fs/gnunet-download.c @@ -0,0 +1,281 @@ +/* + This file is part of GNUnet. + (C) 2001, 2002, 2004, 2005, 2006, 2007, 2009 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ +/** + * @file fs/gnunet-download.c + * @brief downloading for files on GNUnet + * @author Christian Grothoff + * @author Krista Bennett + * @author James Blackwell + * @author Igor Wronsky + */ +#include "platform.h" +#include "gnunet_fs_service.h" + +static int ret; + +static int verbose; + +static int delete_incomplete; + +static const struct GNUNET_CONFIGURATION_Handle *cfg; + +static struct GNUNET_FS_Handle *ctx; + +static struct GNUNET_FS_DownloadContext *dc; + +static unsigned int anonymity = 1; + +static unsigned int parallelism = 16; + +static unsigned int request_parallelism = 4092; + +static int do_recursive; + +static char *filename; + +static int local_only; + +static void +cleanup_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + GNUNET_FS_stop (ctx); + ctx = NULL; +} + + +static void +shutdown_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct GNUNET_FS_DownloadContext *d; + + if (dc != NULL) + { + d = dc; + dc = NULL; + GNUNET_FS_download_stop (d, delete_incomplete); + } +} + + +/** + * Called by FS client to give information about the progress of an + * operation. + * + * @param cls closure + * @param info details about the event, specifying the event type + * and various bits about the event + * @return client-context (for the next progress call + * for this operation; should be set to NULL for + * SUSPEND and STOPPED events). The value returned + * will be passed to future callbacks in the respective + * field in the GNUNET_FS_ProgressInfo struct. + */ +static void * +progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *info) +{ + char *s, *s2; + char *t; + + switch (info->status) + { + case GNUNET_FS_STATUS_DOWNLOAD_START: + if (verbose > 1) + FPRINTF (stderr, _("Starting download `%s'.\n"), + info->value.download.filename); + break; + case GNUNET_FS_STATUS_DOWNLOAD_PROGRESS: + if (verbose) + { + s = GNUNET_STRINGS_relative_time_to_string (info->value.download.eta); + if (info->value.download.specifics.progress.block_download_duration.rel_value + == GNUNET_TIME_UNIT_FOREVER_REL.rel_value) + s2 = GNUNET_strdup (_("<unknown time>")); + else + s2 = GNUNET_STRINGS_relative_time_to_string ( + info->value.download.specifics.progress.block_download_duration); + t = GNUNET_STRINGS_byte_size_fancy (info->value.download.completed * + 1000LL / + (info->value.download. + duration.rel_value + 1)); + FPRINTF (stdout, + _("Downloading `%s' at %llu/%llu (%s remaining, %s/s). Block took %s to download\n"), + info->value.download.filename, + (unsigned long long) info->value.download.completed, + (unsigned long long) info->value.download.size, s, t, s2); + GNUNET_free (s); + GNUNET_free (s2); + GNUNET_free (t); + } + break; + case GNUNET_FS_STATUS_DOWNLOAD_ERROR: + FPRINTF (stderr, _("Error downloading: %s.\n"), + info->value.download.specifics.error.message); + GNUNET_SCHEDULER_shutdown (); + break; + case GNUNET_FS_STATUS_DOWNLOAD_COMPLETED: + s = GNUNET_STRINGS_byte_size_fancy (info->value.download.completed * 1000 / + (info->value.download. + duration.rel_value + 1)); + FPRINTF (stdout, _("Downloading `%s' done (%s/s).\n"), + info->value.download.filename, s); + GNUNET_free (s); + if (info->value.download.dc == dc) + GNUNET_SCHEDULER_shutdown (); + break; + case GNUNET_FS_STATUS_DOWNLOAD_STOPPED: + if (info->value.download.dc == dc) + GNUNET_SCHEDULER_add_continuation (&cleanup_task, NULL, + GNUNET_SCHEDULER_REASON_PREREQ_DONE); + break; + case GNUNET_FS_STATUS_DOWNLOAD_ACTIVE: + case GNUNET_FS_STATUS_DOWNLOAD_INACTIVE: + break; + default: + FPRINTF (stderr, _("Unexpected status: %d\n"), info->status); + break; + } + return NULL; +} + + +/** + * Main function that will be run by the scheduler. + * + * @param cls closure + * @param args remaining command-line arguments + * @param cfgfile name of the configuration file used (for saving, can be NULL!) + * @param c configuration + */ +static void +run (void *cls, char *const *args, const char *cfgfile, + const struct GNUNET_CONFIGURATION_Handle *c) +{ + struct GNUNET_FS_Uri *uri; + char *emsg; + enum GNUNET_FS_DownloadOptions options; + + if (NULL == args[0]) + { + FPRINTF (stderr, "%s", _("You need to specify a URI argument.\n")); + return; + } + uri = GNUNET_FS_uri_parse (args[0], &emsg); + if (NULL == uri) + { + FPRINTF (stderr, _("Failed to parse URI: %s\n"), emsg); + GNUNET_free (emsg); + ret = 1; + return; + } + if ((!GNUNET_FS_uri_test_chk (uri)) && (!GNUNET_FS_uri_test_loc (uri))) + { + FPRINTF (stderr, "%s", _("Only CHK or LOC URIs supported.\n")); + ret = 1; + GNUNET_FS_uri_destroy (uri); + return; + } + if (NULL == filename) + { + FPRINTF (stderr, "%s", _("Target filename must be specified.\n")); + ret = 1; + GNUNET_FS_uri_destroy (uri); + return; + } + cfg = c; + ctx = + GNUNET_FS_start (cfg, "gnunet-download", &progress_cb, NULL, + GNUNET_FS_FLAGS_NONE, + GNUNET_FS_OPTIONS_DOWNLOAD_PARALLELISM, parallelism, + GNUNET_FS_OPTIONS_REQUEST_PARALLELISM, + request_parallelism, GNUNET_FS_OPTIONS_END); + if (NULL == ctx) + { + FPRINTF (stderr, _("Could not initialize `%s' subsystem.\n"), "FS"); + GNUNET_FS_uri_destroy (uri); + ret = 1; + return; + } + options = GNUNET_FS_DOWNLOAD_OPTION_NONE; + if (do_recursive) + options |= GNUNET_FS_DOWNLOAD_OPTION_RECURSIVE; + if (local_only) + options |= GNUNET_FS_DOWNLOAD_OPTION_LOOPBACK_ONLY; + dc = GNUNET_FS_download_start (ctx, uri, NULL, filename, NULL, 0, + GNUNET_FS_uri_chk_get_file_size (uri), + anonymity, options, NULL, NULL); + GNUNET_FS_uri_destroy (uri); + if (dc == NULL) + { + GNUNET_FS_stop (ctx); + ctx = NULL; + return; + } + GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_FOREVER_REL, &shutdown_task, + NULL); +} + + +/** + * The main function to download GNUnet. + * + * @param argc number of arguments from the command line + * @param argv command line arguments + * @return 0 ok, 1 on error + */ +int +main (int argc, char *const *argv) +{ + static const struct GNUNET_GETOPT_CommandLineOption options[] = { + {'a', "anonymity", "LEVEL", + gettext_noop ("set the desired LEVEL of receiver-anonymity"), + 1, &GNUNET_GETOPT_set_uint, &anonymity}, + {'D', "delete-incomplete", NULL, + gettext_noop ("delete incomplete downloads (when aborted with CTRL-C)"), + 0, &GNUNET_GETOPT_set_one, &delete_incomplete}, + {'n', "no-network", NULL, + gettext_noop ("only search the local peer (no P2P network search)"), + 0, &GNUNET_GETOPT_set_uint, &local_only}, + {'o', "output", "FILENAME", + gettext_noop ("write the file to FILENAME"), + 1, &GNUNET_GETOPT_set_string, &filename}, + {'p', "parallelism", "DOWNLOADS", + gettext_noop + ("set the maximum number of parallel downloads that is allowed"), + 1, &GNUNET_GETOPT_set_uint, ¶llelism}, + {'r', "request-parallelism", "REQUESTS", + gettext_noop + ("set the maximum number of parallel requests for blocks that is allowed"), + 1, &GNUNET_GETOPT_set_uint, &request_parallelism}, + {'R', "recursive", NULL, + gettext_noop ("download a GNUnet directory recursively"), + 0, &GNUNET_GETOPT_set_one, &do_recursive}, + {'V', "verbose", NULL, + gettext_noop ("be verbose (print progress information)"), + 0, &GNUNET_GETOPT_increment_value, &verbose}, + GNUNET_GETOPT_OPTION_END + }; + return (GNUNET_OK == + GNUNET_PROGRAM_run (argc, argv, "gnunet-download [OPTIONS] URI", + gettext_noop + ("Download files from GNUnet using a GNUnet CHK or LOC URI (gnunet://fs/chk/...)"), + options, &run, NULL)) ? ret : 1; +} + +/* end of gnunet-download.c */ diff --git a/src/fs/gnunet-fs.c b/src/fs/gnunet-fs.c new file mode 100644 index 0000000..0b28923 --- /dev/null +++ b/src/fs/gnunet-fs.c @@ -0,0 +1,128 @@ +/* + This file is part of GNUnet. + (C) 2011 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ +/** + * @file fs/gnunet-fs.c + * @brief special file-sharing functions + * @author Christian Grothoff + */ +#include "platform.h" +#include "gnunet_fs_service.h" + +/** + * Return value. + */ +static int ret; + +/** + * Handle to FS service. + */ +static struct GNUNET_FS_Handle *fs; + +/** + * Option -i given? + */ +static int list_indexed_files; + +/** + * Option -v given? + */ +static int verbose; + + +/** + * Print indexed filenames to stdout. + * + * @param cls closure + * @param filename the name of the file + * @param file_id hash of the contents of the indexed file + * @return GNUNET_OK to continue iteration + */ +static int +print_indexed (void *cls, const char *filename, const GNUNET_HashCode * file_id) +{ + if (NULL == filename) + { + GNUNET_FS_stop (fs); + fs = NULL; + return GNUNET_OK; + } + if (verbose) + FPRINTF (stdout, "%s: %s\n", GNUNET_h2s (file_id), filename); + else + FPRINTF (stdout, "%s\n", filename); + return GNUNET_OK; +} + + +/** + * Main function that will be run by the scheduler. + * + * @param cls closure + * @param args remaining command-line arguments + * @param cfgfile name of the configuration file used (for saving, can be NULL!) + * @param cfg configuration + */ +static void +run (void *cls, char *const *args, const char *cfgfile, + const struct GNUNET_CONFIGURATION_Handle *cfg) +{ + if (list_indexed_files) + { + fs = GNUNET_FS_start (cfg, "gnunet-fs", NULL, NULL, GNUNET_FS_FLAGS_NONE, + GNUNET_FS_OPTIONS_END); + if (NULL == fs) + { + ret = 1; + return; + } + if (NULL == GNUNET_FS_get_indexed_files (fs, &print_indexed, NULL)) + { + ret = 2; + GNUNET_FS_stop (fs); + fs = NULL; + return; + } + } +} + +/** + * The main function to access special file-sharing functions. + * + * @param argc number of arguments from the command line + * @param argv command line arguments + * @return 0 ok, 1 on error + */ +int +main (int argc, char *const *argv) +{ + static struct GNUNET_GETOPT_CommandLineOption options[] = { + {'i', "list-indexed", NULL, + gettext_noop ("print a list of all indexed files"), 0, + &GNUNET_GETOPT_set_one, &list_indexed_files}, + GNUNET_GETOPT_OPTION_VERBOSE (&verbose), + GNUNET_GETOPT_OPTION_END + }; + return (GNUNET_OK == + GNUNET_PROGRAM_run (argc, argv, "gnunet-fs [OPTIONS]", + gettext_noop ("Special file-sharing operations"), + options, &run, NULL)) ? ret : 1; +} + +/* end of gnunet-fs.c */ diff --git a/src/fs/gnunet-helper-fs-publish.c b/src/fs/gnunet-helper-fs-publish.c new file mode 100644 index 0000000..4f70464 --- /dev/null +++ b/src/fs/gnunet-helper-fs-publish.c @@ -0,0 +1,457 @@ +/* + This file is part of GNUnet. + (C) 2012 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file src/fs/gnunet-helper-fs-publish.c + * @brief Tool to help extract meta data asynchronously + * @author Christian Grothoff + * + * This program will scan a directory for files with meta data + * and report the results to stdout. + */ +#include "platform.h" +#include "gnunet_fs_service.h" + + +/** + * A node of a directory tree. + */ +struct ScanTreeNode +{ + + /** + * This is a doubly-linked list + */ + struct ScanTreeNode *next; + + /** + * This is a doubly-linked list + */ + struct ScanTreeNode *prev; + + /** + * Parent of this node, NULL for top-level entries. + */ + struct ScanTreeNode *parent; + + /** + * This is a doubly-linked tree + * NULL for files and empty directories + */ + struct ScanTreeNode *children_head; + + /** + * This is a doubly-linked tree + * NULL for files and empty directories + */ + struct ScanTreeNode *children_tail; + + /** + * Name of the file/directory + */ + char *filename; + + /** + * Size of the file (if it is a file), in bytes + */ + uint64_t file_size; + + /** + * GNUNET_YES if this is a directory + */ + int is_directory; + +}; + + +/** + * List of libextractor plugins to use for extracting. + */ +static struct EXTRACTOR_PluginList *plugins; + + +/** + * Add meta data that libextractor finds to our meta data + * container. + * + * @param cls closure, our meta data container + * @param plugin_name name of the plugin that produced this value; + * special values can be used (i.e. '<zlib>' for zlib being + * used in the main libextractor library and yielding + * meta data). + * @param type libextractor-type describing the meta data + * @param format basic format information about data + * @param data_mime_type mime-type of data (not of the original file); + * can be NULL (if mime-type is not known) + * @param data actual meta-data found + * @param data_len number of bytes in data + * @return always 0 to continue extracting + */ +static int +add_to_md (void *cls, const char *plugin_name, enum EXTRACTOR_MetaType type, + enum EXTRACTOR_MetaFormat format, const char *data_mime_type, + const char *data, size_t data_len) +{ + struct GNUNET_CONTAINER_MetaData *md = cls; + + (void) GNUNET_CONTAINER_meta_data_insert (md, plugin_name, type, format, + data_mime_type, data, data_len); + return 0; +} + + +/** + * Free memory of the 'tree' structure + * + * @param tree tree to free + */ +static void +free_tree (struct ScanTreeNode *tree) +{ + struct ScanTreeNode *pos; + + while (NULL != (pos = tree->children_head)) + free_tree (pos); + if (NULL != tree->parent) + GNUNET_CONTAINER_DLL_remove (tree->parent->children_head, + tree->parent->children_tail, + tree); + GNUNET_free (tree->filename); + GNUNET_free (tree); +} + + +/** + * Write 'size' bytes from 'buf' into 'out'. + * + * @param buf buffer with data to write + * @param size number of bytes to write + * @return GNUNET_OK on success, GNUNET_SYSERR on error + */ +static int +write_all (const void *buf, + size_t size) +{ + const char *cbuf = buf; + size_t total; + ssize_t wr; + + total = 0; + do + { + wr = write (1, + &cbuf[total], + size - total); + if (wr > 0) + total += wr; + } while ( (wr > 0) && (total < size) ); + if (wr <= 0) + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Failed to write to stdout: %s\n", + strerror (errno)); + return (total == size) ? GNUNET_OK : GNUNET_SYSERR; +} + + +/** + * Write message to the master process. + * + * @param message_type message type to use + * @param data data to append, NULL for none + * @param data_length number of bytes in data + * @return GNUNET_SYSERR to stop scanning (the pipe was broken somehow) + */ +static int +write_message (uint16_t message_type, + const char *data, + size_t data_length) +{ + struct GNUNET_MessageHeader hdr; + + hdr.type = htons (message_type); + hdr.size = htons (sizeof (struct GNUNET_MessageHeader) + data_length); + if ( (GNUNET_OK != + write_all (&hdr, + sizeof (hdr))) || + (GNUNET_OK != + write_all (data, + data_length)) ) + return GNUNET_SYSERR; + return GNUNET_OK; +} + + +/** + * Function called to (recursively) add all of the files in the + * directory to the tree. Called by the directory scanner to initiate + * the scan. Does NOT yet add any metadata. + * + * @param filename file or directory to scan + * @param dst where to store the resulting share tree item + * @return GNUNET_OK on success, GNUNET_SYSERR on error + */ +static int +preprocess_file (const char *filename, + struct ScanTreeNode **dst); + + +/** + * Closure for the 'scan_callback' + */ +struct RecursionContext +{ + /** + * Parent to add the files to. + */ + struct ScanTreeNode *parent; + + /** + * Flag to set to GNUNET_YES on serious errors. + */ + int stop; +}; + + +/** + * Function called by the directory iterator to (recursively) add all + * of the files in the directory to the tree. Called by the directory + * scanner to initiate the scan. Does NOT yet add any metadata. + * + * @param cls the 'struct RecursionContext' + * @param filename file or directory to scan + * @return GNUNET_OK on success, GNUNET_SYSERR on error + */ +static int +scan_callback (void *cls, + const char *filename) +{ + struct RecursionContext *rc = cls; + struct ScanTreeNode *chld; + + if (GNUNET_OK != + preprocess_file (filename, + &chld)) + { + rc->stop = GNUNET_YES; + return GNUNET_SYSERR; + } + chld->parent = rc->parent; + GNUNET_CONTAINER_DLL_insert (rc->parent->children_head, + rc->parent->children_tail, + chld); + return GNUNET_OK; +} + + +/** + * Function called to (recursively) add all of the files in the + * directory to the tree. Called by the directory scanner to initiate + * the scan. Does NOT yet add any metadata. + * + * @param filename file or directory to scan + * @param dst where to store the resulting share tree item + * @return GNUNET_OK on success, GNUNET_SYSERR on error + */ +static int +preprocess_file (const char *filename, + struct ScanTreeNode **dst) +{ + struct ScanTreeNode *item; + struct stat sbuf; + + if (0 != STAT (filename, &sbuf)) + { + /* If the file doesn't exist (or is not stat-able for any other reason) + skip it (but report it), but do continue. */ + if (GNUNET_OK != + write_message (GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_SKIP_FILE, + filename, strlen (filename) + 1)) + return GNUNET_SYSERR; + return GNUNET_OK; + } + + /* Report the progress */ + if (GNUNET_OK != + write_message (S_ISDIR (sbuf.st_mode) + ? GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_DIRECTORY + : GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_FILE, + filename, strlen (filename) + 1)) + return GNUNET_SYSERR; + item = GNUNET_malloc (sizeof (struct ScanTreeNode)); + item->filename = GNUNET_strdup (filename); + item->is_directory = (S_ISDIR (sbuf.st_mode)) ? GNUNET_YES : GNUNET_NO; + item->file_size = (uint64_t) sbuf.st_size; + if (item->is_directory == GNUNET_YES) + { + struct RecursionContext rc; + + rc.parent = item; + rc.stop = GNUNET_NO; + GNUNET_DISK_directory_scan (filename, + &scan_callback, + &rc); + if ( (rc.stop == GNUNET_YES) || + (GNUNET_OK != + write_message (GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_DIRECTORY, + "..", 3)) ) + { + free_tree (item); + return GNUNET_SYSERR; + } + } + *dst = item; + return GNUNET_OK; +} + + +/** + * Extract metadata from files. + * + * @param item entry we are processing + * @return GNUNET_OK on success, GNUNET_SYSERR on fatal errors + */ +static int +extract_files (struct ScanTreeNode *item) +{ + struct GNUNET_CONTAINER_MetaData *meta; + ssize_t size; + size_t slen; + + if (item->is_directory == GNUNET_YES) + { + /* for directories, we simply only descent, no extraction, no + progress reporting */ + struct ScanTreeNode *pos; + + for (pos = item->children_head; NULL != pos; pos = pos->next) + if (GNUNET_OK != + extract_files (pos)) + return GNUNET_SYSERR; + return GNUNET_OK; + } + + /* this is the expensive operation, *afterwards* we'll check for aborts */ + meta = GNUNET_CONTAINER_meta_data_create (); + if (NULL != plugins) + EXTRACTOR_extract (plugins, item->filename, NULL, 0, &add_to_md, meta); + slen = strlen (item->filename) + 1; + size = GNUNET_CONTAINER_meta_data_get_serialized_size (meta); + if (-1 == size) + { + /* no meta data */ + GNUNET_CONTAINER_meta_data_destroy (meta); + if (GNUNET_OK != + write_message (GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_META_DATA, + item->filename, slen)) + return GNUNET_SYSERR; + return GNUNET_OK; + } + { + char buf[size + slen]; + char *dst = &buf[slen]; + + memcpy (buf, item->filename, slen); + size = GNUNET_CONTAINER_meta_data_serialize (meta, + &dst, size - slen, + GNUNET_CONTAINER_META_DATA_SERIALIZE_PART); + GNUNET_CONTAINER_meta_data_destroy (meta); + if (GNUNET_OK != + write_message (GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_META_DATA, + buf, + slen + size)) + return GNUNET_SYSERR; + } + return GNUNET_OK; +} + + +/** + * Main function of the helper process to extract meta data. + * + * @param argc should be 3 + * @param argv [0] our binary name + * [1] name of the file or directory to process + * [2] "-" to disable extraction, NULL for defaults, + * otherwise custom plugins to load from LE + * @return 0 on success + */ +int main(int argc, + char **argv) +{ + const char *filename_expanded; + const char *ex; + struct ScanTreeNode *root; + +#if WINDOWS + /* We're using stdout to communicate binary data back to the parent; use + * binary mode. + */ + _setmode (1, _O_BINARY); +#endif + + /* parse command line */ + if ( (argc != 3) && (argc != 2) ) + { + FPRINTF (stderr, + "%s", + "gnunet-helper-fs-publish needs exactly one or two arguments\n"); + return 1; + } + filename_expanded = argv[1]; + ex = argv[2]; + if ( (ex == NULL) || + (0 != strcmp (ex, "-")) ) + { + plugins = EXTRACTOR_plugin_add_defaults (EXTRACTOR_OPTION_DEFAULT_POLICY); + if (NULL != ex) + plugins = EXTRACTOR_plugin_add_config (plugins, ex, + EXTRACTOR_OPTION_DEFAULT_POLICY); + } + + /* scan tree to find out how much work there is to be done */ + if (GNUNET_OK != preprocess_file (filename_expanded, + &root)) + { + (void) write_message (GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_ERROR, NULL, 0); + return 2; + } + /* signal that we're done counting files, so that a percentage of + progress can now be calculated */ + if (GNUNET_OK != + write_message (GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_COUNTING_DONE, NULL, 0)) + return 3; + if (GNUNET_OK != + extract_files (root)) + { + (void) write_message (GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_ERROR, NULL, 0); + free_tree (root); + return 4; + } + free_tree (root); + /* enable "clean" shutdown by telling parent that we are done */ + (void) write_message (GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_FINISHED, NULL, 0); + if (NULL != plugins) + EXTRACTOR_plugin_remove_all (plugins); + + return 0; +} + +/* end of gnunet-helper-fs-publish.c */ + diff --git a/src/fs/gnunet-pseudonym.c b/src/fs/gnunet-pseudonym.c new file mode 100644 index 0000000..412ddd2 --- /dev/null +++ b/src/fs/gnunet-pseudonym.c @@ -0,0 +1,312 @@ +/* + This file is part of GNUnet. + (C) 2001, 2002, 2004, 2005, 2006, 2007, 2009, 2010 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ +/** + * @file fs/gnunet-pseudonym.c + * @brief manage GNUnet namespaces / pseudonyms + * @author Christian Grothoff + */ +#include "platform.h" +#include "gnunet_fs_service.h" + +/** + * -C option + */ +static char *create_ns; + +/** + * -D option + */ +static char *delete_ns; + +/** + * -k option + */ +static struct GNUNET_FS_Uri *ksk_uri; + +/** + * -l option. + */ +static int print_local_only; + +/** + * -m option. + */ +static struct GNUNET_CONTAINER_MetaData *adv_metadata; + +/** + * Our block options (-p, -r, -a). + */ +static struct GNUNET_FS_BlockOptions bo = { {0LL}, 1, 365, 1 }; + +/** + * -q option given. + */ +static int no_remote_printing; + +/** + * -r option. + */ +static char *root_identifier; + +/** + * -s option. + */ +static char *rating_change; + +/** + * Handle to fs service. + */ +static struct GNUNET_FS_Handle *h; + +/** + * Namespace we are looking at. + */ +static struct GNUNET_FS_Namespace *ns; + +/** + * Our configuration. + */ +static const struct GNUNET_CONFIGURATION_Handle *cfg; + +static int ret; + +static void * +progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *info) +{ + return NULL; +} + + +static void +ns_printer (void *cls, const char *name, const GNUNET_HashCode * id) +{ + struct GNUNET_CRYPTO_HashAsciiEncoded enc; + + GNUNET_CRYPTO_hash_to_enc (id, &enc); + FPRINTF (stdout, "%s (%s)\n", name, (const char *) &enc); +} + + +static int +pseudo_printer (void *cls, const GNUNET_HashCode * pseudonym, + const struct GNUNET_CONTAINER_MetaData *md, int rating) +{ + char *id; + + id = GNUNET_PSEUDONYM_id_to_name (cfg, pseudonym); + if (id == NULL) + { + GNUNET_break (0); + return GNUNET_OK; + } + FPRINTF (stdout, "%s (%d):\n", id, rating); + GNUNET_CONTAINER_meta_data_iterate (md, &EXTRACTOR_meta_data_print, stdout); + FPRINTF (stdout, "%s", "\n"); + GNUNET_free (id); + return GNUNET_OK; +} + + +static void +post_advertising (void *cls, const struct GNUNET_FS_Uri *uri, const char *emsg) +{ + GNUNET_HashCode nsid; + char *set; + int delta; + + if (emsg != NULL) + { + FPRINTF (stderr, "%s", emsg); + ret = 1; + } + if (ns != NULL) + { + if (GNUNET_OK != GNUNET_FS_namespace_delete (ns, GNUNET_NO)) + ret = 1; + } + if (NULL != rating_change) + { + set = rating_change; + while ((*set != '\0') && (*set != ':')) + set++; + if (*set != ':') + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, _("Invalid argument `%s'\n"), + rating_change); + } + else + { + *set = '\0'; + delta = strtol (&set[1], NULL, /* no error handling yet */ + 10); + if (GNUNET_OK == GNUNET_PSEUDONYM_name_to_id (cfg, rating_change, &nsid)) + { + (void) GNUNET_PSEUDONYM_rank (cfg, &nsid, delta); + } + else + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, _("Namespace `%s' unknown.\n"), + rating_change); + } + } + GNUNET_free (rating_change); + rating_change = NULL; + } + if (0 != print_local_only) + { + GNUNET_FS_namespace_list (h, &ns_printer, NULL); + } + else if (0 == no_remote_printing) + { + GNUNET_PSEUDONYM_list_all (cfg, &pseudo_printer, NULL); + } + GNUNET_FS_stop (h); +} + + +/** + * Main function that will be run by the scheduler. + * + * @param cls closure + * @param args remaining command-line arguments + * @param cfgfile name of the configuration file used (for saving, can be NULL!) + * @param c configuration + */ +static void +run (void *cls, char *const *args, const char *cfgfile, + const struct GNUNET_CONFIGURATION_Handle *c) +{ + char *emsg; + + cfg = c; + h = GNUNET_FS_start (cfg, "gnunet-pseudonym", &progress_cb, NULL, + GNUNET_FS_FLAGS_NONE, GNUNET_FS_OPTIONS_END); + if (NULL != delete_ns) + { + ns = GNUNET_FS_namespace_create (h, delete_ns); + if (ns == NULL) + { + ret = 1; + } + else + { + if (GNUNET_OK != GNUNET_FS_namespace_delete (ns, GNUNET_YES)) + ret = 1; + ns = NULL; + } + } + if (NULL != create_ns) + { + ns = GNUNET_FS_namespace_create (h, create_ns); + if (ns == NULL) + { + ret = 1; + } + else + { + if (NULL != root_identifier) + { + if (ksk_uri == NULL) + { + emsg = NULL; + ksk_uri = GNUNET_FS_uri_parse ("gnunet://fs/ksk/namespace", &emsg); + GNUNET_assert (NULL == emsg); + } + GNUNET_FS_namespace_advertise (h, ksk_uri, ns, adv_metadata, &bo, + root_identifier, &post_advertising, + NULL); + return; + } + else + { + if (ksk_uri != NULL) + FPRINTF (stderr, _("Option `%s' ignored\n"), "-k"); + } + } + } + else + { + if (root_identifier != NULL) + FPRINTF (stderr, _("Option `%s' ignored\n"), "-r"); + if (ksk_uri != NULL) + FPRINTF (stderr, _("Option `%s' ignored\n"), "-k"); + } + + post_advertising (NULL, NULL, NULL); +} + + +/** + * The main function to manipulate GNUnet pseudonyms (and publish + * to namespaces). + * + * @param argc number of arguments from the command line + * @param argv command line arguments + * @return 0 ok, 1 on error + */ +int +main (int argc, char *const *argv) +{ + static const struct GNUNET_GETOPT_CommandLineOption options[] = { + {'a', "anonymity", "LEVEL", + gettext_noop ("set the desired LEVEL of sender-anonymity"), + 1, &GNUNET_GETOPT_set_uint, &bo.anonymity_level}, + {'C', "create", "NAME", + gettext_noop ("create or advertise namespace NAME"), + 1, &GNUNET_GETOPT_set_string, &create_ns}, + {'D', "delete", "NAME", + gettext_noop ("delete namespace NAME "), + 1, &GNUNET_GETOPT_set_string, &delete_ns}, + {'k', "keyword", "VALUE", + gettext_noop ("add an additional keyword for the advertisment" + " (this option can be specified multiple times)"), + 1, &GNUNET_FS_getopt_set_keywords, &ksk_uri}, + {'m', "meta", "TYPE:VALUE", + gettext_noop ("set the meta-data for the given TYPE to the given VALUE"), + 1, &GNUNET_FS_getopt_set_metadata, &adv_metadata}, + {'o', "only-local", NULL, + gettext_noop ("print names of local namespaces"), + 0, &GNUNET_GETOPT_set_one, &print_local_only}, + {'p', "priority", "PRIORITY", + gettext_noop ("use the given PRIORITY for the advertisments"), + 1, &GNUNET_GETOPT_set_uint, &bo.content_priority}, + {'q', "quiet", NULL, + gettext_noop ("do not print names of remote namespaces"), + 0, &GNUNET_GETOPT_set_one, &no_remote_printing}, + {'r', "replication", "LEVEL", + gettext_noop ("set the desired replication LEVEL"), + 1, &GNUNET_GETOPT_set_uint, &bo.replication_level}, + {'R', "root", "ID", + gettext_noop ("specify ID of the root of the namespace"), + 1, &GNUNET_GETOPT_set_string, &root_identifier}, + {'s', "set-rating", "ID:VALUE", + gettext_noop ("change rating of namespace ID by VALUE"), + 1, &GNUNET_GETOPT_set_string, &rating_change}, + GNUNET_GETOPT_OPTION_END + }; + bo.expiration_time = + GNUNET_FS_year_to_time (GNUNET_FS_get_current_year () + 2); + return (GNUNET_OK == + GNUNET_PROGRAM_run (argc, argv, "gnunet-pseudonym [OPTIONS]", + gettext_noop ("Manage GNUnet pseudonyms."), + options, &run, NULL)) ? ret : 1; +} + +/* end of gnunet-pseudonym.c */ diff --git a/src/fs/gnunet-publish.c b/src/fs/gnunet-publish.c new file mode 100644 index 0000000..50f507d --- /dev/null +++ b/src/fs/gnunet-publish.c @@ -0,0 +1,740 @@ +/* + This file is part of GNUnet. + (C) 2001, 2002, 2004, 2005, 2006, 2007, 2009, 2010, 2011 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ +/** + * @file fs/gnunet-publish.c + * @brief publishing files on GNUnet + * @author Christian Grothoff + * @author Krista Bennett + * @author James Blackwell + * @author Igor Wronsky + */ +#include "platform.h" +#include "gnunet_fs_service.h" + +static int ret; + +static int verbose; + +static const struct GNUNET_CONFIGURATION_Handle *cfg; + +static struct GNUNET_FS_Handle *ctx; + +static struct GNUNET_FS_PublishContext *pc; + +static struct GNUNET_CONTAINER_MetaData *meta; + +static struct GNUNET_FS_Uri *topKeywords; + +static struct GNUNET_FS_Uri *uri; + +static struct GNUNET_FS_BlockOptions bo = { {0LL}, 1, 365, 1 }; + +static char *uri_string; + +static char *next_id; + +static char *this_id; + +static char *pseudonym; + +static int do_insert; + +static int disable_extractor; + +static int do_simulate; + +static int extract_only; + +static int do_disable_creation_time; + +static GNUNET_SCHEDULER_TaskIdentifier kill_task; + +static struct GNUNET_FS_DirScanner *ds; + +static struct GNUNET_FS_ShareTreeItem * directory_scan_result; + +static struct GNUNET_FS_Namespace *namespace; + +/** + * FIXME: docu + */ +static void +do_stop_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct GNUNET_FS_PublishContext *p; + + kill_task = GNUNET_SCHEDULER_NO_TASK; + if (pc != NULL) + { + p = pc; + pc = NULL; + GNUNET_FS_publish_stop (p); + } + if (NULL != meta) + { + GNUNET_CONTAINER_meta_data_destroy (meta); + meta = NULL; + } +} + + +/** + * Stop the directory scanner (we had an error). + * + * @param cls closure + * @param tc scheduler context + */ +static void +stop_scanner_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + kill_task = GNUNET_SCHEDULER_NO_TASK; + GNUNET_FS_directory_scan_abort (ds); + ds = NULL; + if (namespace != NULL) + { + GNUNET_FS_namespace_delete (namespace, GNUNET_NO); + namespace = NULL; + } + GNUNET_FS_stop (ctx); + ctx = NULL; + ret = 1; +} + + +/** + * Called by FS client to give information about the progress of an + * operation. + * + * @param cls closure + * @param info details about the event, specifying the event type + * and various bits about the event + * @return client-context (for the next progress call + * for this operation; should be set to NULL for + * SUSPEND and STOPPED events). The value returned + * will be passed to future callbacks in the respective + * field in the GNUNET_FS_ProgressInfo struct. + */ +static void * +progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *info) +{ + char *s; + + switch (info->status) + { + case GNUNET_FS_STATUS_PUBLISH_START: + break; + case GNUNET_FS_STATUS_PUBLISH_PROGRESS: + if (verbose) + { + s = GNUNET_STRINGS_relative_time_to_string (info->value.publish.eta); + FPRINTF (stdout, _("Publishing `%s' at %llu/%llu (%s remaining)\n"), + info->value.publish.filename, + (unsigned long long) info->value.publish.completed, + (unsigned long long) info->value.publish.size, s); + GNUNET_free (s); + } + break; + case GNUNET_FS_STATUS_PUBLISH_ERROR: + FPRINTF (stderr, _("Error publishing: %s.\n"), + info->value.publish.specifics.error.message); + if (kill_task != GNUNET_SCHEDULER_NO_TASK) + { + GNUNET_SCHEDULER_cancel (kill_task); + kill_task = GNUNET_SCHEDULER_NO_TASK; + } + kill_task = GNUNET_SCHEDULER_add_now (&do_stop_task, NULL); + break; + case GNUNET_FS_STATUS_PUBLISH_COMPLETED: + FPRINTF (stdout, _("Publishing `%s' done.\n"), + info->value.publish.filename); + s = GNUNET_FS_uri_to_string (info->value.publish.specifics. + completed.chk_uri); + FPRINTF (stdout, _("URI is `%s'.\n"), s); + GNUNET_free (s); + if (info->value.publish.pctx == NULL) + { + if (kill_task != GNUNET_SCHEDULER_NO_TASK) + { + GNUNET_SCHEDULER_cancel (kill_task); + kill_task = GNUNET_SCHEDULER_NO_TASK; + } + kill_task = GNUNET_SCHEDULER_add_now (&do_stop_task, NULL); + } + break; + case GNUNET_FS_STATUS_PUBLISH_STOPPED: + GNUNET_break (NULL == pc); + return NULL; + case GNUNET_FS_STATUS_UNINDEX_PROGRESS: + return NULL; + case GNUNET_FS_STATUS_UNINDEX_COMPLETED: + FPRINTF (stderr, "%s", _("Cleanup after abort complete.\n")); + return NULL; + default: + FPRINTF (stderr, _("Unexpected status: %d\n"), info->status); + return NULL; + } + return ""; /* non-null */ +} + + +/** + * Print metadata entries (except binary + * metadata and the filename). + * + * @param cls closure + * @param plugin_name name of the plugin that generated the meta data + * @param type type of the meta data + * @param format format of data + * @param data_mime_type mime type of data + * @param data value of the meta data + * @param data_size number of bytes in data + * @return always 0 + */ +static int +meta_printer (void *cls, const char *plugin_name, enum EXTRACTOR_MetaType type, + enum EXTRACTOR_MetaFormat format, const char *data_mime_type, + const char *data, size_t data_size) +{ + if ((format != EXTRACTOR_METAFORMAT_UTF8) && + (format != EXTRACTOR_METAFORMAT_C_STRING)) + return 0; + if (type == EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME) + return 0; + FPRINTF (stdout, "\t%s - %s\n", EXTRACTOR_metatype_to_string (type), data); + return 0; +} + + +/** + * Iterator printing keywords + * + * @param cls closure + * @param keyword the keyword + * @param is_mandatory is the keyword mandatory (in a search) + * @return GNUNET_OK to continue to iterate, GNUNET_SYSERR to abort + */ +static int +keyword_printer (void *cls, const char *keyword, int is_mandatory) +{ + FPRINTF (stdout, "\t%s\n", keyword); + return GNUNET_OK; +} + + +/** + * Function called on all entries before the publication. This is + * where we perform modifications to the default based on command-line + * options. + * + * @param cls closure + * @param fi the entry in the publish-structure + * @param length length of the file or directory + * @param m metadata for the file or directory (can be modified) + * @param uri pointer to the keywords that will be used for this entry (can be modified) + * @param bo block options + * @param do_index should we index? + * @param client_info pointer to client context set upon creation (can be modified) + * @return GNUNET_OK to continue, GNUNET_NO to remove + * this entry from the directory, GNUNET_SYSERR + * to abort the iteration + */ +static int +publish_inspector (void *cls, struct GNUNET_FS_FileInformation *fi, + uint64_t length, struct GNUNET_CONTAINER_MetaData *m, + struct GNUNET_FS_Uri **uri, + struct GNUNET_FS_BlockOptions *bo, int *do_index, + void **client_info) +{ + char *fn; + char *fs; + struct GNUNET_FS_Uri *new_uri; + + if (cls == fi) + return GNUNET_OK; + if (NULL != topKeywords) + { + if (*uri != NULL) + { + new_uri = GNUNET_FS_uri_ksk_merge (topKeywords, *uri); + GNUNET_FS_uri_destroy (*uri); + *uri = new_uri; + GNUNET_FS_uri_destroy (topKeywords); + } + else + { + *uri = topKeywords; + } + topKeywords = NULL; + } + if (NULL != meta) + { + GNUNET_CONTAINER_meta_data_merge (m, meta); + GNUNET_CONTAINER_meta_data_destroy (meta); + meta = NULL; + } + if (!do_disable_creation_time) + GNUNET_CONTAINER_meta_data_add_publication_date (m); + if (extract_only) + { + fn = GNUNET_CONTAINER_meta_data_get_by_type (m, + EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME); + fs = GNUNET_STRINGS_byte_size_fancy (length); + FPRINTF (stdout, _("Meta data for file `%s' (%s)\n"), fn, fs); + GNUNET_CONTAINER_meta_data_iterate (m, &meta_printer, NULL); + FPRINTF (stdout, _("Keywords for file `%s' (%s)\n"), fn, fs); + GNUNET_free (fn); + GNUNET_free (fs); + if (NULL != *uri) + GNUNET_FS_uri_ksk_get_keywords (*uri, &keyword_printer, NULL); + FPRINTF (stdout, "%s", "\n"); + } + if (GNUNET_YES == GNUNET_FS_meta_data_test_for_directory (m)) + GNUNET_FS_file_information_inspect (fi, &publish_inspector, fi); + return GNUNET_OK; +} + + +/** + * FIXME: docu + */ +static void +uri_sks_continuation (void *cls, const struct GNUNET_FS_Uri *ksk_uri, + const char *emsg) +{ + if (emsg != NULL) + { + FPRINTF (stderr, "%s\n", emsg); + ret = 1; + } + GNUNET_FS_uri_destroy (uri); + uri = NULL; + GNUNET_FS_stop (ctx); + ctx = NULL; +} + + +/** + * FIXME: docu + */ +static void +uri_ksk_continuation (void *cls, const struct GNUNET_FS_Uri *ksk_uri, + const char *emsg) +{ + struct GNUNET_FS_Namespace *ns; + + if (emsg != NULL) + { + FPRINTF (stderr, "%s\n", emsg); + ret = 1; + } + if (pseudonym != NULL) + { + ns = GNUNET_FS_namespace_create (ctx, pseudonym); + if (ns == NULL) + { + FPRINTF (stderr, _("Failed to create namespace `%s'\n"), pseudonym); + ret = 1; + } + else + { + GNUNET_FS_publish_sks (ctx, ns, this_id, next_id, meta, uri, &bo, + GNUNET_FS_PUBLISH_OPTION_NONE, + &uri_sks_continuation, NULL); + GNUNET_assert (GNUNET_OK == GNUNET_FS_namespace_delete (ns, GNUNET_NO)); + return; + } + } + GNUNET_FS_uri_destroy (uri); + uri = NULL; + GNUNET_FS_stop (ctx); + ctx = NULL; +} + + +/** + * FIXME: docu + */ +static struct GNUNET_FS_FileInformation * +get_file_information (struct GNUNET_FS_ShareTreeItem *item) +{ + struct GNUNET_FS_FileInformation *fi; + struct GNUNET_FS_FileInformation *fic; + struct GNUNET_FS_ShareTreeItem *child; + + if (item->is_directory == GNUNET_YES) + { + GNUNET_CONTAINER_meta_data_delete (item->meta, + EXTRACTOR_METATYPE_MIMETYPE, NULL, 0); + GNUNET_FS_meta_data_make_directory (item->meta); + if (NULL == item->ksk_uri) + { + const char *mime = GNUNET_FS_DIRECTORY_MIME; + item->ksk_uri = GNUNET_FS_uri_ksk_create_from_args (1, &mime); + } + else + GNUNET_FS_uri_ksk_add_keyword (item->ksk_uri, GNUNET_FS_DIRECTORY_MIME, + GNUNET_NO); + fi = GNUNET_FS_file_information_create_empty_directory ( + ctx, NULL, item->ksk_uri, + item->meta, &bo, item->filename); + for (child = item->children_head; child; child = child->next) + { + fic = get_file_information (child); + GNUNET_break (GNUNET_OK == GNUNET_FS_file_information_add (fi, fic)); + } + } + else + { + fi = GNUNET_FS_file_information_create_from_file ( + ctx, NULL, item->filename, + item->ksk_uri, item->meta, !do_insert, + &bo); + } + return fi; +} + + +/** + * FIXME: docu + */ +static void +directory_trim_complete () +{ + struct GNUNET_FS_FileInformation *fi; + + fi = get_file_information (directory_scan_result); + GNUNET_FS_share_tree_free (directory_scan_result); + directory_scan_result = NULL; + if (fi == NULL) + { + FPRINTF (stderr, "%s", _("Could not publish\n")); + if (namespace != NULL) + GNUNET_FS_namespace_delete (namespace, GNUNET_NO); + GNUNET_FS_stop (ctx); + ret = 1; + return; + } + GNUNET_FS_file_information_inspect (fi, &publish_inspector, NULL); + if (extract_only) + { + if (namespace != NULL) + GNUNET_FS_namespace_delete (namespace, GNUNET_NO); + GNUNET_FS_file_information_destroy (fi, NULL, NULL); + GNUNET_FS_stop (ctx); + if (kill_task != GNUNET_SCHEDULER_NO_TASK) + { + GNUNET_SCHEDULER_cancel (kill_task); + kill_task = GNUNET_SCHEDULER_NO_TASK; + } + return; + } + pc = GNUNET_FS_publish_start (ctx, fi, namespace, this_id, next_id, + (do_simulate) ? + GNUNET_FS_PUBLISH_OPTION_SIMULATE_ONLY : + GNUNET_FS_PUBLISH_OPTION_NONE); + if (NULL == pc) + { + FPRINTF (stderr, "%s", _("Could not start publishing.\n")); + GNUNET_FS_stop (ctx); + ret = 1; + return; + } +} + + +/** + * Function called by the directory scanner as we build the tree + * that we will need to publish later. + * + * @param cls closure + * @param filename which file we are making progress on + * @param is_directory GNUNET_YES if this is a directory, + * GNUNET_NO if this is a file + * GNUNET_SYSERR if it is neither (or unknown) + * @param reason kind of progress we are making + */ +static void +directory_scan_cb (void *cls, + const char *filename, + int is_directory, + enum GNUNET_FS_DirScannerProgressUpdateReason reason) +{ + switch (reason) + { + case GNUNET_FS_DIRSCANNER_FILE_START: + if (verbose > 1) + { + if (is_directory == GNUNET_YES) + FPRINTF (stdout, _("Scanning directory `%s'.\n"), filename); + else + FPRINTF (stdout, _("Scanning file `%s'.\n"), filename); + } + break; + case GNUNET_FS_DIRSCANNER_FILE_IGNORED: + FPRINTF (stderr, + _("There was trouble processing file `%s', skipping it.\n"), + filename); + break; + case GNUNET_FS_DIRSCANNER_ALL_COUNTED: + if (verbose) + FPRINTF (stdout, "%s", _("Preprocessing complete.\n")); + break; + case GNUNET_FS_DIRSCANNER_EXTRACT_FINISHED: + if (verbose > 2) + FPRINTF (stdout, _("Extracting meta data from file `%s' complete.\n"), filename); + break; + case GNUNET_FS_DIRSCANNER_FINISHED: + if (verbose > 1) + FPRINTF (stdout, "%s", _("Meta data extraction has finished.\n")); + directory_scan_result = GNUNET_FS_directory_scan_get_result (ds); + ds = NULL; + GNUNET_FS_share_tree_trim (directory_scan_result); + directory_trim_complete (); + break; + case GNUNET_FS_DIRSCANNER_INTERNAL_ERROR: + FPRINTF (stdout, "%s", _("Internal error scanning directory.\n")); + if (kill_task != GNUNET_SCHEDULER_NO_TASK) + { + GNUNET_SCHEDULER_cancel (kill_task); + kill_task = GNUNET_SCHEDULER_NO_TASK; + } + kill_task = GNUNET_SCHEDULER_add_now (&stop_scanner_task, NULL); + break; + default: + GNUNET_assert (0); + break; + } + fflush (stdout); +} + + +/** + * Main function that will be run by the scheduler. + * + * @param cls closure + * @param args remaining command-line arguments + * @param cfgfile name of the configuration file used (for saving, can be NULL!) + * @param c configuration + */ +static void +run (void *cls, char *const *args, const char *cfgfile, + const struct GNUNET_CONFIGURATION_Handle *c) +{ + char *ex; + char *emsg; + + /* check arguments */ + if ((uri_string != NULL) && (extract_only)) + { + printf (_("Cannot extract metadata from a URI!\n")); + ret = -1; + return; + } + if (((uri_string == NULL) || (extract_only)) && + ((args[0] == NULL) || (args[1] != NULL))) + { + printf (_("You must specify one and only one filename for insertion.\n")); + ret = -1; + return; + } + if ((uri_string != NULL) && (args[0] != NULL)) + { + printf (_("You must NOT specify an URI and a filename.\n")); + ret = -1; + return; + } + if (pseudonym != NULL) + { + if (NULL == this_id) + { + FPRINTF (stderr, _("Option `%s' is required when using option `%s'.\n"), + "-t", "-P"); + ret = -1; + return; + } + } + else + { /* ordinary insertion checks */ + if (NULL != next_id) + { + FPRINTF (stderr, _("Option `%s' makes no sense without option `%s'.\n"), + "-N", "-P"); + ret = -1; + return; + } + if (NULL != this_id) + { + FPRINTF (stderr, _("Option `%s' makes no sense without option `%s'.\n"), + "-t", "-P"); + ret = -1; + return; + } + } + cfg = c; + ctx = + GNUNET_FS_start (cfg, "gnunet-publish", &progress_cb, NULL, + GNUNET_FS_FLAGS_NONE, GNUNET_FS_OPTIONS_END); + if (NULL == ctx) + { + FPRINTF (stderr, _("Could not initialize `%s' subsystem.\n"), "FS"); + ret = 1; + return; + } + namespace = NULL; + if (NULL != pseudonym) + { + namespace = GNUNET_FS_namespace_create (ctx, pseudonym); + if (NULL == namespace) + { + FPRINTF (stderr, _("Could not create namespace `%s'\n"), pseudonym); + GNUNET_FS_stop (ctx); + ret = 1; + return; + } + } + if (NULL != uri_string) + { + emsg = NULL; + uri = GNUNET_FS_uri_parse (uri_string, &emsg); + if (uri == NULL) + { + FPRINTF (stderr, _("Failed to parse URI: %s\n"), emsg); + GNUNET_free (emsg); + if (namespace != NULL) + GNUNET_FS_namespace_delete (namespace, GNUNET_NO); + GNUNET_FS_stop (ctx); + ret = 1; + return; + } + GNUNET_FS_publish_ksk (ctx, topKeywords, meta, uri, &bo, + GNUNET_FS_PUBLISH_OPTION_NONE, &uri_ksk_continuation, + NULL); + if (namespace != NULL) + GNUNET_FS_namespace_delete (namespace, GNUNET_NO); + return; + } + if (GNUNET_OK != + GNUNET_CONFIGURATION_get_value_string (cfg, "FS", "EXTRACTORS", &ex)) + ex = NULL; + if (0 != ACCESS (args[0], R_OK)) + { + FPRINTF (stderr, + _("Failed to access `%s': %s\n"), + args[0], + STRERROR (errno)); + return; + } + ds = GNUNET_FS_directory_scan_start (args[0], + disable_extractor, + ex, + &directory_scan_cb, NULL); + if (NULL == ds) + { + FPRINTF (stderr, + "%s", _("Failed to start meta directory scanner. Is gnunet-helper-publish-fs installed?\n")); + return; + } + kill_task = + GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_FOREVER_REL, &do_stop_task, + NULL); +} + + +/** + * The main function to publish content to GNUnet. + * + * @param argc number of arguments from the command line + * @param argv command line arguments + * @return 0 ok, 1 on error + */ +int +main (int argc, char *const *argv) +{ + static const struct GNUNET_GETOPT_CommandLineOption options[] = { + {'a', "anonymity", "LEVEL", + gettext_noop ("set the desired LEVEL of sender-anonymity"), + 1, &GNUNET_GETOPT_set_uint, &bo.anonymity_level}, + {'d', "disable-creation-time", NULL, + gettext_noop + ("disable adding the creation time to the metadata of the uploaded file"), + 0, &GNUNET_GETOPT_set_one, &do_disable_creation_time}, + {'D', "disable-extractor", NULL, + gettext_noop ("do not use libextractor to add keywords or metadata"), + 0, &GNUNET_GETOPT_set_one, &disable_extractor}, + {'e', "extract", NULL, + gettext_noop + ("print list of extracted keywords that would be used, but do not perform upload"), + 0, &GNUNET_GETOPT_set_one, &extract_only}, + {'k', "key", "KEYWORD", + gettext_noop + ("add an additional keyword for the top-level file or directory" + " (this option can be specified multiple times)"), + 1, &GNUNET_FS_getopt_set_keywords, &topKeywords}, + {'m', "meta", "TYPE:VALUE", + gettext_noop ("set the meta-data for the given TYPE to the given VALUE"), + 1, &GNUNET_FS_getopt_set_metadata, &meta}, + {'n', "noindex", NULL, + gettext_noop ("do not index, perform full insertion (stores entire " + "file in encrypted form in GNUnet database)"), + 0, &GNUNET_GETOPT_set_one, &do_insert}, + {'N', "next", "ID", + gettext_noop + ("specify ID of an updated version to be published in the future" + " (for namespace insertions only)"), + 1, &GNUNET_GETOPT_set_string, &next_id}, + {'p', "priority", "PRIORITY", + gettext_noop ("specify the priority of the content"), + 1, &GNUNET_GETOPT_set_uint, &bo.content_priority}, + {'P', "pseudonym", "NAME", + gettext_noop + ("publish the files under the pseudonym NAME (place file into namespace)"), + 1, &GNUNET_GETOPT_set_string, &pseudonym}, + {'r', "replication", "LEVEL", + gettext_noop ("set the desired replication LEVEL"), + 1, &GNUNET_GETOPT_set_uint, &bo.replication_level}, + {'s', "simulate-only", NULL, + gettext_noop ("only simulate the process but do not do any " + "actual publishing (useful to compute URIs)"), + 0, &GNUNET_GETOPT_set_one, &do_simulate}, + {'t', "this", "ID", + gettext_noop ("set the ID of this version of the publication" + " (for namespace insertions only)"), + 1, &GNUNET_GETOPT_set_string, &this_id}, + {'u', "uri", "URI", + gettext_noop ("URI to be published (can be used instead of passing a " + "file to add keywords to the file with the respective URI)"), + 1, &GNUNET_GETOPT_set_string, &uri_string}, + {'V', "verbose", NULL, + gettext_noop ("be verbose (print progress information)"), + 0, &GNUNET_GETOPT_set_one, &verbose}, + GNUNET_GETOPT_OPTION_END + }; + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "GNUnet publish starts\n"); + bo.expiration_time = + GNUNET_FS_year_to_time (GNUNET_FS_get_current_year () + 2); + return (GNUNET_OK == + GNUNET_PROGRAM_run (argc, argv, "gnunet-publish [OPTIONS] FILENAME", + gettext_noop + ("Publish a file or directory on GNUnet"), + options, &run, NULL)) ? ret : 1; +} + +/* end of gnunet-publish.c */ diff --git a/src/fs/gnunet-search.c b/src/fs/gnunet-search.c new file mode 100644 index 0000000..60620a4 --- /dev/null +++ b/src/fs/gnunet-search.c @@ -0,0 +1,312 @@ +/* + This file is part of GNUnet. + (C) 2001, 2002, 2004, 2005, 2006, 2007, 2009 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ +/** + * @file fs/gnunet-search.c + * @brief searching for files on GNUnet + * @author Christian Grothoff + * @author Krista Bennett + * @author James Blackwell + * @author Igor Wronsky + */ +#include "platform.h" +#include "gnunet_fs_service.h" + +static int ret; + +static const struct GNUNET_CONFIGURATION_Handle *cfg; + +static struct GNUNET_FS_Handle *ctx; + +static struct GNUNET_FS_SearchContext *sc; + +static char *output_filename; + +static struct GNUNET_FS_DirectoryBuilder *db; + +static unsigned int anonymity = 1; + +static unsigned long long timeout; + +static unsigned int results_limit; + +static unsigned int results = 0; + +static int verbose; + +static int local_only; + +/** + * Type of a function that libextractor calls for each + * meta data item found. + * + * @param cls closure (user-defined, unused) + * @param plugin_name name of the plugin that produced this value; + * special values can be used (i.e. '<zlib>' for zlib being + * used in the main libextractor library and yielding + * meta data). + * @param type libextractor-type describing the meta data + * @param format basic format information about data + * @param data_mime_type mime-type of data (not of the original file); + * can be NULL (if mime-type is not known) + * @param data actual meta-data found + * @param data_size number of bytes in data + * @return 0 to continue extracting, 1 to abort + */ +static int +item_printer (void *cls, const char *plugin_name, enum EXTRACTOR_MetaType type, + enum EXTRACTOR_MetaFormat format, const char *data_mime_type, + const char *data, size_t data_size) +{ + if ((format != EXTRACTOR_METAFORMAT_UTF8) && + (format != EXTRACTOR_METAFORMAT_C_STRING)) + return 0; + if (type == EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME) + return 0; + printf ("\t%20s: %s\n", + dgettext (LIBEXTRACTOR_GETTEXT_DOMAIN, + EXTRACTOR_metatype_to_string (type)), data); + return 0; +} + + +static void +clean_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + size_t dsize; + void *ddata; + + GNUNET_FS_stop (ctx); + ctx = NULL; + if (output_filename == NULL) + return; + if (GNUNET_OK != GNUNET_FS_directory_builder_finish (db, &dsize, &ddata)) + { + GNUNET_break (0); + GNUNET_free (output_filename); + return; + } + if (dsize != + GNUNET_DISK_fn_write (output_filename, ddata, dsize, + GNUNET_DISK_PERM_USER_READ | + GNUNET_DISK_PERM_USER_WRITE)) + { + FPRINTF (stderr, + _("Failed to write directory with search results to `%s'\n"), + output_filename); + } + GNUNET_free_non_null (ddata); + GNUNET_free (output_filename); +} + + +/** + * Called by FS client to give information about the progress of an + * operation. + * + * @param cls closure + * @param info details about the event, specifying the event type + * and various bits about the event + * @return client-context (for the next progress call + * for this operation; should be set to NULL for + * SUSPEND and STOPPED events). The value returned + * will be passed to future callbacks in the respective + * field in the GNUNET_FS_ProgressInfo struct. + */ +static void * +progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *info) +{ + static unsigned int cnt; + char *uri; + char *dotdot; + char *filename; + + switch (info->status) + { + case GNUNET_FS_STATUS_SEARCH_START: + break; + case GNUNET_FS_STATUS_SEARCH_RESULT: + if (db != NULL) + GNUNET_FS_directory_builder_add (db, + info->value.search.specifics.result.uri, + info->value.search.specifics.result.meta, + NULL); + uri = GNUNET_FS_uri_to_string (info->value.search.specifics.result.uri); + printf ("#%u:\n", cnt++); + filename = + GNUNET_CONTAINER_meta_data_get_by_type (info->value.search. + specifics.result.meta, + EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME); + if (filename != NULL) + { + while (NULL != (dotdot = strstr (filename, ".."))) + dotdot[0] = dotdot[1] = '_'; + printf ("gnunet-download -o \"%s\" %s\n", filename, uri); + } + else + printf ("gnunet-download %s\n", uri); + if (verbose) + GNUNET_CONTAINER_meta_data_iterate (info->value.search.specifics. + result.meta, &item_printer, NULL); + printf ("\n"); + fflush (stdout); + GNUNET_free_non_null (filename); + GNUNET_free (uri); + results++; + if ((results_limit > 0) && (results >= results_limit)) + GNUNET_SCHEDULER_shutdown (); + break; + case GNUNET_FS_STATUS_SEARCH_UPDATE: + break; + case GNUNET_FS_STATUS_SEARCH_RESULT_STOPPED: + /* ignore */ + break; + case GNUNET_FS_STATUS_SEARCH_ERROR: + FPRINTF (stderr, _("Error searching: %s.\n"), + info->value.search.specifics.error.message); + GNUNET_SCHEDULER_shutdown (); + break; + case GNUNET_FS_STATUS_SEARCH_STOPPED: + GNUNET_SCHEDULER_add_continuation (&clean_task, NULL, + GNUNET_SCHEDULER_REASON_PREREQ_DONE); + break; + default: + FPRINTF (stderr, _("Unexpected status: %d\n"), info->status); + break; + } + return NULL; +} + + +static void +shutdown_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + if (sc != NULL) + { + GNUNET_FS_search_stop (sc); + sc = NULL; + } +} + + +/** + * Main function that will be run by the scheduler. + * + * @param cls closure + * @param args remaining command-line arguments + * @param cfgfile name of the configuration file used (for saving, can be NULL!) + * @param c configuration + */ +static void +run (void *cls, char *const *args, const char *cfgfile, + const struct GNUNET_CONFIGURATION_Handle *c) +{ + struct GNUNET_FS_Uri *uri; + unsigned int argc; + enum GNUNET_FS_SearchOptions options; + struct GNUNET_TIME_Relative delay; + + argc = 0; + while (NULL != args[argc]) + argc++; + uri = GNUNET_FS_uri_ksk_create_from_args (argc, (const char **) args); + if (NULL == uri) + { + FPRINTF (stderr, "%s", _("Could not create keyword URI from arguments.\n")); + ret = 1; + return; + } + cfg = c; + ctx = + GNUNET_FS_start (cfg, "gnunet-search", &progress_cb, NULL, + GNUNET_FS_FLAGS_NONE, GNUNET_FS_OPTIONS_END); + if (NULL == ctx) + { + FPRINTF (stderr, _("Could not initialize `%s' subsystem.\n"), "FS"); + GNUNET_FS_uri_destroy (uri); + ret = 1; + return; + } + if (output_filename != NULL) + db = GNUNET_FS_directory_builder_create (NULL); + options = GNUNET_FS_SEARCH_OPTION_NONE; + if (local_only) + options |= GNUNET_FS_SEARCH_OPTION_LOOPBACK_ONLY; + sc = GNUNET_FS_search_start (ctx, uri, anonymity, options, NULL); + GNUNET_FS_uri_destroy (uri); + if (NULL == sc) + { + FPRINTF (stderr, "%s", _("Could not start searching.\n")); + GNUNET_FS_stop (ctx); + ret = 1; + return; + } + if (timeout != 0) + { + delay.rel_value = timeout; + GNUNET_SCHEDULER_add_delayed (delay, &shutdown_task, NULL); + } + else + { + GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_FOREVER_REL, &shutdown_task, + NULL); + } +} + + +/** + * The main function to search GNUnet. + * + * @param argc number of arguments from the command line + * @param argv command line arguments + * @return 0 ok, 1 on error + */ +int +main (int argc, char *const *argv) +{ + static const struct GNUNET_GETOPT_CommandLineOption options[] = { + {'a', "anonymity", "LEVEL", + gettext_noop ("set the desired LEVEL of receiver-anonymity"), + 1, &GNUNET_GETOPT_set_uint, &anonymity}, + {'n', "no-network", NULL, + gettext_noop ("only search the local peer (no P2P network search)"), + 0, &GNUNET_GETOPT_set_one, &local_only}, + {'o', "output", "PREFIX", + gettext_noop ("write search results to file starting with PREFIX"), + 1, &GNUNET_GETOPT_set_string, &output_filename}, + {'t', "timeout", "VALUE", + gettext_noop ("automatically terminate search after VALUE ms"), + 1, &GNUNET_GETOPT_set_ulong, &timeout}, + {'V', "verbose", NULL, + gettext_noop ("be verbose (print progress information)"), + 0, &GNUNET_GETOPT_set_one, &verbose}, + {'N', "results", "VALUE", + gettext_noop + ("automatically terminate search after VALUE results are found"), + 1, &GNUNET_GETOPT_set_uint, &results_limit}, + GNUNET_GETOPT_OPTION_END + }; + return (GNUNET_OK == + GNUNET_PROGRAM_run (argc, argv, "gnunet-search [OPTIONS] KEYWORD", + gettext_noop + ("Search GNUnet for files that were published on GNUnet"), + options, &run, NULL)) ? ret : 1; +} + +/* end of gnunet-search.c */ diff --git a/src/fs/gnunet-service-fs.c b/src/fs/gnunet-service-fs.c new file mode 100644 index 0000000..06ac91c --- /dev/null +++ b/src/fs/gnunet-service-fs.c @@ -0,0 +1,654 @@ +/* + This file is part of GNUnet. + (C) 2009, 2010, 2011 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/gnunet-service-fs.c + * @brief gnunet anonymity protocol implementation + * @author Christian Grothoff + * + * To use: + * - consider re-issue GSF_dht_lookup_ after non-DHT reply received + */ +#include "platform.h" +#include <float.h> +#include "gnunet_constants.h" +#include "gnunet_core_service.h" +#include "gnunet_dht_service.h" +#include "gnunet_datastore_service.h" +#include "gnunet_load_lib.h" +#include "gnunet_peer_lib.h" +#include "gnunet_protocols.h" +#include "gnunet_signatures.h" +#include "gnunet_statistics_service.h" +#include "gnunet_transport_service.h" +#include "gnunet_util_lib.h" +#include "gnunet-service-fs_cp.h" +#include "gnunet-service-fs_indexing.h" +#include "gnunet-service-fs_lc.h" +#include "gnunet-service-fs_pe.h" +#include "gnunet-service-fs_pr.h" +#include "gnunet-service-fs_push.h" +#include "gnunet-service-fs_put.h" +#include "fs.h" + +/** + * Size for the hash map for DHT requests from the FS + * service. Should be about the number of concurrent + * DHT requests we plan to make. + */ +#define FS_DHT_HT_SIZE 1024 + + +/** + * How quickly do we age cover traffic? At the given + * time interval, remaining cover traffic counters are + * decremented by 1/16th. + */ +#define COVER_AGE_FREQUENCY GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 5) + + +/* ****************************** globals ****************************** */ + +/** + * Our connection to the datastore. + */ +struct GNUNET_DATASTORE_Handle *GSF_dsh; + +/** + * Our configuration. + */ +const struct GNUNET_CONFIGURATION_Handle *GSF_cfg; + +/** + * Handle for reporting statistics. + */ +struct GNUNET_STATISTICS_Handle *GSF_stats; + +/** + * Handle for DHT operations. + */ +struct GNUNET_DHT_Handle *GSF_dht; + +/** + * How long do requests typically stay in the routing table? + */ +struct GNUNET_LOAD_Value *GSF_rt_entry_lifetime; + +/** + * Running average of the observed latency to other peers (round trip). + * Initialized to 5s as the initial default. + */ +struct GNUNET_TIME_Relative GSF_avg_latency = { 500 }; + +/** + * Typical priorities we're seeing from other peers right now. Since + * most priorities will be zero, this value is the weighted average of + * non-zero priorities seen "recently". In order to ensure that new + * values do not dramatically change the ratio, values are first + * "capped" to a reasonable range (+N of the current value) and then + * averaged into the existing value by a ratio of 1:N. Hence + * receiving the largest possible priority can still only raise our + * "current_priorities" by at most 1. + */ +double GSF_current_priorities; + +/** + * How many query messages have we received 'recently' that + * have not yet been claimed as cover traffic? + */ +unsigned int GSF_cover_query_count; + +/** + * How many content messages have we received 'recently' that + * have not yet been claimed as cover traffic? + */ +unsigned int GSF_cover_content_count; + +/** + * Our block context. + */ +struct GNUNET_BLOCK_Context *GSF_block_ctx; + +/** + * Pointer to handle to the core service (points to NULL until we've + * connected to it). + */ +struct GNUNET_CORE_Handle *GSF_core; + +/** + * Are we introducing randomized delays for better anonymity? + */ +int GSF_enable_randomized_delays; + +/* ***************************** locals ******************************* */ + +/** + * Configuration for block library. + */ +static struct GNUNET_CONFIGURATION_Handle *block_cfg; + +/** + * ID of our task that we use to age the cover counters. + */ +static GNUNET_SCHEDULER_TaskIdentifier cover_age_task; + +/** + * Datastore 'GET' load tracking. + */ +static struct GNUNET_LOAD_Value *datastore_get_load; + +/** + * Identity of this peer. + */ +static struct GNUNET_PeerIdentity my_id; + +/** + * Task that periodically ages our cover traffic statistics. + * + * @param cls unused closure + * @param tc task context + */ +static void +age_cover_counters (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + GSF_cover_content_count = (GSF_cover_content_count * 15) / 16; + GSF_cover_query_count = (GSF_cover_query_count * 15) / 16; + cover_age_task = + GNUNET_SCHEDULER_add_delayed (COVER_AGE_FREQUENCY, &age_cover_counters, + NULL); +} + + +/** + * We've just now completed a datastore request. Update our + * datastore load calculations. + * + * @param start time when the datastore request was issued + */ +void +GSF_update_datastore_delay_ (struct GNUNET_TIME_Absolute start) +{ + struct GNUNET_TIME_Relative delay; + + delay = GNUNET_TIME_absolute_get_duration (start); + GNUNET_LOAD_update (datastore_get_load, delay.rel_value); +} + + +/** + * Test if the DATABASE (GET) load on this peer is too high + * to even consider processing the query at + * all. + * + * @return GNUNET_YES if the load is too high to do anything (load high) + * GNUNET_NO to process normally (load normal) + * GNUNET_SYSERR to process for free (load low) + */ +int +GSF_test_get_load_too_high_ (uint32_t priority) +{ + double ld; + + ld = GNUNET_LOAD_get_load (datastore_get_load); + if (ld < 1) + return GNUNET_SYSERR; + if (ld <= priority) + return GNUNET_NO; + return GNUNET_YES; +} + + +/** + * We've received peer performance information. Update + * our running average for the P2P latency. + * + * @param atsi performance information + * @param atsi_count number of 'atsi' records + */ +static void +update_latencies (const struct GNUNET_ATS_Information *atsi, + unsigned int atsi_count) +{ + unsigned int i; + + for (i = 0; i < atsi_count; i++) + { + if (ntohl (atsi[i].type) == GNUNET_ATS_QUALITY_NET_DELAY) + { + GSF_avg_latency.rel_value = + (GSF_avg_latency.rel_value * 31 + + GNUNET_MIN (5000, ntohl (atsi[i].value))) / 32; + GNUNET_STATISTICS_set (GSF_stats, + gettext_noop + ("# running average P2P latency (ms)"), + GSF_avg_latency.rel_value, GNUNET_NO); + break; + } + } +} + + +/** + * Handle P2P "PUT" message. + * + * @param cls closure, always NULL + * @param other the other peer involved (sender or receiver, NULL + * for loopback messages where we are both sender and receiver) + * @param message the actual message + * @param atsi performance information + * @param atsi_count number of records in 'atsi' + * @return GNUNET_OK to keep the connection open, + * GNUNET_SYSERR to close it (signal serious error) + */ +static int +handle_p2p_put (void *cls, const struct GNUNET_PeerIdentity *other, + const struct GNUNET_MessageHeader *message, + const struct GNUNET_ATS_Information *atsi, + unsigned int atsi_count) +{ + struct GSF_ConnectedPeer *cp; + + cp = GSF_peer_get_ (other); + if (NULL == cp) + { + GNUNET_break (0); + return GNUNET_OK; + } + GSF_cover_content_count++; + update_latencies (atsi, atsi_count); + return GSF_handle_p2p_content_ (cp, message); +} + + +/** + * We have a new request, consider forwarding it to the given + * peer. + * + * @param cls the 'struct GSF_PendingRequest' + * @param peer identity of the peer + * @param cp handle to the connected peer record + * @param ppd peer performance data + */ +static void +consider_request_for_forwarding (void *cls, + const struct GNUNET_PeerIdentity *peer, + struct GSF_ConnectedPeer *cp, + const struct GSF_PeerPerformanceData *ppd) +{ + struct GSF_PendingRequest *pr = cls; + + if (GNUNET_YES != GSF_pending_request_test_target_ (pr, peer)) + { + GNUNET_STATISTICS_update (GSF_stats, + gettext_noop ("# Loopback routes suppressed"), 1, + GNUNET_NO); + return; + } + GSF_plan_add_ (cp, pr); +} + + +/** + * Function to be called after we're done processing + * replies from the local lookup. If the result status + * code indicates that there may be more replies, plan + * forwarding the request. + * + * @param cls closure (NULL) + * @param pr the pending request we were processing + * @param result final datastore lookup result + */ +static void +consider_forwarding (void *cls, struct GSF_PendingRequest *pr, + enum GNUNET_BLOCK_EvaluationResult result) +{ + if (GNUNET_BLOCK_EVALUATION_OK_LAST == result) + return; /* we're done... */ + GSF_iterate_connected_peers_ (&consider_request_for_forwarding, pr); +} + + +/** + * Handle P2P "GET" request. + * + * @param cls closure, always NULL + * @param other the other peer involved (sender or receiver, NULL + * for loopback messages where we are both sender and receiver) + * @param message the actual message + * @param atsi performance information + * @param atsi_count number of records in 'atsi' + * @return GNUNET_OK to keep the connection open, + * GNUNET_SYSERR to close it (signal serious error) + */ +static int +handle_p2p_get (void *cls, const struct GNUNET_PeerIdentity *other, + const struct GNUNET_MessageHeader *message, + const struct GNUNET_ATS_Information *atsi, + unsigned int atsi_count) +{ + struct GSF_PendingRequest *pr; + + pr = GSF_handle_p2p_query_ (other, message); + if (NULL == pr) + return GNUNET_SYSERR; + GSF_pending_request_get_data_ (pr)->has_started = GNUNET_YES; + GSF_local_lookup_ (pr, &consider_forwarding, NULL); + update_latencies (atsi, atsi_count); + return GNUNET_OK; +} + + +/** + * We're done with the local lookup, now consider + * P2P processing (depending on request options and + * result status). Also signal that we can now + * receive more request information from the client. + * + * @param cls the client doing the request ('struct GNUNET_SERVER_Client') + * @param pr the pending request we were processing + * @param result final datastore lookup result + */ +static void +start_p2p_processing (void *cls, struct GSF_PendingRequest *pr, + enum GNUNET_BLOCK_EvaluationResult result) +{ + struct GNUNET_SERVER_Client *client = cls; + struct GSF_PendingRequestData *prd; + + prd = GSF_pending_request_get_data_ (pr); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Finished database lookup for local request `%s' with result %d\n", + GNUNET_h2s (&prd->query), result); + GNUNET_SERVER_receive_done (client, GNUNET_OK); + if (GNUNET_BLOCK_EVALUATION_OK_LAST == result) + return; /* we're done, 'pr' was already destroyed... */ + if (0 != (GSF_PRO_LOCAL_ONLY & prd->options)) + { + GSF_pending_request_cancel_ (pr, GNUNET_YES); + return; + } + GSF_dht_lookup_ (pr); + consider_forwarding (NULL, pr, result); +} + + +/** + * Handle START_SEARCH-message (search request from client). + * + * @param cls closure + * @param client identification of the client + * @param message the actual message + */ +static void +handle_start_search (void *cls, struct GNUNET_SERVER_Client *client, + const struct GNUNET_MessageHeader *message) +{ + struct GSF_PendingRequest *pr; + int ret; + + pr = NULL; + ret = GSF_local_client_start_search_handler_ (client, message, &pr); + switch (ret) + { + case GNUNET_SYSERR: + GNUNET_SERVER_receive_done (client, GNUNET_SYSERR); + break; + case GNUNET_NO: + GNUNET_SERVER_receive_done (client, GNUNET_OK); + break; + case GNUNET_YES: + GSF_pending_request_get_data_ (pr)->has_started = GNUNET_YES; + GSF_local_lookup_ (pr, &start_p2p_processing, client); + break; + default: + GNUNET_assert (0); + } +} + + +/** + * Task run during shutdown. + * + * @param cls unused + * @param tc unused + */ +static void +shutdown_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + if (NULL != GSF_core) + { + GNUNET_CORE_disconnect (GSF_core); + GSF_core = NULL; + } + GSF_put_done_ (); + GSF_push_done_ (); + GSF_pending_request_done_ (); + GSF_plan_done (); + GSF_connected_peer_done_ (); + GNUNET_DATASTORE_disconnect (GSF_dsh, GNUNET_NO); + GSF_dsh = NULL; + GNUNET_DHT_disconnect (GSF_dht); + GSF_dht = NULL; + GNUNET_BLOCK_context_destroy (GSF_block_ctx); + GSF_block_ctx = NULL; + GNUNET_CONFIGURATION_destroy (block_cfg); + block_cfg = NULL; + GNUNET_STATISTICS_destroy (GSF_stats, GNUNET_NO); + GSF_stats = NULL; + if (GNUNET_SCHEDULER_NO_TASK != cover_age_task) + { + GNUNET_SCHEDULER_cancel (cover_age_task); + cover_age_task = GNUNET_SCHEDULER_NO_TASK; + } + GNUNET_FS_indexing_done (); + GNUNET_LOAD_value_free (datastore_get_load); + datastore_get_load = NULL; + GNUNET_LOAD_value_free (GSF_rt_entry_lifetime); + GSF_rt_entry_lifetime = NULL; +} + + +/** + * Function called for each pending request whenever a new + * peer connects, giving us a chance to decide about submitting + * the existing request to the new peer. + * + * @param cls the 'struct GSF_ConnectedPeer' of the new peer + * @param key query for the request + * @param pr handle to the pending request + * @return GNUNET_YES to continue to iterate + */ +static int +consider_peer_for_forwarding (void *cls, const GNUNET_HashCode * key, + struct GSF_PendingRequest *pr) +{ + struct GSF_ConnectedPeer *cp = cls; + struct GNUNET_PeerIdentity pid; + + GSF_connected_peer_get_identity_ (cp, &pid); + if (GNUNET_YES != GSF_pending_request_test_target_ (pr, &pid)) + { + GNUNET_STATISTICS_update (GSF_stats, + gettext_noop ("# Loopback routes suppressed"), 1, + GNUNET_NO); + return GNUNET_YES; + } + GSF_plan_add_ (cp, pr); + return GNUNET_YES; +} + + +/** + * Method called whenever a given peer connects. + * + * @param cls closure, not used + * @param peer peer identity this notification is about + * @param atsi performance information + * @param atsi_count number of records in 'atsi' + */ +static void +peer_connect_handler (void *cls, const struct GNUNET_PeerIdentity *peer, + const struct GNUNET_ATS_Information *atsi, + unsigned int atsi_count) +{ + struct GSF_ConnectedPeer *cp; + + if (0 == memcmp (&my_id, peer, sizeof (struct GNUNET_PeerIdentity))) + return; + cp = GSF_peer_connect_handler_ (peer, atsi, atsi_count); + if (NULL == cp) + return; + GSF_iterate_pending_requests_ (&consider_peer_for_forwarding, cp); +} + + +/** + * Function called after GNUNET_CORE_connect has succeeded + * (or failed for good). Note that the private key of the + * peer is intentionally not exposed here; if you need it, + * your process should try to read the private key file + * directly (which should work if you are authorized...). + * + * @param cls closure + * @param server handle to the server, NULL if we failed + * @param my_identity ID of this peer, NULL if we failed + */ +static void +peer_init_handler (void *cls, struct GNUNET_CORE_Handle *server, + const struct GNUNET_PeerIdentity *my_identity) +{ + my_id = *my_identity; +} + + +/** + * Process fs requests. + * + * @param server the initialized server + * @param c configuration to use + */ +static int +main_init (struct GNUNET_SERVER_Handle *server, + const struct GNUNET_CONFIGURATION_Handle *c) +{ + static const struct GNUNET_CORE_MessageHandler p2p_handlers[] = { + {&handle_p2p_get, + GNUNET_MESSAGE_TYPE_FS_GET, 0}, + {&handle_p2p_put, + GNUNET_MESSAGE_TYPE_FS_PUT, 0}, + {&GSF_handle_p2p_migration_stop_, + GNUNET_MESSAGE_TYPE_FS_MIGRATION_STOP, + sizeof (struct MigrationStopMessage)}, + {NULL, 0, 0} + }; + static const struct GNUNET_SERVER_MessageHandler handlers[] = { + {&GNUNET_FS_handle_index_start, NULL, + GNUNET_MESSAGE_TYPE_FS_INDEX_START, 0}, + {&GNUNET_FS_handle_index_list_get, NULL, + GNUNET_MESSAGE_TYPE_FS_INDEX_LIST_GET, + sizeof (struct GNUNET_MessageHeader)}, + {&GNUNET_FS_handle_unindex, NULL, GNUNET_MESSAGE_TYPE_FS_UNINDEX, + sizeof (struct UnindexMessage)}, + {&handle_start_search, NULL, GNUNET_MESSAGE_TYPE_FS_START_SEARCH, + 0}, + {NULL, NULL, 0, 0} + }; + + GSF_core = + GNUNET_CORE_connect (GSF_cfg, 1, NULL, &peer_init_handler, + &peer_connect_handler, &GSF_peer_disconnect_handler_, + NULL, GNUNET_NO, NULL, GNUNET_NO, p2p_handlers); + if (NULL == GSF_core) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + _("Failed to connect to `%s' service.\n"), "core"); + return GNUNET_SYSERR; + } + GNUNET_SERVER_disconnect_notify (server, &GSF_client_disconnect_handler_, + NULL); + GNUNET_SERVER_add_handlers (server, handlers); + cover_age_task = + GNUNET_SCHEDULER_add_delayed (COVER_AGE_FREQUENCY, &age_cover_counters, + NULL); + datastore_get_load = GNUNET_LOAD_value_init (DATASTORE_LOAD_AUTODECLINE); + GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_FOREVER_REL, &shutdown_task, + NULL); + return GNUNET_OK; +} + + +/** + * Process fs requests. + * + * @param cls closure + * @param server the initialized server + * @param cfg configuration to use + */ +static void +run (void *cls, struct GNUNET_SERVER_Handle *server, + const struct GNUNET_CONFIGURATION_Handle *cfg) +{ + GSF_cfg = cfg; + GSF_enable_randomized_delays = + GNUNET_CONFIGURATION_get_value_yesno (cfg, "fs", "DELAY"); + GSF_dsh = GNUNET_DATASTORE_connect (cfg); + if (NULL == GSF_dsh) + { + GNUNET_SCHEDULER_shutdown (); + return; + } + GSF_rt_entry_lifetime = GNUNET_LOAD_value_init (GNUNET_TIME_UNIT_FOREVER_REL); + GSF_stats = GNUNET_STATISTICS_create ("fs", cfg); + block_cfg = GNUNET_CONFIGURATION_create (); + GNUNET_CONFIGURATION_set_value_string (block_cfg, "block", "PLUGINS", "fs"); + GSF_block_ctx = GNUNET_BLOCK_context_create (block_cfg); + GNUNET_assert (NULL != GSF_block_ctx); + GSF_dht = GNUNET_DHT_connect (cfg, FS_DHT_HT_SIZE); + GSF_plan_init (); + GSF_pending_request_init_ (); + GSF_connected_peer_init_ (); + GSF_push_init_ (); + GSF_put_init_ (); + if ((GNUNET_OK != GNUNET_FS_indexing_init (cfg, GSF_dsh)) || + (GNUNET_OK != main_init (server, cfg))) + { + GNUNET_SCHEDULER_shutdown (); + shutdown_task (NULL, NULL); + return; + } +} + + +/** + * The main function for the fs service. + * + * @param argc number of arguments from the command line + * @param argv command line arguments + * @return 0 ok, 1 on error + */ +int +main (int argc, char *const *argv) +{ + return (GNUNET_OK == + GNUNET_SERVICE_run (argc, argv, "fs", GNUNET_SERVICE_OPTION_NONE, + &run, NULL)) ? 0 : 1; +} + +/* end of gnunet-service-fs.c */ diff --git a/src/fs/gnunet-service-fs.h b/src/fs/gnunet-service-fs.h new file mode 100644 index 0000000..5ea73ee --- /dev/null +++ b/src/fs/gnunet-service-fs.h @@ -0,0 +1,286 @@ +/* + This file is part of GNUnet. + (C) 2009, 2010 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/gnunet-service-fs.h + * @brief shared data structures of gnunet-service-fs.c + * @author Christian Grothoff + */ +#ifndef GNUNET_SERVICE_FS_H +#define GNUNET_SERVICE_FS_H + +#include "gnunet_util_lib.h" +#include "gnunet_statistics_service.h" +#include "gnunet_transport_service.h" +#include "gnunet_core_service.h" +#include "gnunet_block_lib.h" +#include "fs.h" + + +/** + * By which amount do we decrement the TTL for simple forwarding / + * indirection of the query; in milli-seconds. Set somewhat in + * accordance to your network latency (above the time it'll take you + * to send a packet and get a reply). + */ +#define TTL_DECREMENT 5000 + +/** + * At what frequency should our datastore load decrease + * automatically (since if we don't use it, clearly the + * load must be going down). + */ +#define DATASTORE_LOAD_AUTODECLINE GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MILLISECONDS, 250) + +/** + * Only the (mandatory) query is included. + */ +#define GET_MESSAGE_BIT_QUERY_ONLY 0 + +/** + * The peer identity of a peer waiting for the + * reply is included (used if the response + * should be transmitted to someone other than + * the sender of the GET). + */ +#define GET_MESSAGE_BIT_RETURN_TO 1 + +/** + * The hash of the public key of the target + * namespace is included (for SKS queries). + */ +#define GET_MESSAGE_BIT_SKS_NAMESPACE 2 + +/** + * The peer identity of a peer that had claimed to have the content + * previously is included (can be used if responder-anonymity is not + * desired; note that the precursor presumably lacked a direct + * connection to the specified peer; still, the receiver is in no way + * required to limit forwarding only to the specified peer, it should + * only prefer it somewhat if possible). + */ +#define GET_MESSAGE_BIT_TRANSMIT_TO 4 + + +GNUNET_NETWORK_STRUCT_BEGIN + +/** + * Message sent between peers asking for FS-content. + */ +struct GetMessage +{ + + /** + * Message type will be GNUNET_MESSAGE_TYPE_FS_GET. + */ + struct GNUNET_MessageHeader header; + + /** + * Type of the query (block type). + */ + uint32_t type GNUNET_PACKED; + + /** + * How important is this request (network byte order) + */ + uint32_t priority GNUNET_PACKED; + + /** + * Relative time to live in MILLISECONDS (network byte order) + */ + int32_t ttl GNUNET_PACKED; + + /** + * The content hash should be mutated using this value + * before checking against the bloomfilter (used to + * get many different filters for the same hash codes). + * The number should be in big-endian format when used + * for mingling. + */ + uint32_t filter_mutator GNUNET_PACKED; + + /** + * Which of the optional hash codes are present at the end of the + * message? See GET_MESSAGE_BIT_xx constants. For each bit that is + * set, an additional GNUNET_HashCode with the respective content + * (in order of the bits) will be appended to the end of the GET + * message. + */ + uint32_t hash_bitmap GNUNET_PACKED; + + /** + * Hashcodes of the file(s) we're looking for. + * Details depend on the query type. + */ + GNUNET_HashCode query GNUNET_PACKED; + + /* this is followed by hash codes as specified in the "hash_bitmap"; + * after that, an optional bloomfilter (with bits set for replies + * that should be suppressed) can be present */ +}; + + +/** + * Message send by a peer that wants to be excluded + * from migration for a while. + */ +struct MigrationStopMessage +{ + /** + * Message type will be + * GNUNET_MESSAGE_TYPE_FS_MIGRATION_STOP. + */ + struct GNUNET_MessageHeader header; + + /** + * Always zero. + */ + uint32_t reserved GNUNET_PACKED; + + /** + * How long should the block last? + */ + struct GNUNET_TIME_RelativeNBO duration; + +}; +GNUNET_NETWORK_STRUCT_END + +/** + * A connected peer. + */ +struct GSF_ConnectedPeer; + +/** + * An active request. + */ +struct GSF_PendingRequest; + +/** + * A local client. + */ +struct GSF_LocalClient; + +/** + * Information kept per plan per request ('pe' module). + */ +struct GSF_RequestPlan; + +/** + * DLL of request plans a particular pending request is + * involved with. + */ +struct GSF_RequestPlanReference; + +/** + * Our connection to the datastore. + */ +extern struct GNUNET_DATASTORE_Handle *GSF_dsh; + +/** + * Our configuration. + */ +extern const struct GNUNET_CONFIGURATION_Handle *GSF_cfg; + +/** + * Handle for reporting statistics. + */ +extern struct GNUNET_STATISTICS_Handle *GSF_stats; + +/** + * Pointer to handle to the core service (points to NULL until we've + * connected to it). + */ +extern struct GNUNET_CORE_Handle *GSF_core; + +/** + * Handle for DHT operations. + */ +extern struct GNUNET_DHT_Handle *GSF_dht; + +/** + * How long do requests typically stay in the routing table? + */ +extern struct GNUNET_LOAD_Value *GSF_rt_entry_lifetime; + +/** + * Running average of the observed latency to other peers (round trip). + */ +extern struct GNUNET_TIME_Relative GSF_avg_latency; + +/** + * Typical priorities we're seeing from other peers right now. Since + * most priorities will be zero, this value is the weighted average of + * non-zero priorities seen "recently". In order to ensure that new + * values do not dramatically change the ratio, values are first + * "capped" to a reasonable range (+N of the current value) and then + * averaged into the existing value by a ratio of 1:N. Hence + * receiving the largest possible priority can still only raise our + * "current_priorities" by at most 1. + */ +extern double GSF_current_priorities; + +/** + * How many query messages have we received 'recently' that + * have not yet been claimed as cover traffic? + */ +extern unsigned int GSF_cover_query_count; + +/** + * How many content messages have we received 'recently' that + * have not yet been claimed as cover traffic? + */ +extern unsigned int GSF_cover_content_count; + +/** + * Our block context. + */ +extern struct GNUNET_BLOCK_Context *GSF_block_ctx; + +/** + * Are we introducing randomized delays for better anonymity? + */ +extern int GSF_enable_randomized_delays; + +/** + * Test if the DATABASE (GET) load on this peer is too high + * to even consider processing the query at + * all. + * + * @return GNUNET_YES if the load is too high to do anything (load high) + * GNUNET_NO to process normally (load normal) + * GNUNET_SYSERR to process for free (load low) + */ +int +GSF_test_get_load_too_high_ (uint32_t priority); + + +/** + * We've just now completed a datastore request. Update our + * datastore load calculations. + * + * @param start time when the datastore request was issued + */ +void +GSF_update_datastore_delay_ (struct GNUNET_TIME_Absolute start); + + + +#endif +/* end of gnunet-service-fs.h */ diff --git a/src/fs/gnunet-service-fs_cp.c b/src/fs/gnunet-service-fs_cp.c new file mode 100644 index 0000000..e84993b --- /dev/null +++ b/src/fs/gnunet-service-fs_cp.c @@ -0,0 +1,1898 @@ +/* + This file is part of GNUnet. + (C) 2011 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/gnunet-service-fs_cp.c + * @brief API to handle 'connected peers' + * @author Christian Grothoff + */ +#include "platform.h" +#include "gnunet_load_lib.h" +#include "gnunet_ats_service.h" +#include "gnunet-service-fs.h" +#include "gnunet-service-fs_cp.h" +#include "gnunet-service-fs_pe.h" +#include "gnunet-service-fs_pr.h" +#include "gnunet-service-fs_push.h" + + +/** + * Ratio for moving average delay calculation. The previous + * average goes in with a factor of (n-1) into the calculation. + * Must be > 0. + */ +#define RUNAVG_DELAY_N 16 + +/** + * How often do we flush trust values to disk? + */ +#define TRUST_FLUSH_FREQ GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 5) + +/** + * After how long do we discard a reply? + */ +#define REPLY_TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 2) + + +/** + * Handle to cancel a transmission request. + */ +struct GSF_PeerTransmitHandle +{ + + /** + * Kept in a doubly-linked list. + */ + struct GSF_PeerTransmitHandle *next; + + /** + * Kept in a doubly-linked list. + */ + struct GSF_PeerTransmitHandle *prev; + + /** + * Handle for an active request for transmission to this + * peer, or NULL (if core queue was full). + */ + struct GNUNET_CORE_TransmitHandle *cth; + + /** + * Time when this transmission request was issued. + */ + struct GNUNET_TIME_Absolute transmission_request_start_time; + + /** + * Timeout for this request. + */ + struct GNUNET_TIME_Absolute timeout; + + /** + * Task called on timeout, or 0 for none. + */ + GNUNET_SCHEDULER_TaskIdentifier timeout_task; + + /** + * Function to call to get the actual message. + */ + GSF_GetMessageCallback gmc; + + /** + * Peer this request targets. + */ + struct GSF_ConnectedPeer *cp; + + /** + * Closure for 'gmc'. + */ + void *gmc_cls; + + /** + * Size of the message to be transmitted. + */ + size_t size; + + /** + * Set to 1 if we're currently in the process of calling + * 'GNUNET_CORE_notify_transmit_ready' (so while cth is + * NULL, we should not call notify_transmit_ready for this + * handle right now). + */ + unsigned int cth_in_progress; + + /** + * GNUNET_YES if this is a query, GNUNET_NO for content. + */ + int is_query; + + /** + * Did we get a reservation already? + */ + int was_reserved; + + /** + * Priority of this request. + */ + uint32_t priority; + +}; + + +/** + * Handle for an entry in our delay list. + */ +struct GSF_DelayedHandle +{ + + /** + * Kept in a doubly-linked list. + */ + struct GSF_DelayedHandle *next; + + /** + * Kept in a doubly-linked list. + */ + struct GSF_DelayedHandle *prev; + + /** + * Peer this transmission belongs to. + */ + struct GSF_ConnectedPeer *cp; + + /** + * The PUT that was delayed. + */ + struct PutMessage *pm; + + /** + * Task for the delay. + */ + GNUNET_SCHEDULER_TaskIdentifier delay_task; + + /** + * Size of the message. + */ + size_t msize; + +}; + + +/** + * Information per peer and request. + */ +struct PeerRequest +{ + + /** + * Handle to generic request. + */ + struct GSF_PendingRequest *pr; + + /** + * Handle to specific peer. + */ + struct GSF_ConnectedPeer *cp; + + /** + * Task for asynchronous stopping of this request. + */ + GNUNET_SCHEDULER_TaskIdentifier kill_task; + +}; + + +/** + * A connected peer. + */ +struct GSF_ConnectedPeer +{ + + /** + * Performance data for this peer. + */ + struct GSF_PeerPerformanceData ppd; + + /** + * Time until when we blocked this peer from migrating + * data to us. + */ + struct GNUNET_TIME_Absolute last_migration_block; + + /** + * Task scheduled to revive migration to this peer. + */ + GNUNET_SCHEDULER_TaskIdentifier mig_revive_task; + + /** + * Messages (replies, queries, content migration) we would like to + * send to this peer in the near future. Sorted by priority, head. + */ + struct GSF_PeerTransmitHandle *pth_head; + + /** + * Messages (replies, queries, content migration) we would like to + * send to this peer in the near future. Sorted by priority, tail. + */ + struct GSF_PeerTransmitHandle *pth_tail; + + /** + * Messages (replies, queries, content migration) we would like to + * send to this peer in the near future. Sorted by priority, head. + */ + struct GSF_DelayedHandle *delayed_head; + + /** + * Messages (replies, queries, content migration) we would like to + * send to this peer in the near future. Sorted by priority, tail. + */ + struct GSF_DelayedHandle *delayed_tail; + + /** + * Migration stop message in our queue, or NULL if we have none pending. + */ + struct GSF_PeerTransmitHandle *migration_pth; + + /** + * Context of our GNUNET_ATS_reserve_bandwidth call (or NULL). + */ + struct GNUNET_ATS_ReservationContext *rc; + + /** + * Task scheduled if we need to retry bandwidth reservation later. + */ + GNUNET_SCHEDULER_TaskIdentifier rc_delay_task; + + /** + * Active requests from this neighbour, map of query to 'struct PeerRequest'. + */ + struct GNUNET_CONTAINER_MultiHashMap *request_map; + + /** + * Increase in traffic preference still to be submitted + * to the core service for this peer. + */ + uint64_t inc_preference; + + /** + * Trust rating for this peer on disk. + */ + uint32_t disk_trust; + + /** + * Which offset in "last_p2p_replies" will be updated next? + * (we go round-robin). + */ + unsigned int last_p2p_replies_woff; + + /** + * Which offset in "last_client_replies" will be updated next? + * (we go round-robin). + */ + unsigned int last_client_replies_woff; + + /** + * Current offset into 'last_request_times' ring buffer. + */ + unsigned int last_request_times_off; + + /** + * GNUNET_YES if we did successfully reserve 32k bandwidth, + * GNUNET_NO if not. + */ + int did_reserve; + +}; + + +/** + * Map from peer identities to 'struct GSF_ConnectPeer' entries. + */ +static struct GNUNET_CONTAINER_MultiHashMap *cp_map; + +/** + * Where do we store trust information? + */ +static char *trustDirectory; + +/** + * Handle to ATS service. + */ +static struct GNUNET_ATS_PerformanceHandle *ats; + +/** + * Get the filename under which we would store the GNUNET_HELLO_Message + * for the given host and protocol. + * @return filename of the form DIRECTORY/HOSTID + */ +static char * +get_trust_filename (const struct GNUNET_PeerIdentity *id) +{ + struct GNUNET_CRYPTO_HashAsciiEncoded fil; + char *fn; + + GNUNET_CRYPTO_hash_to_enc (&id->hashPubKey, &fil); + GNUNET_asprintf (&fn, "%s%s%s", trustDirectory, DIR_SEPARATOR_STR, &fil); + return fn; +} + + +/** + * Find latency information in 'atsi'. + * + * @param atsi performance data + * @param atsi_count number of records in 'atsi' + * @return connection latency + */ +static struct GNUNET_TIME_Relative +get_latency (const struct GNUNET_ATS_Information *atsi, unsigned int atsi_count) +{ + unsigned int i; + + for (i = 0; i < atsi_count; i++) + if (ntohl (atsi->type) == GNUNET_ATS_QUALITY_NET_DELAY) + return GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MILLISECONDS, + ntohl (atsi->value)); + return GNUNET_TIME_UNIT_SECONDS; +} + + +/** + * Update the performance information kept for the given peer. + * + * @param cp peer record to update + * @param atsi transport performance data + * @param atsi_count number of records in 'atsi' + */ +static void +update_atsi (struct GSF_ConnectedPeer *cp, + const struct GNUNET_ATS_Information *atsi, unsigned int atsi_count) +{ + struct GNUNET_TIME_Relative latency; + + latency = get_latency (atsi, atsi_count); + GNUNET_LOAD_value_set_decline (cp->ppd.transmission_delay, latency); + /* LATER: merge atsi into cp's performance data (if we ever care...) */ +} + + +/** + * Return the performance data record for the given peer + * + * @param cp peer to query + * @return performance data record for the peer + */ +struct GSF_PeerPerformanceData * +GSF_get_peer_performance_data_ (struct GSF_ConnectedPeer *cp) +{ + return &cp->ppd; +} + + +/** + * Core is ready to transmit to a peer, get the message. + * + * @param cls the 'struct GSF_PeerTransmitHandle' of the message + * @param size number of bytes core is willing to take + * @param buf where to copy the message + * @return number of bytes copied to buf + */ +static size_t +peer_transmit_ready_cb (void *cls, size_t size, void *buf); + + +/** + * Function called by core upon success or failure of our bandwidth reservation request. + * + * @param cls the 'struct GSF_ConnectedPeer' of the peer for which we made the request + * @param peer identifies the peer + * @param amount set to the amount that was actually reserved or unreserved; + * either the full requested amount or zero (no partial reservations) + * @param res_delay if the reservation could not be satisfied (amount was 0), how + * long should the client wait until re-trying? + */ +static void +ats_reserve_callback (void *cls, const struct GNUNET_PeerIdentity *peer, + int32_t amount, struct GNUNET_TIME_Relative res_delay); + + +/** + * If ready (bandwidth reserved), try to schedule transmission via + * core for the given handle. + * + * @param pth transmission handle to schedule + */ +static void +schedule_transmission (struct GSF_PeerTransmitHandle *pth) +{ + struct GSF_ConnectedPeer *cp; + struct GNUNET_PeerIdentity target; + + if ((NULL != pth->cth) || (0 != pth->cth_in_progress)) + return; /* already done */ + cp = pth->cp; + GNUNET_assert (0 != cp->ppd.pid); + GNUNET_PEER_resolve (cp->ppd.pid, &target); + + if (0 != cp->inc_preference) + { + GNUNET_ATS_change_preference (ats, &target, GNUNET_ATS_PREFERENCE_BANDWIDTH, + (double) cp->inc_preference, + GNUNET_ATS_PREFERENCE_END); + cp->inc_preference = 0; + } + + if ((GNUNET_YES == pth->is_query) && (GNUNET_YES != pth->was_reserved)) + { + /* query, need reservation */ + if (GNUNET_YES != cp->did_reserve) + return; /* not ready */ + cp->did_reserve = GNUNET_NO; + /* reservation already done! */ + pth->was_reserved = GNUNET_YES; + cp->rc = + GNUNET_ATS_reserve_bandwidth (ats, &target, DBLOCK_SIZE, + &ats_reserve_callback, cp); + } + GNUNET_assert (pth->cth == NULL); + pth->cth_in_progress++; + pth->cth = + GNUNET_CORE_notify_transmit_ready (GSF_core, GNUNET_YES, pth->priority, + GNUNET_TIME_absolute_get_remaining + (pth->timeout), &target, pth->size, + &peer_transmit_ready_cb, pth); + GNUNET_assert (0 < pth->cth_in_progress--); +} + + +/** + * Core is ready to transmit to a peer, get the message. + * + * @param cls the 'struct GSF_PeerTransmitHandle' of the message + * @param size number of bytes core is willing to take + * @param buf where to copy the message + * @return number of bytes copied to buf + */ +static size_t +peer_transmit_ready_cb (void *cls, size_t size, void *buf) +{ + struct GSF_PeerTransmitHandle *pth = cls; + struct GSF_PeerTransmitHandle *pos; + struct GSF_ConnectedPeer *cp; + size_t ret; + + GNUNET_assert ((NULL == buf) || (pth->size <= size)); + pth->cth = NULL; + if (pth->timeout_task != GNUNET_SCHEDULER_NO_TASK) + { + GNUNET_SCHEDULER_cancel (pth->timeout_task); + pth->timeout_task = GNUNET_SCHEDULER_NO_TASK; + } + cp = pth->cp; + GNUNET_CONTAINER_DLL_remove (cp->pth_head, cp->pth_tail, pth); + if (GNUNET_YES == pth->is_query) + { + cp->ppd.last_request_times[(cp->last_request_times_off++) % + MAX_QUEUE_PER_PEER] = + GNUNET_TIME_absolute_get (); + GNUNET_assert (0 < cp->ppd.pending_queries--); + } + else if (GNUNET_NO == pth->is_query) + { + GNUNET_assert (0 < cp->ppd.pending_replies--); + } + GNUNET_LOAD_update (cp->ppd.transmission_delay, + GNUNET_TIME_absolute_get_duration + (pth->transmission_request_start_time).rel_value); + ret = pth->gmc (pth->gmc_cls, size, buf); + GNUNET_assert (NULL == pth->cth); + for (pos = cp->pth_head; pos != NULL; pos = pos->next) + { + GNUNET_assert (pos != pth); + schedule_transmission (pos); + } + GNUNET_assert (pth->cth == NULL); + GNUNET_assert (pth->cth_in_progress == 0); + GNUNET_free (pth); + return ret; +} + + +/** + * (re)try to reserve bandwidth from the given peer. + * + * @param cls the 'struct GSF_ConnectedPeer' to reserve from + * @param tc scheduler context + */ +static void +retry_reservation (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct GSF_ConnectedPeer *cp = cls; + struct GNUNET_PeerIdentity target; + + GNUNET_PEER_resolve (cp->ppd.pid, &target); + cp->rc_delay_task = GNUNET_SCHEDULER_NO_TASK; + cp->rc = + GNUNET_ATS_reserve_bandwidth (ats, &target, DBLOCK_SIZE, + &ats_reserve_callback, cp); +} + + +/** + * Function called by core upon success or failure of our bandwidth reservation request. + * + * @param cls the 'struct GSF_ConnectedPeer' of the peer for which we made the request + * @param peer identifies the peer + * @param amount set to the amount that was actually reserved or unreserved; + * either the full requested amount or zero (no partial reservations) + * @param res_delay if the reservation could not be satisfied (amount was 0), how + * long should the client wait until re-trying? + */ +static void +ats_reserve_callback (void *cls, const struct GNUNET_PeerIdentity *peer, + int32_t amount, struct GNUNET_TIME_Relative res_delay) +{ + struct GSF_ConnectedPeer *cp = cls; + struct GSF_PeerTransmitHandle *pth; + + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Reserved %d bytes / need to wait %llu ms for reservation\n", + (int) amount, (unsigned long long) res_delay.rel_value); + cp->rc = NULL; + if (0 == amount) + { + cp->rc_delay_task = + GNUNET_SCHEDULER_add_delayed (res_delay, &retry_reservation, cp); + return; + } + cp->did_reserve = GNUNET_YES; + pth = cp->pth_head; + if ((NULL != pth) && (NULL == pth->cth)) + { + /* reservation success, try transmission now! */ + pth->cth_in_progress++; + pth->cth = + GNUNET_CORE_notify_transmit_ready (GSF_core, GNUNET_YES, pth->priority, + GNUNET_TIME_absolute_get_remaining + (pth->timeout), peer, pth->size, + &peer_transmit_ready_cb, pth); + GNUNET_assert (0 < pth->cth_in_progress--); + } +} + + +/** + * A peer connected to us. Setup the connected peer + * records. + * + * @param peer identity of peer that connected + * @param atsi performance data for the connection + * @param atsi_count number of records in 'atsi' + * @return handle to connected peer entry + */ +struct GSF_ConnectedPeer * +GSF_peer_connect_handler_ (const struct GNUNET_PeerIdentity *peer, + const struct GNUNET_ATS_Information *atsi, + unsigned int atsi_count) +{ + struct GSF_ConnectedPeer *cp; + char *fn; + uint32_t trust; + + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Connected to peer %s\n", + GNUNET_i2s (peer)); + cp = GNUNET_malloc (sizeof (struct GSF_ConnectedPeer)); + cp->ppd.pid = GNUNET_PEER_intern (peer); + cp->ppd.transmission_delay = GNUNET_LOAD_value_init (GNUNET_TIME_UNIT_ZERO); + cp->rc = + GNUNET_ATS_reserve_bandwidth (ats, peer, DBLOCK_SIZE, + &ats_reserve_callback, cp); + fn = get_trust_filename (peer); + if ((GNUNET_DISK_file_test (fn) == GNUNET_YES) && + (sizeof (trust) == GNUNET_DISK_fn_read (fn, &trust, sizeof (trust)))) + cp->disk_trust = cp->ppd.trust = ntohl (trust); + GNUNET_free (fn); + cp->request_map = GNUNET_CONTAINER_multihashmap_create (128); + GNUNET_break (GNUNET_OK == + GNUNET_CONTAINER_multihashmap_put (cp_map, &peer->hashPubKey, + cp, + GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY)); + GNUNET_STATISTICS_set (GSF_stats, gettext_noop ("# peers connected"), + GNUNET_CONTAINER_multihashmap_size (cp_map), + GNUNET_NO); + update_atsi (cp, atsi, atsi_count); + GSF_push_start_ (cp); + return cp; +} + + +/** + * It may be time to re-start migrating content to this + * peer. Check, and if so, restart migration. + * + * @param cls the 'struct GSF_ConnectedPeer' + * @param tc scheduler context + */ +static void +revive_migration (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct GSF_ConnectedPeer *cp = cls; + struct GNUNET_TIME_Relative bt; + + cp->mig_revive_task = GNUNET_SCHEDULER_NO_TASK; + bt = GNUNET_TIME_absolute_get_remaining (cp->ppd.migration_blocked_until); + if (0 != bt.rel_value) + { + /* still time left... */ + cp->mig_revive_task = + GNUNET_SCHEDULER_add_delayed (bt, &revive_migration, cp); + return; + } + GSF_push_start_ (cp); +} + + +/** + * Get a handle for a connected peer. + * + * @param peer peer's identity + * @return NULL if the peer is not currently connected + */ +struct GSF_ConnectedPeer * +GSF_peer_get_ (const struct GNUNET_PeerIdentity *peer) +{ + if (NULL == cp_map) + return NULL; + return GNUNET_CONTAINER_multihashmap_get (cp_map, &peer->hashPubKey); +} + + +/** + * Handle P2P "MIGRATION_STOP" message. + * + * @param cls closure, always NULL + * @param other the other peer involved (sender or receiver, NULL + * for loopback messages where we are both sender and receiver) + * @param message the actual message + * @param atsi performance information + * @param atsi_count number of records in 'atsi' + * @return GNUNET_OK to keep the connection open, + * GNUNET_SYSERR to close it (signal serious error) + */ +int +GSF_handle_p2p_migration_stop_ (void *cls, + const struct GNUNET_PeerIdentity *other, + const struct GNUNET_MessageHeader *message, + const struct GNUNET_ATS_Information *atsi, + unsigned int atsi_count) +{ + struct GSF_ConnectedPeer *cp; + const struct MigrationStopMessage *msm; + struct GNUNET_TIME_Relative bt; + + msm = (const struct MigrationStopMessage *) message; + cp = GSF_peer_get_ (other); + if (cp == NULL) + { + GNUNET_break (0); + return GNUNET_OK; + } + GNUNET_STATISTICS_update (GSF_stats, + gettext_noop ("# migration stop messages received"), + 1, GNUNET_NO); + bt = GNUNET_TIME_relative_ntoh (msm->duration); + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + _("Migration of content to peer `%s' blocked for %llu ms\n"), + GNUNET_i2s (other), (unsigned long long) bt.rel_value); + cp->ppd.migration_blocked_until = GNUNET_TIME_relative_to_absolute (bt); + if (cp->mig_revive_task == GNUNET_SCHEDULER_NO_TASK) + { + GSF_push_stop_ (cp); + cp->mig_revive_task = + GNUNET_SCHEDULER_add_delayed (bt, &revive_migration, cp); + } + update_atsi (cp, atsi, atsi_count); + return GNUNET_OK; +} + + +/** + * Copy reply and free put message. + * + * @param cls the 'struct PutMessage' + * @param buf_size number of bytes available in buf + * @param buf where to copy the message, NULL on error (peer disconnect) + * @return number of bytes copied to 'buf', can be 0 (without indicating an error) + */ +static size_t +copy_reply (void *cls, size_t buf_size, void *buf) +{ + struct PutMessage *pm = cls; + size_t size; + + if (buf != NULL) + { + GNUNET_assert (buf_size >= ntohs (pm->header.size)); + size = ntohs (pm->header.size); + memcpy (buf, pm, size); + GNUNET_STATISTICS_update (GSF_stats, + gettext_noop + ("# replies transmitted to other peers"), 1, + GNUNET_NO); + } + else + { + size = 0; + GNUNET_STATISTICS_update (GSF_stats, gettext_noop ("# replies dropped"), 1, + GNUNET_NO); + } + GNUNET_free (pm); + return size; +} + + +/** + * Free resources associated with the given peer request. + * + * @param peerreq request to free + * @param query associated key for the request + */ +static void +free_pending_request (struct PeerRequest *peerreq, + const GNUNET_HashCode *query) +{ + struct GSF_ConnectedPeer *cp = peerreq->cp; + + if (peerreq->kill_task != GNUNET_SCHEDULER_NO_TASK) + { + GNUNET_SCHEDULER_cancel (peerreq->kill_task); + peerreq->kill_task = GNUNET_SCHEDULER_NO_TASK; + } + GNUNET_STATISTICS_update (GSF_stats, gettext_noop ("# P2P searches active"), + -1, GNUNET_NO); + GNUNET_break (GNUNET_YES == + GNUNET_CONTAINER_multihashmap_remove (cp->request_map, + query, peerreq)); + GNUNET_free (peerreq); +} + + +/** + * Cancel all requests associated with the peer. + * + * @param cls unused + * @param query hash code of the request + * @param value the 'struct GSF_PendingRequest' + * @return GNUNET_YES (continue to iterate) + */ +static int +cancel_pending_request (void *cls, const GNUNET_HashCode * query, void *value) +{ + struct PeerRequest *peerreq = value; + struct GSF_PendingRequest *pr = peerreq->pr; + struct GSF_PendingRequestData *prd; + + prd = GSF_pending_request_get_data_ (pr); + GSF_pending_request_cancel_ (pr, GNUNET_NO); + free_pending_request (peerreq, &prd->query); + return GNUNET_OK; +} + + +/** + * Free the given request. + * + * @param cls the request to free + * @param tc task context + */ +static void +peer_request_destroy (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct PeerRequest *peerreq = cls; + struct GSF_PendingRequest *pr = peerreq->pr; + struct GSF_PendingRequestData *prd; + + peerreq->kill_task = GNUNET_SCHEDULER_NO_TASK; + prd = GSF_pending_request_get_data_ (pr); + cancel_pending_request (NULL, &prd->query, peerreq); +} + + +/** + * The artificial delay is over, transmit the message now. + * + * @param cls the 'struct GSF_DelayedHandle' with the message + * @param tc scheduler context + */ +static void +transmit_delayed_now (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct GSF_DelayedHandle *dh = cls; + struct GSF_ConnectedPeer *cp = dh->cp; + + GNUNET_CONTAINER_DLL_remove (cp->delayed_head, cp->delayed_tail, dh); + if (0 != (GNUNET_SCHEDULER_REASON_SHUTDOWN & tc->reason)) + { + GNUNET_free (dh->pm); + GNUNET_free (dh); + return; + } + (void) GSF_peer_transmit_ (cp, GNUNET_NO, UINT32_MAX, REPLY_TIMEOUT, + dh->msize, ©_reply, dh->pm); + GNUNET_free (dh); +} + + +/** + * Get the randomized delay a response should be subjected to. + * + * @return desired delay + */ +static struct GNUNET_TIME_Relative +get_randomized_delay () +{ + struct GNUNET_TIME_Relative ret; + + ret = + GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MILLISECONDS, + GNUNET_CRYPTO_random_u32 + (GNUNET_CRYPTO_QUALITY_WEAK, + 2 * GSF_avg_latency.rel_value + 1)); + GNUNET_STATISTICS_update (GSF_stats, + gettext_noop + ("# artificial delays introduced (ms)"), + ret.rel_value, GNUNET_NO); + + return ret; +} + + +/** + * Handle a reply to a pending request. Also called if a request + * expires (then with data == NULL). The handler may be called + * many times (depending on the request type), but will not be + * called during or after a call to GSF_pending_request_cancel + * and will also not be called anymore after a call signalling + * expiration. + * + * @param cls 'struct PeerRequest' this is an answer for + * @param eval evaluation of the result + * @param pr handle to the original pending request + * @param reply_anonymity_level anonymity level for the reply, UINT32_MAX for "unknown" + * @param expiration when does 'data' expire? + * @param last_transmission when did we last transmit a request for this block + * @param type type of the block + * @param data response data, NULL on request expiration + * @param data_len number of bytes in data + */ +static void +handle_p2p_reply (void *cls, enum GNUNET_BLOCK_EvaluationResult eval, + struct GSF_PendingRequest *pr, uint32_t reply_anonymity_level, + struct GNUNET_TIME_Absolute expiration, + struct GNUNET_TIME_Absolute last_transmission, + enum GNUNET_BLOCK_Type type, const void *data, + size_t data_len) +{ + struct PeerRequest *peerreq = cls; + struct GSF_ConnectedPeer *cp = peerreq->cp; + struct GSF_PendingRequestData *prd; + struct PutMessage *pm; + size_t msize; + + GNUNET_assert (data_len + sizeof (struct PutMessage) < + GNUNET_SERVER_MAX_MESSAGE_SIZE); + GNUNET_assert (peerreq->pr == pr); + prd = GSF_pending_request_get_data_ (pr); + if (NULL == data) + { + free_pending_request (peerreq, &prd->query); + return; + } + GNUNET_break (type != GNUNET_BLOCK_TYPE_ANY); + if ((prd->type != type) && (prd->type != GNUNET_BLOCK_TYPE_ANY)) + { + GNUNET_STATISTICS_update (GSF_stats, + gettext_noop + ("# replies dropped due to type mismatch"), + 1, GNUNET_NO); + return; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Transmitting result for query `%s' to peer\n", + GNUNET_h2s (&prd->query)); + GNUNET_STATISTICS_update (GSF_stats, + gettext_noop ("# replies received for other peers"), + 1, GNUNET_NO); + msize = sizeof (struct PutMessage) + data_len; + if (msize >= GNUNET_SERVER_MAX_MESSAGE_SIZE) + { + GNUNET_break (0); + return; + } + if ((reply_anonymity_level != UINT32_MAX) && (reply_anonymity_level > 1)) + { + if (reply_anonymity_level - 1 > GSF_cover_content_count) + { + GNUNET_STATISTICS_update (GSF_stats, + gettext_noop + ("# replies dropped due to insufficient cover traffic"), + 1, GNUNET_NO); + return; + } + GSF_cover_content_count -= (reply_anonymity_level - 1); + } + + pm = GNUNET_malloc (msize); + pm->header.type = htons (GNUNET_MESSAGE_TYPE_FS_PUT); + pm->header.size = htons (msize); + pm->type = htonl (type); + pm->expiration = GNUNET_TIME_absolute_hton (expiration); + memcpy (&pm[1], data, data_len); + if ((reply_anonymity_level != UINT32_MAX) && (reply_anonymity_level != 0) && + (GSF_enable_randomized_delays == GNUNET_YES)) + { + struct GSF_DelayedHandle *dh; + + dh = GNUNET_malloc (sizeof (struct GSF_DelayedHandle)); + dh->cp = cp; + dh->pm = pm; + dh->msize = msize; + GNUNET_CONTAINER_DLL_insert (cp->delayed_head, cp->delayed_tail, dh); + dh->delay_task = + GNUNET_SCHEDULER_add_delayed (get_randomized_delay (), + &transmit_delayed_now, dh); + } + else + { + (void) GSF_peer_transmit_ (cp, GNUNET_NO, UINT32_MAX, REPLY_TIMEOUT, msize, + ©_reply, pm); + } + if (eval != GNUNET_BLOCK_EVALUATION_OK_LAST) + return; + if (GNUNET_SCHEDULER_NO_TASK == peerreq->kill_task) + { + GNUNET_STATISTICS_update (GSF_stats, + gettext_noop + ("# P2P searches destroyed due to ultimate reply"), + 1, GNUNET_NO); + peerreq->kill_task = + GNUNET_SCHEDULER_add_now (&peer_request_destroy, peerreq); + } +} + + +/** + * Increase the host credit by a value. + * + * @param cp which peer to change the trust value on + * @param value is the int value by which the + * host credit is to be increased or decreased + * @returns the actual change in trust (positive or negative) + */ +static int +change_host_trust (struct GSF_ConnectedPeer *cp, int value) +{ + if (value == 0) + return 0; + GNUNET_assert (cp != NULL); + if (value > 0) + { + if (cp->ppd.trust + value < cp->ppd.trust) + { + value = UINT32_MAX - cp->ppd.trust; + cp->ppd.trust = UINT32_MAX; + } + else + cp->ppd.trust += value; + } + else + { + if (cp->ppd.trust < -value) + { + value = -cp->ppd.trust; + cp->ppd.trust = 0; + } + else + cp->ppd.trust += value; + } + return value; +} + + +/** + * We've received a request with the specified priority. Bound it + * according to how much we trust the given peer. + * + * @param prio_in requested priority + * @param cp the peer making the request + * @return effective priority + */ +static int32_t +bound_priority (uint32_t prio_in, struct GSF_ConnectedPeer *cp) +{ +#define N ((double)128.0) + uint32_t ret; + double rret; + int ld; + + ld = GSF_test_get_load_too_high_ (0); + if (ld == GNUNET_SYSERR) + { + GNUNET_STATISTICS_update (GSF_stats, + gettext_noop + ("# requests done for free (low load)"), 1, + GNUNET_NO); + return 0; /* excess resources */ + } + if (prio_in > INT32_MAX) + prio_in = INT32_MAX; + ret = -change_host_trust (cp, -(int) prio_in); + if (ret > 0) + { + if (ret > GSF_current_priorities + N) + rret = GSF_current_priorities + N; + else + rret = ret; + GSF_current_priorities = (GSF_current_priorities * (N - 1) + rret) / N; + } + if ((ld == GNUNET_YES) && (ret > 0)) + { + /* try with charging */ + ld = GSF_test_get_load_too_high_ (ret); + } + if (ld == GNUNET_YES) + { + GNUNET_STATISTICS_update (GSF_stats, + gettext_noop + ("# request dropped, priority insufficient"), 1, + GNUNET_NO); + /* undo charge */ + change_host_trust (cp, (int) ret); + return -1; /* not enough resources */ + } + else + { + GNUNET_STATISTICS_update (GSF_stats, + gettext_noop + ("# requests done for a price (normal load)"), 1, + GNUNET_NO); + } +#undef N + return ret; +} + + +/** + * The priority level imposes a bound on the maximum + * value for the ttl that can be requested. + * + * @param ttl_in requested ttl + * @param prio given priority + * @return ttl_in if ttl_in is below the limit, + * otherwise the ttl-limit for the given priority + */ +static int32_t +bound_ttl (int32_t ttl_in, uint32_t prio) +{ + unsigned long long allowed; + + if (ttl_in <= 0) + return ttl_in; + allowed = ((unsigned long long) prio) * TTL_DECREMENT / 1000; + if (ttl_in > allowed) + { + if (allowed >= (1 << 30)) + return 1 << 30; + return allowed; + } + return ttl_in; +} + + +/** + * Handle P2P "QUERY" message. Creates the pending request entry + * and sets up all of the data structures to that we will + * process replies properly. Does not initiate forwarding or + * local database lookups. + * + * @param other the other peer involved (sender or receiver, NULL + * for loopback messages where we are both sender and receiver) + * @param message the actual message + * @return pending request handle, NULL on error + */ +struct GSF_PendingRequest * +GSF_handle_p2p_query_ (const struct GNUNET_PeerIdentity *other, + const struct GNUNET_MessageHeader *message) +{ + struct PeerRequest *peerreq; + struct GSF_PendingRequest *pr; + struct GSF_PendingRequestData *prd; + struct GSF_ConnectedPeer *cp; + struct GSF_ConnectedPeer *cps; + const GNUNET_HashCode *namespace; + const struct GNUNET_PeerIdentity *target; + enum GSF_PendingRequestOptions options; + uint16_t msize; + const struct GetMessage *gm; + unsigned int bits; + const GNUNET_HashCode *opt; + uint32_t bm; + size_t bfsize; + uint32_t ttl_decrement; + int32_t priority; + int32_t ttl; + enum GNUNET_BLOCK_Type type; + GNUNET_PEER_Id spid; + + GNUNET_assert (other != NULL); + msize = ntohs (message->size); + if (msize < sizeof (struct GetMessage)) + { + GNUNET_break_op (0); + return NULL; + } + GNUNET_STATISTICS_update (GSF_stats, + gettext_noop + ("# GET requests received (from other peers)"), 1, + GNUNET_NO); + gm = (const struct GetMessage *) message; + type = ntohl (gm->type); + bm = ntohl (gm->hash_bitmap); + bits = 0; + while (bm > 0) + { + if (1 == (bm & 1)) + bits++; + bm >>= 1; + } + if (msize < sizeof (struct GetMessage) + bits * sizeof (GNUNET_HashCode)) + { + GNUNET_break_op (0); + return NULL; + } + opt = (const GNUNET_HashCode *) &gm[1]; + bfsize = msize - sizeof (struct GetMessage) - bits * sizeof (GNUNET_HashCode); + /* bfsize must be power of 2, check! */ + if (0 != ((bfsize - 1) & bfsize)) + { + GNUNET_break_op (0); + return NULL; + } + GSF_cover_query_count++; + bm = ntohl (gm->hash_bitmap); + bits = 0; + cps = GSF_peer_get_ (other); + if (NULL == cps) + { + /* peer must have just disconnected */ + GNUNET_STATISTICS_update (GSF_stats, + gettext_noop + ("# requests dropped due to initiator not being connected"), + 1, GNUNET_NO); + return NULL; + } + if (0 != (bm & GET_MESSAGE_BIT_RETURN_TO)) + cp = GSF_peer_get_ ((const struct GNUNET_PeerIdentity *) &opt[bits++]); + else + cp = cps; + if (cp == NULL) + { + if (0 != (bm & GET_MESSAGE_BIT_RETURN_TO)) + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Failed to find RETURN-TO peer `%4s' in connection set. Dropping query.\n", + GNUNET_i2s ((const struct GNUNET_PeerIdentity *) + &opt[bits - 1])); + + else + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Failed to find peer `%4s' in connection set. Dropping query.\n", + GNUNET_i2s (other)); + GNUNET_STATISTICS_update (GSF_stats, + gettext_noop + ("# requests dropped due to missing reverse route"), + 1, GNUNET_NO); + return NULL; + } + /* note that we can really only check load here since otherwise + * peers could find out that we are overloaded by not being + * disconnected after sending us a malformed query... */ + priority = bound_priority (ntohl (gm->priority), cps); + if (priority < 0) + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Dropping query from `%s', this peer is too busy.\n", + GNUNET_i2s (other)); + return NULL; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Received request for `%s' of type %u from peer `%4s' with flags %u\n", + GNUNET_h2s (&gm->query), (unsigned int) type, GNUNET_i2s (other), + (unsigned int) bm); + namespace = (0 != (bm & GET_MESSAGE_BIT_SKS_NAMESPACE)) ? &opt[bits++] : NULL; + if ((type == GNUNET_BLOCK_TYPE_FS_SBLOCK) && (namespace == NULL)) + { + GNUNET_break_op (0); + return NULL; + } + if ((type != GNUNET_BLOCK_TYPE_FS_SBLOCK) && (namespace != NULL)) + { + GNUNET_break_op (0); + return NULL; + } + target = + (0 != + (bm & GET_MESSAGE_BIT_TRANSMIT_TO)) ? ((const struct GNUNET_PeerIdentity + *) &opt[bits++]) : NULL; + options = GSF_PRO_DEFAULTS; + spid = 0; + if ((GNUNET_LOAD_get_load (cp->ppd.transmission_delay) > 3 * (1 + priority)) + || (GNUNET_LOAD_get_average (cp->ppd.transmission_delay) > + GNUNET_CONSTANTS_MAX_CORK_DELAY.rel_value * 2 + + GNUNET_LOAD_get_average (GSF_rt_entry_lifetime))) + { + /* don't have BW to send to peer, or would likely take longer than we have for it, + * so at best indirect the query */ + priority = 0; + options |= GSF_PRO_FORWARD_ONLY; + spid = GNUNET_PEER_intern (other); + GNUNET_assert (0 != spid); + } + ttl = bound_ttl (ntohl (gm->ttl), priority); + /* decrement ttl (always) */ + ttl_decrement = + 2 * TTL_DECREMENT + GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, + TTL_DECREMENT); + if ((ttl < 0) && (((int32_t) (ttl - ttl_decrement)) > 0)) + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Dropping query from `%s' due to TTL underflow (%d - %u).\n", + GNUNET_i2s (other), ttl, ttl_decrement); + GNUNET_STATISTICS_update (GSF_stats, + gettext_noop + ("# requests dropped due TTL underflow"), 1, + GNUNET_NO); + /* integer underflow => drop (should be very rare)! */ + return NULL; + } + ttl -= ttl_decrement; + + /* test if the request already exists */ + peerreq = GNUNET_CONTAINER_multihashmap_get (cp->request_map, &gm->query); + if (peerreq != NULL) + { + pr = peerreq->pr; + prd = GSF_pending_request_get_data_ (pr); + if ((prd->type == type) && + ((type != GNUNET_BLOCK_TYPE_FS_SBLOCK) || + (0 == memcmp (&prd->namespace, namespace, sizeof (GNUNET_HashCode))))) + { + if (prd->ttl.abs_value >= GNUNET_TIME_absolute_get ().abs_value + ttl) + { + /* existing request has higher TTL, drop new one! */ + prd->priority += priority; + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Have existing request with higher TTL, dropping new request.\n", + GNUNET_i2s (other)); + GNUNET_STATISTICS_update (GSF_stats, + gettext_noop + ("# requests dropped due to higher-TTL request"), + 1, GNUNET_NO); + return NULL; + } + /* existing request has lower TTL, drop old one! */ + priority += prd->priority; + GSF_pending_request_cancel_ (pr, GNUNET_YES); + free_pending_request (peerreq, &gm->query); + } + } + + peerreq = GNUNET_malloc (sizeof (struct PeerRequest)); + peerreq->cp = cp; + pr = GSF_pending_request_create_ (options, type, &gm->query, namespace, + target, + (bfsize > + 0) ? (const char *) &opt[bits] : NULL, + bfsize, ntohl (gm->filter_mutator), + 1 /* anonymity */ , + (uint32_t) priority, ttl, spid, GNUNET_PEER_intern (other), NULL, 0, /* replies_seen */ + &handle_p2p_reply, peerreq); + GNUNET_assert (NULL != pr); + peerreq->pr = pr; + GNUNET_break (GNUNET_OK == + GNUNET_CONTAINER_multihashmap_put (cp->request_map, &gm->query, + peerreq, + GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE)); + GNUNET_STATISTICS_update (GSF_stats, + gettext_noop + ("# P2P query messages received and processed"), 1, + GNUNET_NO); + GNUNET_STATISTICS_update (GSF_stats, gettext_noop ("# P2P searches active"), + 1, GNUNET_NO); + return pr; +} + + +/** + * Function called if there has been a timeout trying to satisfy + * a transmission request. + * + * @param cls the 'struct GSF_PeerTransmitHandle' of the request + * @param tc scheduler context + */ +static void +peer_transmit_timeout (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct GSF_PeerTransmitHandle *pth = cls; + struct GSF_ConnectedPeer *cp; + + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Timeout trying to transmit to other peer\n"); + pth->timeout_task = GNUNET_SCHEDULER_NO_TASK; + cp = pth->cp; + GNUNET_CONTAINER_DLL_remove (cp->pth_head, cp->pth_tail, pth); + if (GNUNET_YES == pth->is_query) + GNUNET_assert (0 < cp->ppd.pending_queries--); + else if (GNUNET_NO == pth->is_query) + GNUNET_assert (0 < cp->ppd.pending_replies--); + GNUNET_LOAD_update (cp->ppd.transmission_delay, UINT64_MAX); + if (NULL != pth->cth) + { + GNUNET_CORE_notify_transmit_ready_cancel (pth->cth); + pth->cth = NULL; + } + pth->gmc (pth->gmc_cls, 0, NULL); + GNUNET_assert (0 == pth->cth_in_progress); + GNUNET_free (pth); +} + + +/** + * Transmit a message to the given peer as soon as possible. + * If the peer disconnects before the transmission can happen, + * the callback is invoked with a 'NULL' buffer. + * + * @param cp target peer + * @param is_query is this a query (GNUNET_YES) or content (GNUNET_NO) or neither (GNUNET_SYSERR) + * @param priority how important is this request? + * @param timeout when does this request timeout (call gmc with error) + * @param size number of bytes we would like to send to the peer + * @param gmc function to call to get the message + * @param gmc_cls closure for gmc + * @return handle to cancel request + */ +struct GSF_PeerTransmitHandle * +GSF_peer_transmit_ (struct GSF_ConnectedPeer *cp, int is_query, + uint32_t priority, struct GNUNET_TIME_Relative timeout, + size_t size, GSF_GetMessageCallback gmc, void *gmc_cls) +{ + struct GSF_PeerTransmitHandle *pth; + struct GSF_PeerTransmitHandle *pos; + struct GSF_PeerTransmitHandle *prev; + + pth = GNUNET_malloc (sizeof (struct GSF_PeerTransmitHandle)); + pth->transmission_request_start_time = GNUNET_TIME_absolute_get (); + pth->timeout = GNUNET_TIME_relative_to_absolute (timeout); + pth->gmc = gmc; + pth->gmc_cls = gmc_cls; + pth->size = size; + pth->is_query = is_query; + pth->priority = priority; + pth->cp = cp; + /* insertion sort (by priority, descending) */ + prev = NULL; + pos = cp->pth_head; + while ((pos != NULL) && (pos->priority > priority)) + { + prev = pos; + pos = pos->next; + } + if (prev == NULL) + GNUNET_CONTAINER_DLL_insert (cp->pth_head, cp->pth_tail, pth); + else + GNUNET_CONTAINER_DLL_insert_after (cp->pth_head, cp->pth_tail, prev, pth); + if (GNUNET_YES == is_query) + cp->ppd.pending_queries++; + else if (GNUNET_NO == is_query) + cp->ppd.pending_replies++; + pth->timeout_task = + GNUNET_SCHEDULER_add_delayed (timeout, &peer_transmit_timeout, pth); + schedule_transmission (pth); + return pth; +} + + +/** + * Cancel an earlier request for transmission. + * + * @param pth request to cancel + */ +void +GSF_peer_transmit_cancel_ (struct GSF_PeerTransmitHandle *pth) +{ + struct GSF_ConnectedPeer *cp; + + if (pth->timeout_task != GNUNET_SCHEDULER_NO_TASK) + { + GNUNET_SCHEDULER_cancel (pth->timeout_task); + pth->timeout_task = GNUNET_SCHEDULER_NO_TASK; + } + if (NULL != pth->cth) + { + GNUNET_CORE_notify_transmit_ready_cancel (pth->cth); + pth->cth = NULL; + } + cp = pth->cp; + GNUNET_CONTAINER_DLL_remove (cp->pth_head, cp->pth_tail, pth); + if (GNUNET_YES == pth->is_query) + GNUNET_assert (0 < cp->ppd.pending_queries--); + else if (GNUNET_NO == pth->is_query) + GNUNET_assert (0 < cp->ppd.pending_replies--); + GNUNET_assert (0 == pth->cth_in_progress); + GNUNET_free (pth); +} + + +/** + * Report on receiving a reply; update the performance record of the given peer. + * + * @param cp responding peer (will be updated) + * @param request_time time at which the original query was transmitted + * @param request_priority priority of the original request + */ +void +GSF_peer_update_performance_ (struct GSF_ConnectedPeer *cp, + struct GNUNET_TIME_Absolute request_time, + uint32_t request_priority) +{ + struct GNUNET_TIME_Relative delay; + + delay = GNUNET_TIME_absolute_get_duration (request_time); + cp->ppd.avg_reply_delay.rel_value = + (cp->ppd.avg_reply_delay.rel_value * (RUNAVG_DELAY_N - 1) + + delay.rel_value) / RUNAVG_DELAY_N; + cp->ppd.avg_priority = + (cp->ppd.avg_priority * (RUNAVG_DELAY_N - 1) + + request_priority) / RUNAVG_DELAY_N; +} + + +/** + * Report on receiving a reply in response to an initiating client. + * Remember that this peer is good for this client. + * + * @param cp responding peer (will be updated) + * @param initiator_client local client on responsible for query + */ +void +GSF_peer_update_responder_client_ (struct GSF_ConnectedPeer *cp, + struct GSF_LocalClient *initiator_client) +{ + cp->ppd.last_client_replies[cp->last_client_replies_woff++ % + CS2P_SUCCESS_LIST_SIZE] = initiator_client; +} + + +/** + * Report on receiving a reply in response to an initiating peer. + * Remember that this peer is good for this initiating peer. + * + * @param cp responding peer (will be updated) + * @param initiator_peer other peer responsible for query + */ +void +GSF_peer_update_responder_peer_ (struct GSF_ConnectedPeer *cp, + const struct GSF_ConnectedPeer *initiator_peer) +{ + unsigned int woff; + + woff = cp->last_p2p_replies_woff % P2P_SUCCESS_LIST_SIZE; + GNUNET_PEER_change_rc (cp->ppd.last_p2p_replies[woff], -1); + cp->ppd.last_p2p_replies[woff] = initiator_peer->ppd.pid; + GNUNET_PEER_change_rc (initiator_peer->ppd.pid, 1); + cp->last_p2p_replies_woff = (woff + 1) % P2P_SUCCESS_LIST_SIZE; +} + + +/** + * A peer disconnected from us. Tear down the connected peer + * record. + * + * @param cls unused + * @param peer identity of peer that connected + */ +void +GSF_peer_disconnect_handler_ (void *cls, const struct GNUNET_PeerIdentity *peer) +{ + struct GSF_ConnectedPeer *cp; + struct GSF_PeerTransmitHandle *pth; + struct GSF_DelayedHandle *dh; + + cp = GSF_peer_get_ (peer); + if (NULL == cp) + return; /* must have been disconnect from core with + * 'peer' == my_id, ignore */ + GNUNET_assert (GNUNET_YES == + GNUNET_CONTAINER_multihashmap_remove (cp_map, + &peer->hashPubKey, cp)); + GNUNET_STATISTICS_set (GSF_stats, gettext_noop ("# peers connected"), + GNUNET_CONTAINER_multihashmap_size (cp_map), + GNUNET_NO); + if (NULL != cp->migration_pth) + { + GSF_peer_transmit_cancel_ (cp->migration_pth); + cp->migration_pth = NULL; + } + if (NULL != cp->rc) + { + GNUNET_ATS_reserve_bandwidth_cancel (cp->rc); + cp->rc = NULL; + } + if (GNUNET_SCHEDULER_NO_TASK != cp->rc_delay_task) + { + GNUNET_SCHEDULER_cancel (cp->rc_delay_task); + cp->rc_delay_task = GNUNET_SCHEDULER_NO_TASK; + } + GNUNET_CONTAINER_multihashmap_iterate (cp->request_map, + &cancel_pending_request, cp); + GNUNET_CONTAINER_multihashmap_destroy (cp->request_map); + cp->request_map = NULL; + GSF_plan_notify_peer_disconnect_ (cp); + GNUNET_LOAD_value_free (cp->ppd.transmission_delay); + GNUNET_PEER_decrement_rcs (cp->ppd.last_p2p_replies, P2P_SUCCESS_LIST_SIZE); + memset (cp->ppd.last_p2p_replies, 0, sizeof (cp->ppd.last_p2p_replies)); + GSF_push_stop_ (cp); + while (NULL != (pth = cp->pth_head)) + { + if (NULL != pth->cth) + { + GNUNET_CORE_notify_transmit_ready_cancel (pth->cth); + pth->cth = NULL; + } + if (pth->timeout_task != GNUNET_SCHEDULER_NO_TASK) + { + GNUNET_SCHEDULER_cancel (pth->timeout_task); + pth->timeout_task = GNUNET_SCHEDULER_NO_TASK; + } + GNUNET_CONTAINER_DLL_remove (cp->pth_head, cp->pth_tail, pth); + GNUNET_assert (0 == pth->cth_in_progress); + pth->gmc (pth->gmc_cls, 0, NULL); + GNUNET_free (pth); + } + while (NULL != (dh = cp->delayed_head)) + { + GNUNET_CONTAINER_DLL_remove (cp->delayed_head, cp->delayed_tail, dh); + GNUNET_SCHEDULER_cancel (dh->delay_task); + GNUNET_free (dh->pm); + GNUNET_free (dh); + } + GNUNET_PEER_change_rc (cp->ppd.pid, -1); + if (GNUNET_SCHEDULER_NO_TASK != cp->mig_revive_task) + { + GNUNET_SCHEDULER_cancel (cp->mig_revive_task); + cp->mig_revive_task = GNUNET_SCHEDULER_NO_TASK; + } + GNUNET_free (cp); +} + + +/** + * Closure for 'call_iterator'. + */ +struct IterationContext +{ + /** + * Function to call on each entry. + */ + GSF_ConnectedPeerIterator it; + + /** + * Closure for 'it'. + */ + void *it_cls; +}; + + +/** + * Function that calls the callback for each peer. + * + * @param cls the 'struct IterationContext*' + * @param key identity of the peer + * @param value the 'struct GSF_ConnectedPeer*' + * @return GNUNET_YES to continue iteration + */ +static int +call_iterator (void *cls, const GNUNET_HashCode * key, void *value) +{ + struct IterationContext *ic = cls; + struct GSF_ConnectedPeer *cp = value; + + ic->it (ic->it_cls, (const struct GNUNET_PeerIdentity *) key, cp, &cp->ppd); + return GNUNET_YES; +} + + +/** + * Iterate over all connected peers. + * + * @param it function to call for each peer + * @param it_cls closure for it + */ +void +GSF_iterate_connected_peers_ (GSF_ConnectedPeerIterator it, void *it_cls) +{ + struct IterationContext ic; + + ic.it = it; + ic.it_cls = it_cls; + GNUNET_CONTAINER_multihashmap_iterate (cp_map, &call_iterator, &ic); +} + + +/** + * Obtain the identity of a connected peer. + * + * @param cp peer to reserve bandwidth from + * @param id identity to set (written to) + */ +void +GSF_connected_peer_get_identity_ (const struct GSF_ConnectedPeer *cp, + struct GNUNET_PeerIdentity *id) +{ + GNUNET_assert (0 != cp->ppd.pid); + GNUNET_PEER_resolve (cp->ppd.pid, id); +} + + +/** + * Assemble a migration stop message for transmission. + * + * @param cls the 'struct GSF_ConnectedPeer' to use + * @param size number of bytes we're allowed to write to buf + * @param buf where to copy the message + * @return number of bytes copied to buf + */ +static size_t +create_migration_stop_message (void *cls, size_t size, void *buf) +{ + struct GSF_ConnectedPeer *cp = cls; + struct MigrationStopMessage msm; + + cp->migration_pth = NULL; + if (NULL == buf) + return 0; + GNUNET_assert (size >= sizeof (struct MigrationStopMessage)); + msm.header.size = htons (sizeof (struct MigrationStopMessage)); + msm.header.type = htons (GNUNET_MESSAGE_TYPE_FS_MIGRATION_STOP); + msm.reserved = htonl (0); + msm.duration = + GNUNET_TIME_relative_hton (GNUNET_TIME_absolute_get_remaining + (cp->last_migration_block)); + memcpy (buf, &msm, sizeof (struct MigrationStopMessage)); + GNUNET_STATISTICS_update (GSF_stats, + gettext_noop ("# migration stop messages sent"), + 1, GNUNET_NO); + return sizeof (struct MigrationStopMessage); +} + + +/** + * Ask a peer to stop migrating data to us until the given point + * in time. + * + * @param cp peer to ask + * @param block_time until when to block + */ +void +GSF_block_peer_migration_ (struct GSF_ConnectedPeer *cp, + struct GNUNET_TIME_Absolute block_time) +{ + if (cp->last_migration_block.abs_value > block_time.abs_value) + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Migration already blocked for another %llu ms\n", + (unsigned long long) + GNUNET_TIME_absolute_get_remaining + (cp->last_migration_block).rel_value); + return; /* already blocked */ + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Asking to stop migration for %llu ms\n", + (unsigned long long) GNUNET_TIME_absolute_get_remaining (block_time).rel_value); + cp->last_migration_block = block_time; + if (cp->migration_pth != NULL) + GSF_peer_transmit_cancel_ (cp->migration_pth); + cp->migration_pth = + GSF_peer_transmit_ (cp, GNUNET_SYSERR, UINT32_MAX, + GNUNET_TIME_UNIT_FOREVER_REL, + sizeof (struct MigrationStopMessage), + &create_migration_stop_message, cp); +} + + +/** + * Write host-trust information to a file - flush the buffer entry! + * + * @param cls closure, not used + * @param key host identity + * @param value the 'struct GSF_ConnectedPeer' to flush + * @return GNUNET_OK to continue iteration + */ +static int +flush_trust (void *cls, const GNUNET_HashCode * key, void *value) +{ + struct GSF_ConnectedPeer *cp = value; + char *fn; + uint32_t trust; + struct GNUNET_PeerIdentity pid; + + if (cp->ppd.trust == cp->disk_trust) + return GNUNET_OK; /* unchanged */ + GNUNET_assert (0 != cp->ppd.pid); + GNUNET_PEER_resolve (cp->ppd.pid, &pid); + fn = get_trust_filename (&pid); + if (cp->ppd.trust == 0) + { + if ((0 != UNLINK (fn)) && (errno != ENOENT)) + GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING | + GNUNET_ERROR_TYPE_BULK, "unlink", fn); + } + else + { + trust = htonl (cp->ppd.trust); + if (sizeof (uint32_t) == + GNUNET_DISK_fn_write (fn, &trust, sizeof (uint32_t), + GNUNET_DISK_PERM_USER_READ | + GNUNET_DISK_PERM_USER_WRITE | + GNUNET_DISK_PERM_GROUP_READ | + GNUNET_DISK_PERM_OTHER_READ)) + cp->disk_trust = cp->ppd.trust; + } + GNUNET_free (fn); + return GNUNET_OK; +} + + +/** + * Notify core about a preference we have for the given peer + * (to allocate more resources towards it). The change will + * be communicated the next time we reserve bandwidth with + * core (not instantly). + * + * @param cp peer to reserve bandwidth from + * @param pref preference change + */ +void +GSF_connected_peer_change_preference_ (struct GSF_ConnectedPeer *cp, + uint64_t pref) +{ + cp->inc_preference += pref; +} + + +/** + * Call this method periodically to flush trust information to disk. + * + * @param cls closure, not used + * @param tc task context, not used + */ +static void +cron_flush_trust (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + + if (NULL == cp_map) + return; + GNUNET_CONTAINER_multihashmap_iterate (cp_map, &flush_trust, NULL); + if (NULL == tc) + return; + if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN)) + return; + GNUNET_SCHEDULER_add_delayed_with_priority (TRUST_FLUSH_FREQ, + GNUNET_SCHEDULER_PRIORITY_HIGH, + &cron_flush_trust, NULL); +} + + +/** + * Initialize peer management subsystem. + */ +void +GSF_connected_peer_init_ () +{ + cp_map = GNUNET_CONTAINER_multihashmap_create (128); + ats = GNUNET_ATS_performance_init (GSF_cfg, NULL, NULL); + GNUNET_assert (GNUNET_OK == + GNUNET_CONFIGURATION_get_value_filename (GSF_cfg, "fs", + "TRUST", + &trustDirectory)); + GNUNET_DISK_directory_create (trustDirectory); + GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_HIGH, + &cron_flush_trust, NULL); +} + + +/** + * Iterator to free peer entries. + * + * @param cls closure, unused + * @param key current key code + * @param value value in the hash map (peer entry) + * @return GNUNET_YES (we should continue to iterate) + */ +static int +clean_peer (void *cls, const GNUNET_HashCode * key, void *value) +{ + GSF_peer_disconnect_handler_ (NULL, (const struct GNUNET_PeerIdentity *) key); + return GNUNET_YES; +} + + +/** + * Shutdown peer management subsystem. + */ +void +GSF_connected_peer_done_ () +{ + cron_flush_trust (NULL, NULL); + GNUNET_CONTAINER_multihashmap_iterate (cp_map, &clean_peer, NULL); + GNUNET_CONTAINER_multihashmap_destroy (cp_map); + cp_map = NULL; + GNUNET_free (trustDirectory); + trustDirectory = NULL; + GNUNET_ATS_performance_done (ats); + ats = NULL; +} + + +/** + * Iterator to remove references to LC entry. + * + * @param cls the 'struct GSF_LocalClient*' to look for + * @param key current key code + * @param value value in the hash map (peer entry) + * @return GNUNET_YES (we should continue to iterate) + */ +static int +clean_local_client (void *cls, const GNUNET_HashCode * key, void *value) +{ + const struct GSF_LocalClient *lc = cls; + struct GSF_ConnectedPeer *cp = value; + unsigned int i; + + for (i = 0; i < CS2P_SUCCESS_LIST_SIZE; i++) + if (cp->ppd.last_client_replies[i] == lc) + cp->ppd.last_client_replies[i] = NULL; + return GNUNET_YES; +} + + +/** + * Notification that a local client disconnected. Clean up all of our + * references to the given handle. + * + * @param lc handle to the local client (henceforth invalid) + */ +void +GSF_handle_local_client_disconnect_ (const struct GSF_LocalClient *lc) +{ + if (NULL == cp_map) + return; /* already cleaned up */ + GNUNET_CONTAINER_multihashmap_iterate (cp_map, &clean_local_client, + (void *) lc); +} + + +/* end of gnunet-service-fs_cp.c */ diff --git a/src/fs/gnunet-service-fs_cp.h b/src/fs/gnunet-service-fs_cp.h new file mode 100644 index 0000000..e3c7cd2 --- /dev/null +++ b/src/fs/gnunet-service-fs_cp.h @@ -0,0 +1,420 @@ +/* + This file is part of GNUnet. + (C) 2011 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/gnunet-service-fs_cp.h + * @brief API to handle 'connected peers' + * @author Christian Grothoff + */ +#ifndef GNUNET_SERVICE_FS_CP_H +#define GNUNET_SERVICE_FS_CP_H + +#include "fs.h" +#include "gnunet-service-fs.h" + + +/** + * Maximum number of outgoing messages we queue per peer. + * + * Performance measurements for 2 peer setup for 50 MB file + * (with MAX_DATASTORE_QUEUE = 1 and RETRY_PROBABILITY_INV = 1): + * + * 2: 1700 kb/s, 1372 kb/s + * 8: 2117 kb/s, 1284 kb/s, 1112 kb/s + * 16: 3500 kb/s, 3200 kb/s, 3388 kb/s + * 32: 3441 kb/s, 3163 kb/s, 3277 kb/s + * 128: 1700 kb/s; 2010 kb/s, 3383 kb/s, 1156 kb/s + * + * Conclusion: 16 seems to be a pretty good value (stable + * and high performance, no excessive memory use). + */ +#define MAX_QUEUE_PER_PEER 16 + +/** + * Length of the P2P success tracker. Note that having a very long + * list can also hurt performance. + */ +#define P2P_SUCCESS_LIST_SIZE 8 + +/** + * Length of the CS-2-P success tracker. Note that + * having a very long list can also hurt performance. + */ +#define CS2P_SUCCESS_LIST_SIZE 8 + + +/** + * Performance data kept for a peer. + */ +struct GSF_PeerPerformanceData +{ + + /** + * Transport performance data. + */ + struct GNUNET_ATS_Information *atsi; + + /** + * List of the last clients for which this peer successfully + * answered a query. + */ + struct GSF_LocalClient *last_client_replies[CS2P_SUCCESS_LIST_SIZE]; + + /** + * List of the last PIDs for which + * this peer successfully answered a query; + * We use 0 to indicate no successful reply. + */ + GNUNET_PEER_Id last_p2p_replies[P2P_SUCCESS_LIST_SIZE]; + + /** + * Average delay between sending the peer a request and + * getting a reply (only calculated over the requests for + * which we actually got a reply). Calculated + * as a moving average: new_delay = ((n-1)*last_delay+curr_delay) / n + */ + struct GNUNET_TIME_Relative avg_reply_delay; + + /** + * If we get content we already have from this peer, for how + * long do we block him? Adjusted based on the fraction of + * redundant data we receive, between 1s and 1h. + */ + struct GNUNET_TIME_Relative migration_delay; + + /** + * Point in time until which this peer does not want us to migrate content + * to it. + */ + struct GNUNET_TIME_Absolute migration_blocked_until; + + /** + * Transmission times for the last MAX_QUEUE_PER_PEER + * requests for this peer. Used as a ring buffer, current + * offset is stored in 'last_request_times_off'. If the + * oldest entry is more recent than the 'avg_delay', we should + * not send any more requests right now. + */ + struct GNUNET_TIME_Absolute last_request_times[MAX_QUEUE_PER_PEER]; + + /** + * How long does it typically take for us to transmit a message + * to this peer? (delay between the request being issued and + * the callback being invoked). + */ + struct GNUNET_LOAD_Value *transmission_delay; + + /** + * Average priority of successful replies. Calculated + * as a moving average: new_avg = ((n-1)*last_avg+curr_prio) / n + */ + double avg_priority; + + /** + * The peer's identity. + */ + GNUNET_PEER_Id pid; + + /** + * Trust rating for this peer + */ + uint32_t trust; + + /** + * Number of pending queries (replies are not counted) + */ + unsigned int pending_queries; + + /** + * Number of pending replies (queries are not counted) + */ + unsigned int pending_replies; + +}; + + +/** + * Signature of function called on a connected peer. + * + * @param cls closure + * @param peer identity of the peer + * @param cp handle to the connected peer record + * @param perf peer performance data + */ +typedef void (*GSF_ConnectedPeerIterator) (void *cls, + const struct GNUNET_PeerIdentity * + peer, struct GSF_ConnectedPeer * cp, + const struct GSF_PeerPerformanceData + * ppd); + + +/** + * Function called to get a message for transmission. + * + * @param cls closure + * @param buf_size number of bytes available in buf + * @param buf where to copy the message, NULL on error (peer disconnect) + * @return number of bytes copied to 'buf', can be 0 (without indicating an error) + */ +typedef size_t (*GSF_GetMessageCallback) (void *cls, size_t buf_size, + void *buf); + + +/** + * Signature of function called on a reservation success or failure. + * + * @param cls closure + * @param cp handle to the connected peer record + * @param success GNUNET_YES on success, GNUNET_NO on failure + */ +typedef void (*GSF_PeerReserveCallback) (void *cls, + struct GSF_ConnectedPeer * cp, + int success); + + +/** + * Handle to cancel a transmission request. + */ +struct GSF_PeerTransmitHandle; + + +/** + * A peer connected to us. Setup the connected peer + * records. + * + * @param peer identity of peer that connected + * @param atsi performance data for the connection + * @param atsi_count number of records in 'atsi' + * @return handle to connected peer entry + */ +struct GSF_ConnectedPeer * +GSF_peer_connect_handler_ (const struct GNUNET_PeerIdentity *peer, + const struct GNUNET_ATS_Information *atsi, + unsigned int atsi_count); + + +/** + * Get a handle for a connected peer. + * + * @param peer peer's identity + * @return NULL if this peer is not currently connected + */ +struct GSF_ConnectedPeer * +GSF_peer_get_ (const struct GNUNET_PeerIdentity *peer); + + +/** + * Transmit a message to the given peer as soon as possible. + * If the peer disconnects before the transmission can happen, + * the callback is invoked with a 'NULL' buffer. + * + * @param cp target peer + * @param is_query is this a query (GNUNET_YES) or content (GNUNET_NO) + * @param priority how important is this request? + * @param timeout when does this request timeout (call gmc with error) + * @param size number of bytes we would like to send to the peer + * @param gmc function to call to get the message + * @param gmc_cls closure for gmc + * @return handle to cancel request + */ +struct GSF_PeerTransmitHandle * +GSF_peer_transmit_ (struct GSF_ConnectedPeer *cp, int is_query, + uint32_t priority, struct GNUNET_TIME_Relative timeout, + size_t size, GSF_GetMessageCallback gmc, void *gmc_cls); + + +/** + * Cancel an earlier request for transmission. + * + * @param pth request to cancel + */ +void +GSF_peer_transmit_cancel_ (struct GSF_PeerTransmitHandle *pth); + + +/** + * Report on receiving a reply; update the performance record of the given peer. + * + * @param cp responding peer (will be updated) + * @param request_time time at which the original query was transmitted + * @param request_priority priority of the original request + */ +void +GSF_peer_update_performance_ (struct GSF_ConnectedPeer *cp, + struct GNUNET_TIME_Absolute request_time, + uint32_t request_priority); + + +/** + * Report on receiving a reply in response to an initiating client. + * Remember that this peer is good for this client. + * + * @param cp responding peer (will be updated) + * @param initiator_client local client on responsible for query + */ +void +GSF_peer_update_responder_client_ (struct GSF_ConnectedPeer *cp, + struct GSF_LocalClient *initiator_client); + + +/** + * Report on receiving a reply in response to an initiating peer. + * Remember that this peer is good for this initiating peer. + * + * @param cp responding peer (will be updated) + * @param initiator_peer other peer responsible for query + */ +void +GSF_peer_update_responder_peer_ (struct GSF_ConnectedPeer *cp, + const struct GSF_ConnectedPeer + *initiator_peer); + + +/** + * Handle P2P "MIGRATION_STOP" message. + * + * @param cls closure, always NULL + * @param other the other peer involved (sender or receiver, NULL + * for loopback messages where we are both sender and receiver) + * @param message the actual message + * @param atsi performance information + * @param atsi_count number of records in 'atsi' + * @return GNUNET_OK to keep the connection open, + * GNUNET_SYSERR to close it (signal serious error) + */ +int +GSF_handle_p2p_migration_stop_ (void *cls, + const struct GNUNET_PeerIdentity *other, + const struct GNUNET_MessageHeader *message, + const struct GNUNET_ATS_Information *atsi, + unsigned int atsi_count); + + +/** + * Handle P2P "QUERY" message. Only responsible for creating the + * request entry itself and setting up reply callback and cancellation + * on peer disconnect. Does NOT execute the actual request strategy + * (planning) or local database operations. + * + * @param other the other peer involved (sender or receiver, NULL + * for loopback messages where we are both sender and receiver) + * @param message the actual message + * @return pending request handle, NULL on error + */ +struct GSF_PendingRequest * +GSF_handle_p2p_query_ (const struct GNUNET_PeerIdentity *other, + const struct GNUNET_MessageHeader *message); + + +/** + * Return the performance data record for the given peer + * + * @param cp peer to query + * @return performance data record for the peer + */ +struct GSF_PeerPerformanceData * +GSF_get_peer_performance_data_ (struct GSF_ConnectedPeer *cp); + + +/** + * Ask a peer to stop migrating data to us until the given point + * in time. + * + * @param cp peer to ask + * @param block_time until when to block + */ +void +GSF_block_peer_migration_ (struct GSF_ConnectedPeer *cp, + struct GNUNET_TIME_Absolute block_time); + + +/** + * A peer disconnected from us. Tear down the connected peer + * record. + * + * @param cls unused + * @param peer identity of peer that connected + */ +void +GSF_peer_disconnect_handler_ (void *cls, + const struct GNUNET_PeerIdentity *peer); + + +/** + * Notification that a local client disconnected. Clean up all of our + * references to the given handle. + * + * @param lc handle to the local client (henceforth invalid) + */ +void +GSF_handle_local_client_disconnect_ (const struct GSF_LocalClient *lc); + + +/** + * Notify core about a preference we have for the given peer + * (to allocate more resources towards it). The change will + * be communicated the next time we reserve bandwidth with + * core (not instantly). + * + * @param cp peer to reserve bandwidth from + * @param pref preference change + */ +void +GSF_connected_peer_change_preference_ (struct GSF_ConnectedPeer *cp, + uint64_t pref); + + +/** + * Obtain the identity of a connected peer. + * + * @param cp peer to reserve bandwidth from + * @param id identity to set (written to) + */ +void +GSF_connected_peer_get_identity_ (const struct GSF_ConnectedPeer *cp, + struct GNUNET_PeerIdentity *id); + + +/** + * Iterate over all connected peers. + * + * @param it function to call for each peer + * @param it_cls closure for it + */ +void +GSF_iterate_connected_peers_ (GSF_ConnectedPeerIterator it, void *it_cls); + + +/** + * Initialize peer management subsystem. + */ +void +GSF_connected_peer_init_ (void); + + +/** + * Shutdown peer management subsystem. + */ +void +GSF_connected_peer_done_ (void); + + +#endif +/* end of gnunet-service-fs_cp.h */ diff --git a/src/fs/gnunet-service-fs_indexing.c b/src/fs/gnunet-service-fs_indexing.c new file mode 100644 index 0000000..b563019 --- /dev/null +++ b/src/fs/gnunet-service-fs_indexing.c @@ -0,0 +1,623 @@ +/* + This file is part of GNUnet. + (C) 2009, 2010 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/gnunet-service-fs_indexing.c + * @brief program that provides indexing functions of the file-sharing service + * @author Christian Grothoff + */ +#include "platform.h" +#include <float.h> +#include "gnunet_core_service.h" +#include "gnunet_datastore_service.h" +#include "gnunet_peer_lib.h" +#include "gnunet_protocols.h" +#include "gnunet_signatures.h" +#include "gnunet_util_lib.h" +#include "gnunet-service-fs.h" +#include "gnunet-service-fs_indexing.h" +#include "fs.h" + +/** + * In-memory information about indexed files (also available + * on-disk). + */ +struct IndexInfo +{ + + /** + * This is a linked list. + */ + struct IndexInfo *next; + + /** + * Name of the indexed file. Memory allocated + * at the end of this struct (do not free). + */ + const char *filename; + + /** + * Context for transmitting confirmation to client, + * NULL if we've done this already. + */ + struct GNUNET_SERVER_TransmitContext *tc; + + /** + * Context for hashing of the file. + */ + struct GNUNET_CRYPTO_FileHashContext *fhc; + + /** + * Hash of the contents of the file. + */ + GNUNET_HashCode file_id; + +}; + + +/** + * Linked list of indexed files. + */ +static struct IndexInfo *indexed_files; + +/** + * Maps hash over content of indexed files to the respective filename. + * The filenames are pointers into the indexed_files linked list and + * do not need to be freed. + */ +static struct GNUNET_CONTAINER_MultiHashMap *ifm; + +/** + * Our configuration. + */ +static const struct GNUNET_CONFIGURATION_Handle *cfg; + +/** + * Datastore handle. Created and destroyed by code in + * gnunet-service-fs (this is an alias). + */ +static struct GNUNET_DATASTORE_Handle *dsh; + + +/** + * Write the current index information list to disk. + */ +static void +write_index_list () +{ + struct GNUNET_BIO_WriteHandle *wh; + char *fn; + struct IndexInfo *pos; + + if (GNUNET_OK != + GNUNET_CONFIGURATION_get_value_filename (cfg, "FS", "INDEXDB", &fn)) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + _("Configuration option `%s' in section `%s' missing.\n"), + "INDEXDB", "FS"); + return; + } + wh = GNUNET_BIO_write_open (fn); + if (NULL == wh) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + _("Could not open `%s'.\n"), fn); + GNUNET_free (fn); + return; + } + pos = indexed_files; + while (pos != NULL) + { + if ((GNUNET_OK != + GNUNET_BIO_write (wh, &pos->file_id, sizeof (GNUNET_HashCode))) || + (GNUNET_OK != GNUNET_BIO_write_string (wh, pos->filename))) + break; + pos = pos->next; + } + if (GNUNET_OK != GNUNET_BIO_write_close (wh)) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + _("Error writing `%s'.\n"), fn); + GNUNET_free (fn); + return; + } + GNUNET_free (fn); +} + + +/** + * Read index information from disk. + */ +static void +read_index_list () +{ + struct GNUNET_BIO_ReadHandle *rh; + char *fn; + struct IndexInfo *pos; + char *fname; + GNUNET_HashCode hc; + size_t slen; + char *emsg; + + if (GNUNET_OK != + GNUNET_CONFIGURATION_get_value_filename (cfg, "FS", "INDEXDB", &fn)) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + _("Configuration option `%s' in section `%s' missing.\n"), + "INDEXDB", "FS"); + return; + } + if (GNUNET_NO == GNUNET_DISK_file_test (fn)) + { + /* no index info yet */ + GNUNET_free (fn); + return; + } + rh = GNUNET_BIO_read_open (fn); + if (NULL == rh) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + _("Could not open `%s'.\n"), fn); + GNUNET_free (fn); + return; + } + while ((GNUNET_OK == + GNUNET_BIO_read (rh, "Hash of indexed file", &hc, + sizeof (GNUNET_HashCode))) && + (GNUNET_OK == + GNUNET_BIO_read_string (rh, "Name of indexed file", &fname, + 1024 * 16)) && (fname != NULL)) + { + slen = strlen (fname) + 1; + pos = GNUNET_malloc (sizeof (struct IndexInfo) + slen); + pos->file_id = hc; + pos->filename = (const char *) &pos[1]; + memcpy (&pos[1], fname, slen); + if (GNUNET_SYSERR == + GNUNET_CONTAINER_multihashmap_put (ifm, &hc, (void *) pos->filename, + GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY)) + { + GNUNET_free (pos); + } + else + { + pos->next = indexed_files; + indexed_files = pos; + } + GNUNET_free (fname); + } + if (GNUNET_OK != GNUNET_BIO_read_close (rh, &emsg)) + GNUNET_free (emsg); + GNUNET_free (fn); +} + + +/** + * We've validated the hash of the file we're about to index. Signal + * success to the client and update our internal data structures. + * + * @param ii the index info entry for the request + */ +static void +signal_index_ok (struct IndexInfo *ii) +{ + if (GNUNET_SYSERR == + GNUNET_CONTAINER_multihashmap_put (ifm, &ii->file_id, + (void *) ii->filename, + GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY)) + { + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + _ + ("Index request received for file `%s' is already indexed as `%s'. Permitting anyway.\n"), + ii->filename, + (const char *) GNUNET_CONTAINER_multihashmap_get (ifm, + &ii->file_id)); + GNUNET_SERVER_transmit_context_append_data (ii->tc, NULL, 0, + GNUNET_MESSAGE_TYPE_FS_INDEX_START_OK); + GNUNET_SERVER_transmit_context_run (ii->tc, GNUNET_TIME_UNIT_MINUTES); + GNUNET_free (ii); + return; + } + ii->next = indexed_files; + indexed_files = ii; + write_index_list (); + GNUNET_SERVER_transmit_context_append_data (ii->tc, NULL, 0, + GNUNET_MESSAGE_TYPE_FS_INDEX_START_OK); + GNUNET_SERVER_transmit_context_run (ii->tc, GNUNET_TIME_UNIT_MINUTES); + ii->tc = NULL; +} + + +/** + * Function called once the hash computation over an + * indexed file has completed. + * + * @param cls closure, our publishing context + * @param res resulting hash, NULL on error + */ +static void +hash_for_index_val (void *cls, const GNUNET_HashCode * res) +{ + struct IndexInfo *ii = cls; + + ii->fhc = NULL; + if ((res == NULL) || + (0 != memcmp (res, &ii->file_id, sizeof (GNUNET_HashCode)))) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _ + ("Hash mismatch trying to index file `%s' which has hash `%s'\n"), + ii->filename, GNUNET_h2s (res)); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Wanted `%s'\n", + GNUNET_h2s (&ii->file_id)); + GNUNET_SERVER_transmit_context_append_data (ii->tc, NULL, 0, + GNUNET_MESSAGE_TYPE_FS_INDEX_START_FAILED); + GNUNET_SERVER_transmit_context_run (ii->tc, GNUNET_TIME_UNIT_MINUTES); + GNUNET_free (ii); + return; + } + signal_index_ok (ii); +} + + +/** + * Handle INDEX_START-message. + * + * @param cls closure + * @param client identification of the client + * @param message the actual message + */ +void +GNUNET_FS_handle_index_start (void *cls, struct GNUNET_SERVER_Client *client, + const struct GNUNET_MessageHeader *message) +{ + const struct IndexStartMessage *ism; + char *fn; + uint16_t msize; + struct IndexInfo *ii; + size_t slen; + uint64_t dev; + uint64_t ino; + uint64_t mydev; + uint64_t myino; + + msize = ntohs (message->size); + if ((msize <= sizeof (struct IndexStartMessage)) || + (((const char *) message)[msize - 1] != '\0')) + { + GNUNET_break (0); + GNUNET_SERVER_receive_done (client, GNUNET_SYSERR); + return; + } + ism = (const struct IndexStartMessage *) message; + if (0 != ism->reserved) + { + GNUNET_break (0); + GNUNET_SERVER_receive_done (client, GNUNET_SYSERR); + return; + } + fn = GNUNET_STRINGS_filename_expand ((const char *) &ism[1]); + if (fn == NULL) + { + GNUNET_SERVER_receive_done (client, GNUNET_SYSERR); + return; + } + dev = GNUNET_ntohll (ism->device); + ino = GNUNET_ntohll (ism->inode); + ism = (const struct IndexStartMessage *) message; + slen = strlen (fn) + 1; + ii = GNUNET_malloc (sizeof (struct IndexInfo) + slen); + ii->filename = (const char *) &ii[1]; + memcpy (&ii[1], fn, slen); + ii->file_id = ism->file_id; + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Received `%s' message for file `%s'\n", + "START_INDEX", ii->filename); + ii->tc = GNUNET_SERVER_transmit_context_create (client); + mydev = 0; + myino = 0; + if (((dev != 0) || (ino != 0)) && + (GNUNET_OK == GNUNET_DISK_file_get_identifiers (fn, &mydev, &myino)) && + ((dev == mydev) && (ino == myino))) + { + /* fast validation OK! */ + signal_index_ok (ii); + GNUNET_free (fn); + return; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Mismatch in file identifiers (%llu != %llu or %u != %u), need to hash.\n", + (unsigned long long) ino, (unsigned long long) myino, + (unsigned int) dev, (unsigned int) mydev); + /* slow validation, need to hash full file (again) */ + ii->fhc = + GNUNET_CRYPTO_hash_file (GNUNET_SCHEDULER_PRIORITY_IDLE, fn, + HASHING_BLOCKSIZE, &hash_for_index_val, ii); + if (ii->fhc == NULL) + hash_for_index_val (ii, NULL); + GNUNET_free (fn); +} + + +/** + * Handle INDEX_LIST_GET-message. + * + * @param cls closure + * @param client identification of the client + * @param message the actual message + */ +void +GNUNET_FS_handle_index_list_get (void *cls, struct GNUNET_SERVER_Client *client, + const struct GNUNET_MessageHeader *message) +{ + struct GNUNET_SERVER_TransmitContext *tc; + struct IndexInfoMessage *iim; + char buf[GNUNET_SERVER_MAX_MESSAGE_SIZE - 1]; + size_t slen; + const char *fn; + struct IndexInfo *pos; + + tc = GNUNET_SERVER_transmit_context_create (client); + iim = (struct IndexInfoMessage *) buf; + pos = indexed_files; + while (NULL != pos) + { + fn = pos->filename; + slen = strlen (fn) + 1; + if (slen + sizeof (struct IndexInfoMessage) >= + GNUNET_SERVER_MAX_MESSAGE_SIZE) + { + GNUNET_break (0); + break; + } + iim->header.type = htons (GNUNET_MESSAGE_TYPE_FS_INDEX_LIST_ENTRY); + iim->header.size = htons (slen + sizeof (struct IndexInfoMessage)); + iim->reserved = 0; + iim->file_id = pos->file_id; + memcpy (&iim[1], fn, slen); + GNUNET_SERVER_transmit_context_append_message (tc, &iim->header); + pos = pos->next; + } + GNUNET_SERVER_transmit_context_append_data (tc, NULL, 0, + GNUNET_MESSAGE_TYPE_FS_INDEX_LIST_END); + GNUNET_SERVER_transmit_context_run (tc, GNUNET_TIME_UNIT_MINUTES); +} + + +/** + * Handle UNINDEX-message. + * + * @param cls closure + * @param client identification of the client + * @param message the actual message + */ +void +GNUNET_FS_handle_unindex (void *cls, struct GNUNET_SERVER_Client *client, + const struct GNUNET_MessageHeader *message) +{ + const struct UnindexMessage *um; + struct IndexInfo *pos; + struct IndexInfo *prev; + struct IndexInfo *next; + struct GNUNET_SERVER_TransmitContext *tc; + int found; + + um = (const struct UnindexMessage *) message; + if (0 != um->reserved) + { + GNUNET_break (0); + GNUNET_SERVER_receive_done (client, GNUNET_SYSERR); + return; + } + found = GNUNET_NO; + prev = NULL; + pos = indexed_files; + while (NULL != pos) + { + next = pos->next; + if (0 == memcmp (&pos->file_id, &um->file_id, sizeof (GNUNET_HashCode))) + { + if (prev == NULL) + indexed_files = next; + else + prev->next = next; + GNUNET_break (GNUNET_OK == + GNUNET_CONTAINER_multihashmap_remove (ifm, &pos->file_id, + (void *) + pos->filename)); + GNUNET_free (pos); + found = GNUNET_YES; + } + else + { + prev = pos; + } + pos = next; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Client requested unindexing of file `%s': %s\n", + GNUNET_h2s (&um->file_id), found ? "found" : "not found"); + if (GNUNET_YES == found) + write_index_list (); + tc = GNUNET_SERVER_transmit_context_create (client); + GNUNET_SERVER_transmit_context_append_data (tc, NULL, 0, + GNUNET_MESSAGE_TYPE_FS_UNINDEX_OK); + GNUNET_SERVER_transmit_context_run (tc, GNUNET_TIME_UNIT_MINUTES); +} + + +/** + * Continuation called from datastore's remove + * function. + * + * @param cls unused + * @param success did the deletion work? + * @param min_expiration minimum expiration time required for content to be stored + * @param msg error message + */ +static void +remove_cont (void *cls, int success, + struct GNUNET_TIME_Absolute min_expiration, + const char *msg) +{ + if (GNUNET_OK != success) + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _("Failed to delete bogus block: %s\n"), msg); +} + + +/** + * We've received an on-demand encoded block from the datastore. + * Attempt to do on-demand encoding and (if successful), call the + * continuation with the resulting block. On error, clean up and ask + * the datastore for more results. + * + * @param key key for the content + * @param size number of bytes in data + * @param data content stored + * @param type type of the content + * @param priority priority of the content + * @param anonymity anonymity-level for the content + * @param expiration expiration time for the content + * @param uid unique identifier for the datum; + * maybe 0 if no unique identifier is available + * @param cont function to call with the actual block (at most once, on success) + * @param cont_cls closure for cont + * @return GNUNET_OK on success + */ +int +GNUNET_FS_handle_on_demand_block (const GNUNET_HashCode * key, uint32_t size, + const void *data, enum GNUNET_BLOCK_Type type, + uint32_t priority, uint32_t anonymity, + struct GNUNET_TIME_Absolute expiration, + uint64_t uid, + GNUNET_DATASTORE_DatumProcessor cont, + void *cont_cls) +{ + const struct OnDemandBlock *odb; + GNUNET_HashCode nkey; + struct GNUNET_CRYPTO_AesSessionKey skey; + struct GNUNET_CRYPTO_AesInitializationVector iv; + GNUNET_HashCode query; + ssize_t nsize; + char ndata[DBLOCK_SIZE]; + char edata[DBLOCK_SIZE]; + const char *fn; + struct GNUNET_DISK_FileHandle *fh; + uint64_t off; + + if (size != sizeof (struct OnDemandBlock)) + { + GNUNET_break (0); + GNUNET_DATASTORE_remove (dsh, key, size, data, -1, -1, + GNUNET_TIME_UNIT_FOREVER_REL, &remove_cont, NULL); + return GNUNET_SYSERR; + } + odb = (const struct OnDemandBlock *) data; + off = GNUNET_ntohll (odb->offset); + fn = (const char *) GNUNET_CONTAINER_multihashmap_get (ifm, &odb->file_id); + if ((NULL == fn) || (0 != ACCESS (fn, R_OK))) + { + GNUNET_STATISTICS_update (GSF_stats, + gettext_noop + ("# index blocks removed: original file inaccessible"), + 1, GNUNET_YES); + GNUNET_DATASTORE_remove (dsh, key, size, data, -1, -1, + GNUNET_TIME_UNIT_FOREVER_REL, &remove_cont, NULL); + return GNUNET_SYSERR; + } + if ((NULL == + (fh = + GNUNET_DISK_file_open (fn, GNUNET_DISK_OPEN_READ, + GNUNET_DISK_PERM_NONE))) || + (off != GNUNET_DISK_file_seek (fh, off, GNUNET_DISK_SEEK_SET)) || + (-1 == (nsize = GNUNET_DISK_file_read (fh, ndata, sizeof (ndata))))) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _ + ("Could not access indexed file `%s' (%s) at offset %llu: %s\n"), + GNUNET_h2s (&odb->file_id), fn, (unsigned long long) off, + (fn == NULL) ? _("not indexed") : STRERROR (errno)); + if (fh != NULL) + GNUNET_DISK_file_close (fh); + GNUNET_DATASTORE_remove (dsh, key, size, data, -1, -1, + GNUNET_TIME_UNIT_FOREVER_REL, &remove_cont, NULL); + return GNUNET_SYSERR; + } + GNUNET_DISK_file_close (fh); + GNUNET_CRYPTO_hash (ndata, nsize, &nkey); + GNUNET_CRYPTO_hash_to_aes_key (&nkey, &skey, &iv); + GNUNET_CRYPTO_aes_encrypt (ndata, nsize, &skey, &iv, edata); + GNUNET_CRYPTO_hash (edata, nsize, &query); + if (0 != memcmp (&query, key, sizeof (GNUNET_HashCode))) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _("Indexed file `%s' changed at offset %llu\n"), fn, + (unsigned long long) off); + GNUNET_DATASTORE_remove (dsh, key, size, data, -1, -1, + GNUNET_TIME_UNIT_FOREVER_REL, &remove_cont, NULL); + return GNUNET_SYSERR; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "On-demand encoded block for query `%s'\n", GNUNET_h2s (key)); + cont (cont_cls, key, nsize, edata, GNUNET_BLOCK_TYPE_FS_DBLOCK, priority, + anonymity, expiration, uid); + return GNUNET_OK; +} + + +/** + * Shutdown the module. + */ +void +GNUNET_FS_indexing_done () +{ + struct IndexInfo *pos; + + GNUNET_CONTAINER_multihashmap_destroy (ifm); + ifm = NULL; + while (NULL != (pos = indexed_files)) + { + indexed_files = pos->next; + if (pos->fhc != NULL) + GNUNET_CRYPTO_hash_file_cancel (pos->fhc); + GNUNET_free (pos); + } + cfg = NULL; +} + + +/** + * Initialize the indexing submodule. + * + * @param c configuration to use + * @param d datastore to use + */ +int +GNUNET_FS_indexing_init (const struct GNUNET_CONFIGURATION_Handle *c, + struct GNUNET_DATASTORE_Handle *d) +{ + cfg = c; + dsh = d; + ifm = GNUNET_CONTAINER_multihashmap_create (128); + read_index_list (); + return GNUNET_OK; +} + +/* end of gnunet-service-fs_indexing.c */ diff --git a/src/fs/gnunet-service-fs_indexing.h b/src/fs/gnunet-service-fs_indexing.h new file mode 100644 index 0000000..4295b20 --- /dev/null +++ b/src/fs/gnunet-service-fs_indexing.h @@ -0,0 +1,121 @@ +/* + This file is part of GNUnet. + (C) 2009, 2010, 2011 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/gnunet-service-fs_indexing.h + * @brief indexing for the file-sharing service + * @author Christian Grothoff + */ +#ifndef GNUNET_SERVICE_FS_INDEXING_H +#define GNUNET_SERVICE_FS_INDEXING_H + +#include "gnunet_block_lib.h" +#include "gnunet_core_service.h" +#include "gnunet_datastore_service.h" +#include "gnunet_peer_lib.h" +#include "gnunet_protocols.h" +#include "gnunet_signatures.h" +#include "gnunet_util_lib.h" + + +/** + * We've received an on-demand encoded block from the datastore. + * Attempt to do on-demand encoding and (if successful), call the + * continuation with the resulting block. On error, clean up and ask + * the datastore for more results. + * + * @param key key for the content + * @param size number of bytes in data + * @param data content stored + * @param type type of the content + * @param priority priority of the content + * @param anonymity anonymity-level for the content + * @param expiration expiration time for the content + * @param uid unique identifier for the datum; + * maybe 0 if no unique identifier is available + * @param cont function to call with the actual block (at most once, on success) + * @param cont_cls closure for cont + * @return GNUNET_OK on success + */ +int +GNUNET_FS_handle_on_demand_block (const GNUNET_HashCode * key, uint32_t size, + const void *data, enum GNUNET_BLOCK_Type type, + uint32_t priority, uint32_t anonymity, + struct GNUNET_TIME_Absolute expiration, + uint64_t uid, + GNUNET_DATASTORE_DatumProcessor cont, + void *cont_cls); + +/** + * Handle INDEX_START-message. + * + * @param cls closure + * @param client identification of the client + * @param message the actual message + */ +void +GNUNET_FS_handle_index_start (void *cls, struct GNUNET_SERVER_Client *client, + const struct GNUNET_MessageHeader *message); + + +/** + * Handle INDEX_LIST_GET-message. + * + * @param cls closure + * @param client identification of the client + * @param message the actual message + */ +void +GNUNET_FS_handle_index_list_get (void *cls, struct GNUNET_SERVER_Client *client, + const struct GNUNET_MessageHeader *message); + + +/** + * Handle UNINDEX-message. + * + * @param cls closure + * @param client identification of the client + * @param message the actual message + */ +void +GNUNET_FS_handle_unindex (void *cls, struct GNUNET_SERVER_Client *client, + const struct GNUNET_MessageHeader *message); + + +/** + * Initialize the indexing submodule. + * + * @param c configuration to use + * @param d datastore to use + * @return GNUNET_OK on success + */ +int +GNUNET_FS_indexing_init (const struct GNUNET_CONFIGURATION_Handle *c, + struct GNUNET_DATASTORE_Handle *d); + + +/** + * Shutdown the module. + */ +void +GNUNET_FS_indexing_done (void); + + +#endif diff --git a/src/fs/gnunet-service-fs_lc.c b/src/fs/gnunet-service-fs_lc.c new file mode 100644 index 0000000..36aafdd --- /dev/null +++ b/src/fs/gnunet-service-fs_lc.c @@ -0,0 +1,510 @@ +/* + This file is part of GNUnet. + (C) 2011 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/gnunet-service-fs_lc.c + * @brief API to handle 'local clients' + * @author Christian Grothoff + */ + +#include "platform.h" +#include "gnunet-service-fs.h" +#include "gnunet-service-fs_lc.h" +#include "gnunet-service-fs_cp.h" +#include "gnunet-service-fs_pr.h" + + +/** + * Doubly-linked list of requests we are performing + * on behalf of the same client. + */ +struct ClientRequest +{ + + /** + * This is a doubly-linked list. + */ + struct ClientRequest *next; + + /** + * This is a doubly-linked list. + */ + struct ClientRequest *prev; + + /** + * Request this entry represents. + */ + struct GSF_PendingRequest *pr; + + /** + * Client list this request belongs to. + */ + struct GSF_LocalClient *lc; + + /** + * Task scheduled to destroy the request. + */ + GNUNET_SCHEDULER_TaskIdentifier kill_task; + +}; + + +/** + * Replies to be transmitted to the client. The actual + * response message is allocated after this struct. + */ +struct ClientResponse +{ + /** + * This is a doubly-linked list. + */ + struct ClientResponse *next; + + /** + * This is a doubly-linked list. + */ + struct ClientResponse *prev; + + /** + * Client list entry this response belongs to. + */ + struct GSF_LocalClient *lc; + + /** + * Number of bytes in the response. + */ + size_t msize; +}; + + +/** + * A local client. + */ +struct GSF_LocalClient +{ + + /** + * We keep clients in a DLL. + */ + struct GSF_LocalClient *next; + + /** + * We keep clients in a DLL. + */ + struct GSF_LocalClient *prev; + + /** + * ID of the client. + */ + struct GNUNET_SERVER_Client *client; + + /** + * Head of list of requests performed on behalf + * of this client right now. + */ + struct ClientRequest *cr_head; + + /** + * Tail of list of requests performed on behalf + * of this client right now. + */ + struct ClientRequest *cr_tail; + + /** + * Head of linked list of responses. + */ + struct ClientResponse *res_head; + + /** + * Tail of linked list of responses. + */ + struct ClientResponse *res_tail; + + /** + * Context for sending replies. + */ + struct GNUNET_CONNECTION_TransmitHandle *th; + +}; + + +/** + * Head of linked list of our local clients. + */ +static struct GSF_LocalClient *client_head; + + +/** + * Head of linked list of our local clients. + */ +static struct GSF_LocalClient *client_tail; + + +/** + * Look up a local client record or create one if it + * doesn't exist yet. + * + * @param client handle of the client + * @return handle to local client entry + */ +struct GSF_LocalClient * +GSF_local_client_lookup_ (struct GNUNET_SERVER_Client *client) +{ + struct GSF_LocalClient *pos; + + pos = client_head; + while ((pos != NULL) && (pos->client != client)) + pos = pos->next; + if (pos != NULL) + return pos; + pos = GNUNET_malloc (sizeof (struct GSF_LocalClient)); + pos->client = client; + GNUNET_CONTAINER_DLL_insert (client_head, client_tail, pos); + return pos; +} + + +/** + * Free the given client request. + * + * @param cls the client request to free + * @param tc task context + */ +static void +client_request_destroy (void *cls, + const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct ClientRequest *cr = cls; + struct GSF_LocalClient *lc; + + cr->kill_task = GNUNET_SCHEDULER_NO_TASK; + lc = cr->lc; + GNUNET_CONTAINER_DLL_remove (lc->cr_head, lc->cr_tail, cr); + GSF_pending_request_cancel_ (cr->pr, GNUNET_NO); + GNUNET_STATISTICS_update (GSF_stats, + gettext_noop ("# client searches active"), -1, + GNUNET_NO); + GNUNET_free (cr); +} + + +/** + * Handle a reply to a pending request. Also called if a request + * expires (then with data == NULL). The handler may be called + * many times (depending on the request type), but will not be + * called during or after a call to GSF_pending_request_cancel + * and will also not be called anymore after a call signalling + * expiration. + * + * @param cls user-specified closure + * @param eval evaluation of the result + * @param pr handle to the original pending request + * @param reply_anonymity_level anonymity level for the reply, UINT32_MAX for "unknown" + * @param expiration when does 'data' expire? + * @param last_transmission when was the last time we've tried to download this block? (FOREVER if unknown) + * @param type type of the block + * @param data response data, NULL on request expiration + * @param data_len number of bytes in data + */ +static void +client_response_handler (void *cls, enum GNUNET_BLOCK_EvaluationResult eval, + struct GSF_PendingRequest *pr, + uint32_t reply_anonymity_level, + struct GNUNET_TIME_Absolute expiration, + struct GNUNET_TIME_Absolute last_transmission, + enum GNUNET_BLOCK_Type type, const void *data, + size_t data_len) +{ + struct ClientRequest *cr = cls; + struct GSF_LocalClient *lc; + struct ClientPutMessage *pm; + const struct GSF_PendingRequestData *prd; + size_t msize; + + if (NULL == data) + { + /* ugh, request 'timed out' -- how can this be? */ + GNUNET_break (0); + return; + } + prd = GSF_pending_request_get_data_ (pr); + GNUNET_break (type != GNUNET_BLOCK_TYPE_ANY); + if ((prd->type != type) && (prd->type != GNUNET_BLOCK_TYPE_ANY)) + { + GNUNET_break (0); + return; + } + GNUNET_STATISTICS_update (GSF_stats, + gettext_noop + ("# replies received for local clients"), 1, + GNUNET_NO); + GNUNET_assert (pr == cr->pr); + lc = cr->lc; + msize = sizeof (struct ClientPutMessage) + data_len; + { + char buf[msize]; + + pm = (struct ClientPutMessage *) buf; + pm->header.type = htons (GNUNET_MESSAGE_TYPE_FS_PUT); + pm->header.size = htons (msize); + pm->type = htonl (type); + pm->expiration = GNUNET_TIME_absolute_hton (expiration); + pm->last_transmission = GNUNET_TIME_absolute_hton (last_transmission); + memcpy (&pm[1], data, data_len); + GSF_local_client_transmit_ (lc, &pm->header); + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Queued reply to query `%s' for local client\n", + GNUNET_h2s (&prd->query), (unsigned int) prd->type); + if (eval != GNUNET_BLOCK_EVALUATION_OK_LAST) + return; + if (GNUNET_SCHEDULER_NO_TASK != cr->kill_task) + cr->kill_task = GNUNET_SCHEDULER_add_now (&client_request_destroy, cr); +} + + +/** + * Handle START_SEARCH-message (search request from local client). + * Only responsible for creating the request entry itself and setting + * up reply callback and cancellation on client disconnect. Does NOT + * execute the actual request strategy (planning). + * + * @param client identification of the client + * @param message the actual message + * @param prptr where to store the pending request handle for the request + * @return GNUNET_YES to start local processing, + * GNUNET_NO to not (yet) start local processing, + * GNUNET_SYSERR on error + */ +int +GSF_local_client_start_search_handler_ (struct GNUNET_SERVER_Client *client, + const struct GNUNET_MessageHeader + *message, + struct GSF_PendingRequest **prptr) +{ + static GNUNET_HashCode all_zeros; + const struct SearchMessage *sm; + struct GSF_LocalClient *lc; + struct ClientRequest *cr; + struct GSF_PendingRequestData *prd; + uint16_t msize; + unsigned int sc; + enum GNUNET_BLOCK_Type type; + enum GSF_PendingRequestOptions options; + + msize = ntohs (message->size); + if ((msize < sizeof (struct SearchMessage)) || + (0 != (msize - sizeof (struct SearchMessage)) % sizeof (GNUNET_HashCode))) + { + GNUNET_break (0); + *prptr = NULL; + return GNUNET_SYSERR; + } + GNUNET_STATISTICS_update (GSF_stats, + gettext_noop ("# client searches received"), 1, + GNUNET_NO); + sc = (msize - sizeof (struct SearchMessage)) / sizeof (GNUNET_HashCode); + sm = (const struct SearchMessage *) message; + type = ntohl (sm->type); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Received request for `%s' of type %u from local client\n", + GNUNET_h2s (&sm->query), (unsigned int) type); + lc = GSF_local_client_lookup_ (client); + cr = NULL; + /* detect duplicate KBLOCK requests */ + if ((type == GNUNET_BLOCK_TYPE_FS_KBLOCK) || + (type == GNUNET_BLOCK_TYPE_FS_NBLOCK) || (type == GNUNET_BLOCK_TYPE_ANY)) + { + cr = lc->cr_head; + while (cr != NULL) + { + prd = GSF_pending_request_get_data_ (cr->pr); + /* only unify with queries that hae not yet started local processing + (SEARCH_MESSAGE_OPTION_CONTINUED was always set) and that have a + matching query and type */ + if ((GNUNET_YES != prd->has_started) && + (0 != memcmp (&prd->query, &sm->query, sizeof (GNUNET_HashCode))) && + (prd->type == type)) + break; + cr = cr->next; + } + } + if (cr != NULL) + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Have existing request, merging content-seen lists.\n"); + GSF_pending_request_update_ (cr->pr, (const GNUNET_HashCode *) &sm[1], sc); + GNUNET_STATISTICS_update (GSF_stats, + gettext_noop + ("# client searches updated (merged content seen list)"), + 1, GNUNET_NO); + } + else + { + GNUNET_STATISTICS_update (GSF_stats, + gettext_noop ("# client searches active"), 1, + GNUNET_NO); + cr = GNUNET_malloc (sizeof (struct ClientRequest)); + cr->lc = lc; + GNUNET_CONTAINER_DLL_insert (lc->cr_head, lc->cr_tail, cr); + options = GSF_PRO_LOCAL_REQUEST; + if (0 != (SEARCH_MESSAGE_OPTION_LOOPBACK_ONLY & ntohl (sm->options))) + options |= GSF_PRO_LOCAL_ONLY; + cr->pr = GSF_pending_request_create_ (options, type, &sm->query, (type == GNUNET_BLOCK_TYPE_FS_SBLOCK) ? &sm->target /* namespace */ + : NULL, + (0 != + memcmp (&sm->target, &all_zeros, + sizeof (GNUNET_HashCode))) + ? (const struct GNUNET_PeerIdentity *) + &sm->target : NULL, NULL, 0, + 0 /* bf */ , + ntohl (sm->anonymity_level), + 0 /* priority */ , + 0 /* ttl */ , + 0 /* sender PID */ , + 0 /* origin PID */ , + (const GNUNET_HashCode *) &sm[1], sc, + &client_response_handler, cr); + } + *prptr = cr->pr; + return (0 != + (SEARCH_MESSAGE_OPTION_CONTINUED & ntohl (sm->options))) ? GNUNET_NO : + GNUNET_YES; +} + + +/** + * Transmit the given message by copying it to the target buffer + * "buf". "buf" will be NULL and "size" zero if the socket was closed + * for writing in the meantime. In that case, do nothing + * (the disconnect or shutdown handler will take care of the rest). + * If we were able to transmit messages and there are still more + * pending, ask core again for further calls to this function. + * + * @param cls closure, pointer to the 'struct GSF_LocalClient' + * @param size number of bytes available in buf + * @param buf where the callee should write the message + * @return number of bytes written to buf + */ +static size_t +transmit_to_client (void *cls, size_t size, void *buf) +{ + struct GSF_LocalClient *lc = cls; + char *cbuf = buf; + struct ClientResponse *res; + size_t msize; + + lc->th = NULL; + if (NULL == buf) + return 0; + msize = 0; + while ((NULL != (res = lc->res_head)) && (res->msize <= size)) + { + memcpy (&cbuf[msize], &res[1], res->msize); + msize += res->msize; + size -= res->msize; + GNUNET_CONTAINER_DLL_remove (lc->res_head, lc->res_tail, res); + GNUNET_free (res); + } + if (NULL != res) + lc->th = + GNUNET_SERVER_notify_transmit_ready (lc->client, res->msize, + GNUNET_TIME_UNIT_FOREVER_REL, + &transmit_to_client, lc); + return msize; +} + + +/** + * Transmit a message to the given local client as soon as possible. + * If the client disconnects before transmission, the message is + * simply discarded. + * + * @param lc recipient + * @param msg message to transmit to client + */ +void +GSF_local_client_transmit_ (struct GSF_LocalClient *lc, + const struct GNUNET_MessageHeader *msg) +{ + struct ClientResponse *res; + size_t msize; + + msize = ntohs (msg->size); + res = GNUNET_malloc (sizeof (struct ClientResponse) + msize); + res->lc = lc; + res->msize = msize; + memcpy (&res[1], msg, msize); + GNUNET_CONTAINER_DLL_insert_tail (lc->res_head, lc->res_tail, res); + if (NULL == lc->th) + lc->th = + GNUNET_SERVER_notify_transmit_ready (lc->client, msize, + GNUNET_TIME_UNIT_FOREVER_REL, + &transmit_to_client, lc); +} + + +/** + * A client disconnected from us. Tear down the local client + * record. + * + * @param cls unused + * @param client handle of the client + */ +void +GSF_client_disconnect_handler_ (void *cls, struct GNUNET_SERVER_Client *client) +{ + struct GSF_LocalClient *pos; + struct ClientRequest *cr; + struct ClientResponse *res; + + pos = client_head; + while ((pos != NULL) && (pos->client != client)) + pos = pos->next; + if (pos == NULL) + return; + while (NULL != (cr = pos->cr_head)) + { + GNUNET_CONTAINER_DLL_remove (pos->cr_head, pos->cr_tail, cr); + GSF_pending_request_cancel_ (cr->pr, GNUNET_NO); + GNUNET_STATISTICS_update (GSF_stats, + gettext_noop ("# client searches active"), -1, + GNUNET_NO); + if (GNUNET_SCHEDULER_NO_TASK != cr->kill_task) + GNUNET_SCHEDULER_cancel (cr->kill_task); + GNUNET_free (cr); + } + while (NULL != (res = pos->res_head)) + { + GNUNET_CONTAINER_DLL_remove (pos->res_head, pos->res_tail, res); + GNUNET_free (res); + } + if (pos->th != NULL) + { + GNUNET_CONNECTION_notify_transmit_ready_cancel (pos->th); + pos->th = NULL; + } + GSF_handle_local_client_disconnect_ (pos); + GNUNET_CONTAINER_DLL_remove (client_head, client_tail, pos); + GNUNET_free (pos); +} + + +/* end of gnunet-service-fs_lc.c */ diff --git a/src/fs/gnunet-service-fs_lc.h b/src/fs/gnunet-service-fs_lc.h new file mode 100644 index 0000000..3bddb89 --- /dev/null +++ b/src/fs/gnunet-service-fs_lc.h @@ -0,0 +1,87 @@ +/* + This file is part of GNUnet. + (C) 2011 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/gnunet-service-fs_lc.h + * @brief API to handle 'local clients' + * @author Christian Grothoff + */ +#ifndef GNUNET_SERVICE_FS_LC_H +#define GNUNET_SERVICE_FS_LC_H + +#include "gnunet-service-fs.h" + + +/** + * Look up a local client record or create one if it + * doesn't exist yet. + * + * @param client handle of the client + * @return handle to local client entry + */ +struct GSF_LocalClient * +GSF_local_client_lookup_ (struct GNUNET_SERVER_Client *client); + + +/** + * Handle START_SEARCH-message (search request from local client). + * Only responsible for creating the request entry itself and setting + * up reply callback and cancellation on client disconnect. Does NOT + * execute the actual request strategy (planning). + * + * @param client identification of the client + * @param message the actual message + * @param prptr where to store the pending request handle for the request + * @return GNUNET_YES to start local processing, + * GNUNET_NO to not (yet) start local processing, + * GNUNET_SYSERR on error + */ +int +GSF_local_client_start_search_handler_ (struct GNUNET_SERVER_Client *client, + const struct GNUNET_MessageHeader + *message, + struct GSF_PendingRequest **prptr); + + +/** + * Transmit a message to the given local client as soon as possible. + * If the client disconnects before transmission, the message is + * simply discarded. + * + * @param lc recipient + * @param msg message to transmit to client + */ +void +GSF_local_client_transmit_ (struct GSF_LocalClient *lc, + const struct GNUNET_MessageHeader *msg); + + +/** + * A client disconnected from us. Tear down the local client record. + * + * @param cls unused + * @param client handle of the client + */ +void +GSF_client_disconnect_handler_ (void *cls, struct GNUNET_SERVER_Client *client); + + +#endif +/* end of gnunet-service-fs_lc.h */ diff --git a/src/fs/gnunet-service-fs_pe.c b/src/fs/gnunet-service-fs_pe.c new file mode 100644 index 0000000..71b0fc0 --- /dev/null +++ b/src/fs/gnunet-service-fs_pe.c @@ -0,0 +1,775 @@ +/* + This file is part of GNUnet. + (C) 2011 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/gnunet-service-fs_pe.c + * @brief API to manage query plan + * @author Christian Grothoff + */ +#include "platform.h" +#include "gnunet-service-fs.h" +#include "gnunet-service-fs_cp.h" +#include "gnunet-service-fs_pe.h" +#include "gnunet-service-fs_pr.h" + + +/** + * List of GSF_PendingRequests this request plan + * participates with. + */ +struct PendingRequestList; + +/** + * Transmission plan for a peer. + */ +struct PeerPlan; + + +/** + * DLL of request plans a particular pending request is + * involved with. + */ +struct GSF_RequestPlanReference +{ + + /** + * This is a doubly-linked list. + */ + struct GSF_RequestPlanReference *next; + + /** + * This is a doubly-linked list. + */ + struct GSF_RequestPlanReference *prev; + + /** + * Associated request plan. + */ + struct GSF_RequestPlan *rp; + + /** + * Corresponding PendingRequestList. + */ + struct PendingRequestList *prl; +}; + + +/** + * List of GSF_PendingRequests this request plan + * participates with. + */ +struct PendingRequestList +{ + + /** + * This is a doubly-linked list. + */ + struct PendingRequestList *next; + + /** + * This is a doubly-linked list. + */ + struct PendingRequestList *prev; + + /** + * Associated pending request. + */ + struct GSF_PendingRequest *pr; + + /** + * Corresponding GSF_RequestPlanReference. + */ + struct GSF_RequestPlanReference *rpr; + +}; + + +/** + * Information we keep per request per peer. This is a doubly-linked + * list (with head and tail in the 'struct GSF_PendingRequestData') + * with one entry in each heap of each 'struct PeerPlan'. Each + * entry tracks information relevant for this request and this peer. + */ +struct GSF_RequestPlan +{ + + /** + * This is a doubly-linked list. + */ + struct GSF_RequestPlan *next; + + /** + * This is a doubly-linked list. + */ + struct GSF_RequestPlan *prev; + + /** + * Heap node associated with this request and this peer. + */ + struct GNUNET_CONTAINER_HeapNode *hn; + + /** + * The transmission plan for a peer that this request is associated with. + */ + struct PeerPlan *pp; + + /** + * Head of list of associated pending requests. + */ + struct PendingRequestList *prl_head; + + /** + * Tail of list of associated pending requests. + */ + struct PendingRequestList *prl_tail; + + /** + * Earliest time we'd be happy to (re)transmit this request. + */ + struct GNUNET_TIME_Absolute earliest_transmission; + + /** + * When was the last time we transmitted this request to this peer? 0 for never. + */ + struct GNUNET_TIME_Absolute last_transmission; + + /** + * Current priority for this request for this target. + */ + uint64_t priority; + + /** + * How often did we transmit this request to this peer? + */ + unsigned int transmission_counter; + +}; + + +/** + * Transmission plan for a peer. + */ +struct PeerPlan +{ + /** + * Heap with pending queries (struct GSF_RequestPlan), higher weights mean higher priority. + */ + struct GNUNET_CONTAINER_Heap *priority_heap; + + /** + * Heap with pending queries (struct GSF_RequestPlan), by transmission time, lowest first. + */ + struct GNUNET_CONTAINER_Heap *delay_heap; + + /** + * Map of queries to plan entries. All entries in the priority_heap or delay_heap + * should be in the plan map. Note that it IS possible for the plan map to have + * multiple entries for the same query. + */ + struct GNUNET_CONTAINER_MultiHashMap *plan_map; + + /** + * Current transmission request handle. + */ + struct GSF_PeerTransmitHandle *pth; + + /** + * Peer for which this is the plan. + */ + struct GSF_ConnectedPeer *cp; + + /** + * Current task for executing the plan. + */ + GNUNET_SCHEDULER_TaskIdentifier task; +}; + + +/** + * Hash map from peer identities to PeerPlans. + */ +static struct GNUNET_CONTAINER_MultiHashMap *plans; + +/** + * Sum of all transmission counters (equals total delay for all plan entries). + */ +static unsigned long long total_delay; + +/** + * Number of plan entries. + */ +static unsigned long long plan_count; + + +/** + * Return the query (key in the plan_map) for the given request plan. + * + * @param rp a request plan + * @return the associated query + */ +static const GNUNET_HashCode * +get_rp_key (struct GSF_RequestPlan *rp) +{ + return &GSF_pending_request_get_data_ (rp->prl_head->pr)->query; +} + + +/** + * Figure out when and how to transmit to the given peer. + * + * @param cls the 'struct GSF_ConnectedPeer' for transmission + * @param tc scheduler context + */ +static void +schedule_peer_transmission (void *cls, + const struct GNUNET_SCHEDULER_TaskContext *tc); + + +/** + * Insert the given request plan into the heap with the appropriate weight. + * + * @param pp associated peer's plan + * @param rp request to plan + */ +static void +plan (struct PeerPlan *pp, struct GSF_RequestPlan *rp) +{ +#define N ((double)128.0) + /** + * Running average delay we currently impose. + */ + static double avg_delay; + + struct GSF_PendingRequestData *prd; + struct GNUNET_TIME_Relative delay; + + GNUNET_assert (rp->pp == pp); + GNUNET_STATISTICS_set (GSF_stats, + gettext_noop ("# average retransmission delay (ms)"), + total_delay * 1000LL / plan_count, GNUNET_NO); + prd = GSF_pending_request_get_data_ (rp->prl_head->pr); + + if (rp->transmission_counter < 8) + delay = + GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, + rp->transmission_counter); + else if (rp->transmission_counter < 32) + delay = + GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, + 8 + + (1LL << (rp->transmission_counter - 8))); + else + delay = + GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, + 8 + (1LL << 24)); + delay.rel_value = + GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, + delay.rel_value + 1); + /* Add 0.01 to avg_delay to avoid division-by-zero later */ + avg_delay = (((avg_delay * (N - 1.0)) + delay.rel_value) / N) + 0.01; + + /* + * For the priority, we need to consider a few basic rules: + * 1) if we just started requesting (delay is small), we should + * virtually always have a priority of zero. + * 2) for requests with average latency, our priority should match + * the average priority observed on the network + * 3) even the longest-running requests should not be WAY out of + * the observed average (thus we bound by a factor of 2) + * 4) we add +1 to the observed average priority to avoid everyone + * staying put at zero (2 * 0 = 0...). + * + * Using the specific calculation below, we get: + * + * delay = 0 => priority = 0; + * delay = avg delay => priority = running-average-observed-priority; + * delay >> avg_delay => priority = 2 * running-average-observed-priority; + * + * which satisfies all of the rules above. + * + * Note: M_PI_4 = PI/4 = arctan(1) + */ + rp->priority = + round ((GSF_current_priorities + + 1.0) * atan (delay.rel_value / avg_delay)) / M_PI_4; + /* Note: usage of 'round' and 'atan' requires -lm */ + + if (rp->transmission_counter != 0) + delay.rel_value += TTL_DECREMENT; + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Considering (re)transmission number %u in %llu ms\n", + (unsigned int) rp->transmission_counter, + (unsigned long long) delay.rel_value); + rp->earliest_transmission = GNUNET_TIME_relative_to_absolute (delay); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Earliest (re)transmission for `%s' in %us\n", + GNUNET_h2s (&prd->query), rp->transmission_counter); + GNUNET_assert (rp->hn == NULL); + if (GNUNET_TIME_absolute_get_remaining (rp->earliest_transmission).rel_value + == 0) + rp->hn = GNUNET_CONTAINER_heap_insert (pp->priority_heap, rp, rp->priority); + else + rp->hn = + GNUNET_CONTAINER_heap_insert (pp->delay_heap, rp, + rp->earliest_transmission.abs_value); + GNUNET_assert (GNUNET_YES == + GNUNET_CONTAINER_multihashmap_contains_value (pp->plan_map, + get_rp_key (rp), + rp)); + if (GNUNET_SCHEDULER_NO_TASK != pp->task) + GNUNET_SCHEDULER_cancel (pp->task); + pp->task = GNUNET_SCHEDULER_add_now (&schedule_peer_transmission, pp); +#undef N +} + + +/** + * Get the pending request with the highest TTL from the given plan. + * + * @param rp plan to investigate + * @return pending request with highest TTL + */ +struct GSF_PendingRequest * +get_latest (const struct GSF_RequestPlan *rp) +{ + struct GSF_PendingRequest *ret; + struct PendingRequestList *prl; + + prl = rp->prl_head; + ret = prl->pr; + prl = prl->next; + while (NULL != prl) + { + if (GSF_pending_request_get_data_ (prl->pr)->ttl.abs_value > + GSF_pending_request_get_data_ (ret)->ttl.abs_value) + ret = prl->pr; + prl = prl->next; + } + return ret; +} + + +/** + * Function called to get a message for transmission. + * + * @param cls closure + * @param buf_size number of bytes available in buf + * @param buf where to copy the message, NULL on error (peer disconnect) + * @return number of bytes copied to 'buf', can be 0 (without indicating an error) + */ +static size_t +transmit_message_callback (void *cls, size_t buf_size, void *buf) +{ + struct PeerPlan *pp = cls; + struct GSF_RequestPlan *rp; + size_t msize; + + pp->pth = NULL; + if (NULL == buf) + { + /* failed, try again... */ + pp->task = GNUNET_SCHEDULER_add_now (&schedule_peer_transmission, pp); + GNUNET_STATISTICS_update (GSF_stats, + gettext_noop + ("# transmission failed (core has no bandwidth)"), + 1, GNUNET_NO); + return 0; + } + rp = GNUNET_CONTAINER_heap_peek (pp->priority_heap); + if (NULL == rp) + { + pp->task = GNUNET_SCHEDULER_add_now (&schedule_peer_transmission, pp); + return 0; + } + msize = GSF_pending_request_get_message_ (get_latest (rp), buf_size, buf); + if (msize > buf_size) + { + /* buffer to small (message changed), try again */ + pp->task = GNUNET_SCHEDULER_add_now (&schedule_peer_transmission, pp); + return 0; + } + /* remove from root, add again elsewhere... */ + GNUNET_assert (rp == GNUNET_CONTAINER_heap_remove_root (pp->priority_heap)); + rp->hn = NULL; + rp->last_transmission = GNUNET_TIME_absolute_get (); + rp->transmission_counter++; + total_delay++; + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Executing plan %p executed %u times, planning retransmission\n", + rp, rp->transmission_counter); + plan (pp, rp); + GNUNET_STATISTICS_update (GSF_stats, + gettext_noop + ("# query messages sent to other peers"), 1, + GNUNET_NO); + return msize; +} + + +/** + * Figure out when and how to transmit to the given peer. + * + * @param cls the 'struct PeerPlan' + * @param tc scheduler context + */ +static void +schedule_peer_transmission (void *cls, + const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct PeerPlan *pp = cls; + struct GSF_RequestPlan *rp; + size_t msize; + struct GNUNET_TIME_Relative delay; + + pp->task = GNUNET_SCHEDULER_NO_TASK; + if (pp->pth != NULL) + { + GSF_peer_transmit_cancel_ (pp->pth); + pp->pth = NULL; + } + /* move ready requests to priority queue */ + while ((NULL != (rp = GNUNET_CONTAINER_heap_peek (pp->delay_heap))) && + (GNUNET_TIME_absolute_get_remaining + (rp->earliest_transmission).rel_value == 0)) + { + GNUNET_assert (rp == GNUNET_CONTAINER_heap_remove_root (pp->delay_heap)); + rp->hn = GNUNET_CONTAINER_heap_insert (pp->priority_heap, rp, rp->priority); + } + if (0 == GNUNET_CONTAINER_heap_get_size (pp->priority_heap)) + { + /* priority heap (still) empty, check for delay... */ + rp = GNUNET_CONTAINER_heap_peek (pp->delay_heap); + if (NULL == rp) + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "No active requests for plan %p.\n", + pp); + return; /* both queues empty */ + } + delay = GNUNET_TIME_absolute_get_remaining (rp->earliest_transmission); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Sleeping for %llu ms before retrying requests on plan %p.\n", + (unsigned long long) delay.rel_value, pp); + GNUNET_STATISTICS_set (GSF_stats, gettext_noop ("# delay heap timeout"), + delay.rel_value, GNUNET_NO); + + pp->task = + GNUNET_SCHEDULER_add_delayed (delay, &schedule_peer_transmission, pp); + return; + } + GNUNET_STATISTICS_update (GSF_stats, gettext_noop ("# query plans executed"), + 1, GNUNET_NO); + /* process from priority heap */ + rp = GNUNET_CONTAINER_heap_peek (pp->priority_heap); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Executing query plan %p\n", rp); + GNUNET_assert (NULL != rp); + msize = GSF_pending_request_get_message_ (get_latest (rp), 0, NULL); + pp->pth = + GSF_peer_transmit_ (pp->cp, GNUNET_YES, rp->priority, + GNUNET_TIME_UNIT_FOREVER_REL, msize, + &transmit_message_callback, pp); + GNUNET_assert (NULL != pp->pth); +} + + +/** + * Closure for 'merge_pr'. + */ +struct MergeContext +{ + + struct GSF_PendingRequest *pr; + + int merged; + +}; + + +/** + * Iterator that checks if an equivalent request is already + * present for this peer. + * + * @param cls closure + * @param query the query + * @param element request plan stored at the node + * @return GNUNET_YES if we should continue to iterate, + * GNUNET_NO if not (merge success) + */ +static int +merge_pr (void *cls, const GNUNET_HashCode * query, void *element) +{ + struct MergeContext *mpr = cls; + struct GSF_RequestPlan *rp = element; + struct GSF_PendingRequestData *prd; + struct GSF_RequestPlanReference *rpr; + struct PendingRequestList *prl; + struct GSF_PendingRequest *latest; + + if (GNUNET_OK != + GSF_pending_request_is_compatible_ (mpr->pr, rp->prl_head->pr)) + return GNUNET_YES; + /* merge new request with existing request plan */ + rpr = GNUNET_malloc (sizeof (struct GSF_RequestPlanReference)); + prl = GNUNET_malloc (sizeof (struct PendingRequestList)); + rpr->rp = rp; + rpr->prl = prl; + prl->rpr = rpr; + prl->pr = mpr->pr; + prd = GSF_pending_request_get_data_ (mpr->pr); + GNUNET_CONTAINER_DLL_insert (prd->rpr_head, prd->rpr_tail, rpr); + GNUNET_CONTAINER_DLL_insert (rp->prl_head, rp->prl_tail, prl); + mpr->merged = GNUNET_YES; + GNUNET_STATISTICS_update (GSF_stats, gettext_noop ("# requests merged"), 1, + GNUNET_NO); + latest = get_latest (rp); + if (GSF_pending_request_get_data_ (latest)->ttl.abs_value < + prd->ttl.abs_value) + { + GNUNET_STATISTICS_update (GSF_stats, gettext_noop ("# requests refreshed"), + 1, GNUNET_NO); + rp->transmission_counter = 0; /* reset */ + } + return GNUNET_NO; +} + + +/** + * Create a new query plan entry. + * + * @param cp peer with the entry + * @param pr request with the entry + */ +void +GSF_plan_add_ (struct GSF_ConnectedPeer *cp, struct GSF_PendingRequest *pr) +{ + struct GNUNET_PeerIdentity id; + struct PeerPlan *pp; + struct GSF_PendingRequestData *prd; + struct GSF_RequestPlan *rp; + struct GSF_RequestPlanReference *rpr; + struct PendingRequestList *prl; + struct MergeContext mpc; + + GNUNET_assert (NULL != cp); + GSF_connected_peer_get_identity_ (cp, &id); + pp = GNUNET_CONTAINER_multihashmap_get (plans, &id.hashPubKey); + if (NULL == pp) + { + pp = GNUNET_malloc (sizeof (struct PeerPlan)); + pp->plan_map = GNUNET_CONTAINER_multihashmap_create (128); + pp->priority_heap = + GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MAX); + pp->delay_heap = + GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN); + pp->cp = cp; + GNUNET_CONTAINER_multihashmap_put (plans, &id.hashPubKey, pp, + GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY); + } + mpc.merged = GNUNET_NO; + mpc.pr = pr; + GNUNET_CONTAINER_multihashmap_get_multiple (pp->plan_map, + &GSF_pending_request_get_data_ + (pr)->query, &merge_pr, &mpc); + if (mpc.merged != GNUNET_NO) + return; + GNUNET_CONTAINER_multihashmap_get_multiple (pp->plan_map, + &GSF_pending_request_get_data_ + (pr)->query, &merge_pr, &mpc); + if (mpc.merged != GNUNET_NO) + return; + plan_count++; + GNUNET_STATISTICS_update (GSF_stats, gettext_noop ("# query plan entries"), 1, + GNUNET_NO); + prd = GSF_pending_request_get_data_ (pr); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Planning transmission of query `%s' to peer `%s'\n", + GNUNET_h2s (&prd->query), GNUNET_i2s (&id)); + rp = GNUNET_malloc (sizeof (struct GSF_RequestPlan)); + rpr = GNUNET_malloc (sizeof (struct GSF_RequestPlanReference)); + prl = GNUNET_malloc (sizeof (struct PendingRequestList)); + rpr->rp = rp; + rpr->prl = prl; + prl->rpr = rpr; + prl->pr = pr; + GNUNET_CONTAINER_DLL_insert (prd->rpr_head, prd->rpr_tail, rpr); + GNUNET_CONTAINER_DLL_insert (rp->prl_head, rp->prl_tail, prl); + rp->pp = pp; + GNUNET_assert (GNUNET_YES == + GNUNET_CONTAINER_multihashmap_put (pp->plan_map, + get_rp_key (rp), rp, + GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE)); + plan (pp, rp); +} + + +/** + * Notify the plan about a peer being no longer available; + * destroy all entries associated with this peer. + * + * @param cp connected peer + */ +void +GSF_plan_notify_peer_disconnect_ (const struct GSF_ConnectedPeer *cp) +{ + struct GNUNET_PeerIdentity id; + struct PeerPlan *pp; + struct GSF_RequestPlan *rp; + struct GSF_PendingRequestData *prd; + struct PendingRequestList *prl; + + GSF_connected_peer_get_identity_ (cp, &id); + pp = GNUNET_CONTAINER_multihashmap_get (plans, &id.hashPubKey); + if (NULL == pp) + return; /* nothing was ever planned for this peer */ + GNUNET_assert (GNUNET_YES == + GNUNET_CONTAINER_multihashmap_remove (plans, &id.hashPubKey, + pp)); + if (NULL != pp->pth) + GSF_peer_transmit_cancel_ (pp->pth); + if (GNUNET_SCHEDULER_NO_TASK != pp->task) + { + GNUNET_SCHEDULER_cancel (pp->task); + pp->task = GNUNET_SCHEDULER_NO_TASK; + } + while (NULL != (rp = GNUNET_CONTAINER_heap_remove_root (pp->priority_heap))) + { + GNUNET_break (GNUNET_YES == + GNUNET_CONTAINER_multihashmap_remove (pp->plan_map, + get_rp_key (rp), rp)); + while (NULL != (prl = rp->prl_head)) + { + GNUNET_CONTAINER_DLL_remove (rp->prl_head, rp->prl_tail, prl); + prd = GSF_pending_request_get_data_ (prl->pr); + GNUNET_CONTAINER_DLL_remove (prd->rpr_head, prd->rpr_tail, prl->rpr); + GNUNET_free (prl->rpr); + GNUNET_free (prl); + } + GNUNET_free (rp); + } + GNUNET_CONTAINER_heap_destroy (pp->priority_heap); + while (NULL != (rp = GNUNET_CONTAINER_heap_remove_root (pp->delay_heap))) + { + GNUNET_break (GNUNET_YES == + GNUNET_CONTAINER_multihashmap_remove (pp->plan_map, + get_rp_key (rp), rp)); + while (NULL != (prl = rp->prl_head)) + { + GNUNET_CONTAINER_DLL_remove (rp->prl_head, rp->prl_tail, prl); + prd = GSF_pending_request_get_data_ (prl->pr); + GNUNET_CONTAINER_DLL_remove (prd->rpr_head, prd->rpr_tail, prl->rpr); + GNUNET_free (prl->rpr); + GNUNET_free (prl); + } + GNUNET_free (rp); + } + GNUNET_STATISTICS_set (GSF_stats, gettext_noop ("# query plan entries"), + plan_count, GNUNET_NO); + + GNUNET_CONTAINER_heap_destroy (pp->delay_heap); + GNUNET_CONTAINER_multihashmap_destroy (pp->plan_map); + GNUNET_free (pp); +} + +/** + * Get the last transmission attempt time for the request plan list + * referenced by 'rpr_head', that was sent to 'sender' + * + * @param rpr_head request plan reference list to check. + * @param sender the peer that we've sent the request to. + * @param result the timestamp to fill. + * @return GNUNET_YES if 'result' was changed, GNUNET_NO otherwise. + */ +int +GSF_request_plan_reference_get_last_transmission_ ( + struct GSF_RequestPlanReference *rpr_head, struct GSF_ConnectedPeer *sender, + struct GNUNET_TIME_Absolute *result) +{ + struct GSF_RequestPlanReference *rpr; + for (rpr = rpr_head; rpr; rpr = rpr->next) + { + if (rpr->rp->pp->cp == sender) + { + *result = rpr->rp->last_transmission; + return GNUNET_YES; + } + } + return GNUNET_NO; +} + +/** + * Notify the plan about a request being done; destroy all entries + * associated with this request. + * + * @param pr request that is done + */ +void +GSF_plan_notify_request_done_ (struct GSF_PendingRequest *pr) +{ + struct GSF_RequestPlan *rp; + struct GSF_PendingRequestData *prd; + struct GSF_RequestPlanReference *rpr; + + prd = GSF_pending_request_get_data_ (pr); + while (NULL != (rpr = prd->rpr_head)) + { + GNUNET_CONTAINER_DLL_remove (prd->rpr_head, prd->rpr_tail, rpr); + rp = rpr->rp; + GNUNET_CONTAINER_DLL_remove (rp->prl_head, rp->prl_tail, rpr->prl); + if (NULL == rp->prl_head) + { + GNUNET_CONTAINER_heap_remove_node (rp->hn); + plan_count--; + GNUNET_break (GNUNET_YES == + GNUNET_CONTAINER_multihashmap_remove (rp->pp->plan_map, + &GSF_pending_request_get_data_ + (rpr->prl->pr)->query, + rp)); + GNUNET_free (rp); + } + GNUNET_free (rpr->prl); + GNUNET_free (rpr); + } + GNUNET_STATISTICS_set (GSF_stats, gettext_noop ("# query plan entries"), + plan_count, GNUNET_NO); +} + + +/** + * Initialize plan subsystem. + */ +void +GSF_plan_init () +{ + plans = GNUNET_CONTAINER_multihashmap_create (256); +} + + +/** + * Shutdown plan subsystem. + */ +void +GSF_plan_done () +{ + GNUNET_assert (0 == GNUNET_CONTAINER_multihashmap_size (plans)); + GNUNET_CONTAINER_multihashmap_destroy (plans); +} + + + +/* end of gnunet-service-fs_pe.h */ diff --git a/src/fs/gnunet-service-fs_pe.h b/src/fs/gnunet-service-fs_pe.h new file mode 100644 index 0000000..3a18715 --- /dev/null +++ b/src/fs/gnunet-service-fs_pe.h @@ -0,0 +1,90 @@ +/* + This file is part of GNUnet. + (C) 2011 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/gnunet-service-fs_pe.h + * @brief API to manage query plan + * @author Christian Grothoff + */ +#ifndef GNUNET_SERVICE_FS_PE_H +#define GNUNET_SERVICE_FS_PE_H + +#include "gnunet-service-fs.h" + + +/** + * Create a new query plan entry. + * + * @param cp peer with the entry + * @param pr request with the entry + */ +void +GSF_plan_add_ (struct GSF_ConnectedPeer *cp, struct GSF_PendingRequest *pr); + + +/** + * Notify the plan about a peer being no longer available; + * destroy all entries associated with this peer. + * + * @param cp connected peer + */ +void +GSF_plan_notify_peer_disconnect_ (const struct GSF_ConnectedPeer *cp); + + +/** + * Notify the plan about a request being done; + * destroy all entries associated with this request. + * + * @param pr request that is done + */ +void +GSF_plan_notify_request_done_ (struct GSF_PendingRequest *pr); + +/** + * Get the last transmission attempt time for the request plan list + * referenced by 'rpr_head', that was sent to 'sender' + * + * @param rpr_head request plan reference list to check. + * @param sender the peer that we've sent the request to. + * @param result the timestamp to fill. + * @return GNUNET_YES if 'result' was changed, GNUNET_NO otherwise. + */ +int +GSF_request_plan_reference_get_last_transmission_ ( + struct GSF_RequestPlanReference *rpr_head, struct GSF_ConnectedPeer *sender, + struct GNUNET_TIME_Absolute *result); + +/** + * Initialize plan subsystem. + */ +void +GSF_plan_init (void); + + +/** + * Shutdown plan subsystem. + */ +void +GSF_plan_done (void); + + +#endif +/* end of gnunet-service-fs_pe.h */ diff --git a/src/fs/gnunet-service-fs_pr.c b/src/fs/gnunet-service-fs_pr.c new file mode 100644 index 0000000..d4b4481 --- /dev/null +++ b/src/fs/gnunet-service-fs_pr.c @@ -0,0 +1,1644 @@ +/* + This file is part of GNUnet. + (C) 2009, 2010, 2011 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/gnunet-service-fs_pr.c + * @brief API to handle pending requests + * @author Christian Grothoff + */ +#include "platform.h" +#include "gnunet_load_lib.h" +#include "gnunet-service-fs.h" +#include "gnunet-service-fs_cp.h" +#include "gnunet-service-fs_indexing.h" +#include "gnunet-service-fs_pe.h" +#include "gnunet-service-fs_pr.h" + +/** + * Maximum size of the datastore queue for P2P operations. Needs to + * be large enough to queue MAX_QUEUE_PER_PEER operations for roughly + * the number of active (connected) peers. + */ +#define MAX_DATASTORE_QUEUE (16 * MAX_QUEUE_PER_PEER) + +/** + * Bandwidth value of a 0-priority content (must be fairly high + * compared to query since content is typically significantly larger + * -- and more valueable since it can take many queries to get one + * piece of content). + */ +#define CONTENT_BANDWIDTH_VALUE 800 + +/** + * Hard limit on the number of results we may get from the datastore per query. + */ +#define MAX_RESULTS (100 * 1024) + +/** + * An active request. + */ +struct GSF_PendingRequest +{ + /** + * Public data for the request. + */ + struct GSF_PendingRequestData public_data; + + /** + * Function to call if we encounter a reply. + */ + GSF_PendingRequestReplyHandler rh; + + /** + * Closure for 'rh' + */ + void *rh_cls; + + /** + * Array of hash codes of replies we've already seen. + */ + GNUNET_HashCode *replies_seen; + + /** + * Bloomfilter masking replies we've already seen. + */ + struct GNUNET_CONTAINER_BloomFilter *bf; + + /** + * Entry for this pending request in the expiration heap, or NULL. + */ + struct GNUNET_CONTAINER_HeapNode *hnode; + + /** + * Datastore queue entry for this request (or NULL for none). + */ + struct GNUNET_DATASTORE_QueueEntry *qe; + + /** + * DHT request handle for this request (or NULL for none). + */ + struct GNUNET_DHT_GetHandle *gh; + + /** + * Function to call upon completion of the local get + * request, or NULL for none. + */ + GSF_LocalLookupContinuation llc_cont; + + /** + * Closure for llc_cont. + */ + void *llc_cont_cls; + + /** + * Last result from the local datastore lookup evaluation. + */ + enum GNUNET_BLOCK_EvaluationResult local_result; + + /** + * Identity of the peer that we should use for the 'sender' + * (recipient of the response) when forwarding (0 for none). + */ + GNUNET_PEER_Id sender_pid; + + /** + * Identity of the peer that we should never forward this query + * to since it originated this query (0 for none). + */ + GNUNET_PEER_Id origin_pid; + + /** + * Time we started the last datastore lookup. + */ + struct GNUNET_TIME_Absolute qe_start; + + /** + * Task that warns us if the local datastore lookup takes too long. + */ + GNUNET_SCHEDULER_TaskIdentifier warn_task; + + /** + * Current offset for querying our local datastore for results. + * Starts at a random value, incremented until we get the same + * UID again (detected using 'first_uid'), which is then used + * to termiante the iteration. + */ + uint64_t local_result_offset; + + /** + * Unique ID of the first result from the local datastore; + * used to detect wrap-around of the offset. + */ + uint64_t first_uid; + + /** + * Number of valid entries in the 'replies_seen' array. + */ + unsigned int replies_seen_count; + + /** + * Length of the 'replies_seen' array. + */ + unsigned int replies_seen_size; + + /** + * Mingle value we currently use for the bf. + */ + uint32_t mingle; + + /** + * Do we have a first UID yet? + */ + unsigned int have_first_uid; + +}; + + +/** + * All pending requests, ordered by the query. Entries + * are of type 'struct GSF_PendingRequest*'. + */ +static struct GNUNET_CONTAINER_MultiHashMap *pr_map; + + +/** + * Datastore 'PUT' load tracking. + */ +static struct GNUNET_LOAD_Value *datastore_put_load; + + +/** + * Are we allowed to migrate content to this peer. + */ +static int active_to_migration; + + +/** + * Size of the datastore queue we assume for common requests. + * Determined based on the network quota. + */ +static unsigned int datastore_queue_size; + +/** + * Heap with the request that will expire next at the top. Contains + * pointers of type "struct PendingRequest*"; these will *also* be + * aliased from the "requests_by_peer" data structures and the + * "requests_by_query" table. Note that requests from our clients + * don't expire and are thus NOT in the "requests_by_expiration" + * (or the "requests_by_peer" tables). + */ +static struct GNUNET_CONTAINER_Heap *requests_by_expiration_heap; + + +/** + * Maximum number of requests (from other peers, overall) that we're + * willing to have pending at any given point in time. Can be changed + * via the configuration file (32k is just the default). + */ +static unsigned long long max_pending_requests = (32 * 1024); + + + +/** + * Recalculate our bloom filter for filtering replies. This function + * will create a new bloom filter from scratch, so it should only be + * called if we have no bloomfilter at all (and hence can create a + * fresh one of minimal size without problems) OR if our peer is the + * initiator (in which case we may resize to larger than mimimum size). + * + * @param pr request for which the BF is to be recomputed + */ +static void +refresh_bloomfilter (struct GSF_PendingRequest *pr) +{ + if (pr->bf != NULL) + GNUNET_CONTAINER_bloomfilter_free (pr->bf); + pr->mingle = + GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, UINT32_MAX); + pr->bf = + GNUNET_BLOCK_construct_bloomfilter (pr->mingle, pr->replies_seen, + pr->replies_seen_count); +} + + +/** + * Create a new pending request. + * + * @param options request options + * @param type type of the block that is being requested + * @param query key for the lookup + * @param namespace namespace to lookup, NULL for no namespace + * @param target preferred target for the request, NULL for none + * @param bf_data raw data for bloom filter for known replies, can be NULL + * @param bf_size number of bytes in bf_data + * @param mingle mingle value for bf + * @param anonymity_level desired anonymity level + * @param priority maximum outgoing cummulative request priority to use + * @param ttl current time-to-live for the request + * @param sender_pid peer ID to use for the sender when forwarding, 0 for none + * @param origin_pid peer ID of origin of query (do not loop back) + * @param replies_seen hash codes of known local replies + * @param replies_seen_count size of the 'replies_seen' array + * @param rh handle to call when we get a reply + * @param rh_cls closure for rh + * @return handle for the new pending request + */ +struct GSF_PendingRequest * +GSF_pending_request_create_ (enum GSF_PendingRequestOptions options, + enum GNUNET_BLOCK_Type type, + const GNUNET_HashCode * query, + const GNUNET_HashCode * namespace, + const struct GNUNET_PeerIdentity *target, + const char *bf_data, size_t bf_size, + uint32_t mingle, uint32_t anonymity_level, + uint32_t priority, int32_t ttl, + GNUNET_PEER_Id sender_pid, + GNUNET_PEER_Id origin_pid, + const GNUNET_HashCode * replies_seen, + unsigned int replies_seen_count, + GSF_PendingRequestReplyHandler rh, void *rh_cls) +{ + struct GSF_PendingRequest *pr; + struct GSF_PendingRequest *dpr; + + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Creating request handle for `%s' of type %d\n", + GNUNET_h2s (query), type); + GNUNET_STATISTICS_update (GSF_stats, + gettext_noop ("# Pending requests created"), 1, + GNUNET_NO); + pr = GNUNET_malloc (sizeof (struct GSF_PendingRequest)); + pr->local_result_offset = + GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK, UINT64_MAX); + pr->public_data.query = *query; + if (GNUNET_BLOCK_TYPE_FS_SBLOCK == type) + { + GNUNET_assert (NULL != namespace); + pr->public_data.namespace = *namespace; + } + if (NULL != target) + { + pr->public_data.target = *target; + pr->public_data.has_target = GNUNET_YES; + } + pr->public_data.anonymity_level = anonymity_level; + pr->public_data.priority = priority; + pr->public_data.original_priority = priority; + pr->public_data.options = options; + pr->public_data.type = type; + pr->public_data.start_time = GNUNET_TIME_absolute_get (); + pr->sender_pid = sender_pid; + pr->origin_pid = origin_pid; + pr->rh = rh; + pr->rh_cls = rh_cls; + GNUNET_assert ((sender_pid != 0) || (0 == (options & GSF_PRO_FORWARD_ONLY))); + if (ttl >= 0) + pr->public_data.ttl = + GNUNET_TIME_relative_to_absolute (GNUNET_TIME_relative_multiply + (GNUNET_TIME_UNIT_SECONDS, + (uint32_t) ttl)); + else + pr->public_data.ttl = + GNUNET_TIME_absolute_subtract (pr->public_data.start_time, + GNUNET_TIME_relative_multiply + (GNUNET_TIME_UNIT_SECONDS, + (uint32_t) (-ttl))); + if (replies_seen_count > 0) + { + pr->replies_seen_size = replies_seen_count; + pr->replies_seen = + GNUNET_malloc (sizeof (GNUNET_HashCode) * pr->replies_seen_size); + memcpy (pr->replies_seen, replies_seen, + replies_seen_count * sizeof (GNUNET_HashCode)); + pr->replies_seen_count = replies_seen_count; + } + if (NULL != bf_data) + { + pr->bf = + GNUNET_CONTAINER_bloomfilter_init (bf_data, bf_size, + GNUNET_CONSTANTS_BLOOMFILTER_K); + pr->mingle = mingle; + } + else if ((replies_seen_count > 0) && + (0 != (options & GSF_PRO_BLOOMFILTER_FULL_REFRESH))) + { + refresh_bloomfilter (pr); + } + GNUNET_CONTAINER_multihashmap_put (pr_map, query, pr, + GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE); + if (0 == (options & GSF_PRO_REQUEST_NEVER_EXPIRES)) + { + pr->hnode = + GNUNET_CONTAINER_heap_insert (requests_by_expiration_heap, pr, + pr->public_data.ttl.abs_value); + /* make sure we don't track too many requests */ + while (GNUNET_CONTAINER_heap_get_size (requests_by_expiration_heap) > + max_pending_requests) + { + dpr = GNUNET_CONTAINER_heap_peek (requests_by_expiration_heap); + GNUNET_assert (dpr != NULL); + if (pr == dpr) + break; /* let the request live briefly... */ + if (NULL != dpr->rh) + dpr->rh (dpr->rh_cls, GNUNET_BLOCK_EVALUATION_REQUEST_VALID, dpr, + UINT32_MAX, GNUNET_TIME_UNIT_FOREVER_ABS, GNUNET_TIME_UNIT_FOREVER_ABS, + GNUNET_BLOCK_TYPE_ANY, NULL, 0); + GSF_pending_request_cancel_ (dpr, GNUNET_YES); + } + } + GNUNET_STATISTICS_update (GSF_stats, + gettext_noop ("# Pending requests active"), 1, + GNUNET_NO); + return pr; +} + +/** + * Obtain the public data associated with a pending request + * + * @param pr pending request + * @return associated public data + */ +struct GSF_PendingRequestData * +GSF_pending_request_get_data_ (struct GSF_PendingRequest *pr) +{ + return &pr->public_data; +} + + +/** + * Test if two pending requests are compatible (would generate + * the same query modulo filters and should thus be processed + * jointly). + * + * @param pra a pending request + * @param prb another pending request + * @return GNUNET_OK if the requests are compatible + */ +int +GSF_pending_request_is_compatible_ (struct GSF_PendingRequest *pra, + struct GSF_PendingRequest *prb) +{ + if ((pra->public_data.type != prb->public_data.type) || + (0 != + memcmp (&pra->public_data.query, &prb->public_data.query, + sizeof (GNUNET_HashCode))) || + ((pra->public_data.type == GNUNET_BLOCK_TYPE_FS_SBLOCK) && + (0 != + memcmp (&pra->public_data.namespace, &prb->public_data.namespace, + sizeof (GNUNET_HashCode))))) + return GNUNET_NO; + return GNUNET_OK; +} + + + +/** + * Update a given pending request with additional replies + * that have been seen. + * + * @param pr request to update + * @param replies_seen hash codes of replies that we've seen + * @param replies_seen_count size of the replies_seen array + */ +void +GSF_pending_request_update_ (struct GSF_PendingRequest *pr, + const GNUNET_HashCode * replies_seen, + unsigned int replies_seen_count) +{ + unsigned int i; + GNUNET_HashCode mhash; + + if (replies_seen_count + pr->replies_seen_count < pr->replies_seen_count) + return; /* integer overflow */ + if (0 != (pr->public_data.options & GSF_PRO_BLOOMFILTER_FULL_REFRESH)) + { + /* we're responsible for the BF, full refresh */ + if (replies_seen_count + pr->replies_seen_count > pr->replies_seen_size) + GNUNET_array_grow (pr->replies_seen, pr->replies_seen_size, + replies_seen_count + pr->replies_seen_count); + memcpy (&pr->replies_seen[pr->replies_seen_count], replies_seen, + sizeof (GNUNET_HashCode) * replies_seen_count); + pr->replies_seen_count += replies_seen_count; + refresh_bloomfilter (pr); + } + else + { + if (NULL == pr->bf) + { + /* we're not the initiator, but the initiator did not give us + * any bloom-filter, so we need to create one on-the-fly */ + pr->mingle = + GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, UINT32_MAX); + pr->bf = + GNUNET_BLOCK_construct_bloomfilter (pr->mingle, replies_seen, + replies_seen_count); + } + else + { + for (i = 0; i < pr->replies_seen_count; i++) + { + GNUNET_BLOCK_mingle_hash (&replies_seen[i], pr->mingle, &mhash); + GNUNET_CONTAINER_bloomfilter_add (pr->bf, &mhash); + } + } + } +} + + +/** + * Generate the message corresponding to the given pending request for + * transmission to other peers (or at least determine its size). + * + * @param pr request to generate the message for + * @param buf_size number of bytes available in buf + * @param buf where to copy the message (can be NULL) + * @return number of bytes needed (if > buf_size) or used + */ +size_t +GSF_pending_request_get_message_ (struct GSF_PendingRequest *pr, + size_t buf_size, void *buf) +{ + char lbuf[GNUNET_SERVER_MAX_MESSAGE_SIZE]; + struct GetMessage *gm; + GNUNET_HashCode *ext; + size_t msize; + unsigned int k; + uint32_t bm; + uint32_t prio; + size_t bf_size; + struct GNUNET_TIME_Absolute now; + int64_t ttl; + int do_route; + + if (buf_size > 0) + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Building request message for `%s' of type %d\n", + GNUNET_h2s (&pr->public_data.query), pr->public_data.type); + k = 0; + bm = 0; + do_route = (0 == (pr->public_data.options & GSF_PRO_FORWARD_ONLY)); + if ((!do_route) && (pr->sender_pid == 0)) + { + GNUNET_break (0); + do_route = GNUNET_YES; + } + if (!do_route) + { + bm |= GET_MESSAGE_BIT_RETURN_TO; + k++; + } + if (GNUNET_BLOCK_TYPE_FS_SBLOCK == pr->public_data.type) + { + bm |= GET_MESSAGE_BIT_SKS_NAMESPACE; + k++; + } + if (GNUNET_YES == pr->public_data.has_target) + { + bm |= GET_MESSAGE_BIT_TRANSMIT_TO; + k++; + } + bf_size = GNUNET_CONTAINER_bloomfilter_get_size (pr->bf); + msize = sizeof (struct GetMessage) + bf_size + k * sizeof (GNUNET_HashCode); + GNUNET_assert (msize < GNUNET_SERVER_MAX_MESSAGE_SIZE); + if (buf_size < msize) + return msize; + gm = (struct GetMessage *) lbuf; + gm->header.type = htons (GNUNET_MESSAGE_TYPE_FS_GET); + gm->header.size = htons (msize); + gm->type = htonl (pr->public_data.type); + if (do_route) + prio = + GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, + pr->public_data.priority + 1); + else + prio = 0; + pr->public_data.priority -= prio; + gm->priority = htonl (prio); + now = GNUNET_TIME_absolute_get (); + ttl = (int64_t) (pr->public_data.ttl.abs_value - now.abs_value); + gm->ttl = htonl (ttl / 1000); + gm->filter_mutator = htonl (pr->mingle); + gm->hash_bitmap = htonl (bm); + gm->query = pr->public_data.query; + ext = (GNUNET_HashCode *) & gm[1]; + k = 0; + if (!do_route) + GNUNET_PEER_resolve (pr->sender_pid, + (struct GNUNET_PeerIdentity *) &ext[k++]); + if (GNUNET_BLOCK_TYPE_FS_SBLOCK == pr->public_data.type) + memcpy (&ext[k++], &pr->public_data.namespace, sizeof (GNUNET_HashCode)); + if (GNUNET_YES == pr->public_data.has_target) + ext[k++] = pr->public_data.target.hashPubKey; + if (pr->bf != NULL) + GNUNET_assert (GNUNET_SYSERR != + GNUNET_CONTAINER_bloomfilter_get_raw_data (pr->bf, + (char *) &ext[k], + bf_size)); + memcpy (buf, gm, msize); + return msize; +} + + +/** + * Iterator to free pending requests. + * + * @param cls closure, unused + * @param key current key code + * @param value value in the hash map (pending request) + * @return GNUNET_YES (we should continue to iterate) + */ +static int +clean_request (void *cls, const GNUNET_HashCode * key, void *value) +{ + struct GSF_PendingRequest *pr = value; + GSF_LocalLookupContinuation cont; + + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Cleaning up pending request for `%s'.\n", GNUNET_h2s (key)); + if (NULL != (cont = pr->llc_cont)) + { + pr->llc_cont = NULL; + cont (pr->llc_cont_cls, pr, pr->local_result); + } + GSF_plan_notify_request_done_ (pr); + GNUNET_free_non_null (pr->replies_seen); + if (NULL != pr->bf) + { + GNUNET_CONTAINER_bloomfilter_free (pr->bf); + pr->bf = NULL; + } + GNUNET_PEER_change_rc (pr->sender_pid, -1); + pr->sender_pid = 0; + GNUNET_PEER_change_rc (pr->origin_pid, -1); + pr->origin_pid = 0; + if (NULL != pr->hnode) + { + GNUNET_CONTAINER_heap_remove_node (pr->hnode); + pr->hnode = NULL; + } + if (NULL != pr->qe) + { + GNUNET_DATASTORE_cancel (pr->qe); + pr->qe = NULL; + } + if (NULL != pr->gh) + { + GNUNET_DHT_get_stop (pr->gh); + pr->gh = NULL; + } + if (GNUNET_SCHEDULER_NO_TASK != pr->warn_task) + { + GNUNET_SCHEDULER_cancel (pr->warn_task); + pr->warn_task = GNUNET_SCHEDULER_NO_TASK; + } + GNUNET_assert (GNUNET_OK == + GNUNET_CONTAINER_multihashmap_remove (pr_map, + &pr->public_data.query, + pr)); + GNUNET_STATISTICS_update (GSF_stats, + gettext_noop ("# Pending requests active"), -1, + GNUNET_NO); + GNUNET_free (pr); + return GNUNET_YES; +} + + +/** + * Explicitly cancel a pending request. + * + * @param pr request to cancel + * @param full_cleanup fully purge the request + */ +void +GSF_pending_request_cancel_ (struct GSF_PendingRequest *pr, int full_cleanup) +{ + GSF_LocalLookupContinuation cont; + + if (NULL == pr_map) + return; /* already cleaned up! */ + if (GNUNET_YES != full_cleanup) + { + /* make request inactive (we're no longer interested in more results), + * but do NOT remove from our data-structures, we still need it there + * to prevent the request from looping */ + pr->rh = NULL; + if (NULL != (cont = pr->llc_cont)) + { + pr->llc_cont = NULL; + cont (pr->llc_cont_cls, pr, pr->local_result); + } + GSF_plan_notify_request_done_ (pr); + if (NULL != pr->qe) + { + GNUNET_DATASTORE_cancel (pr->qe); + pr->qe = NULL; + } + if (NULL != pr->gh) + { + GNUNET_DHT_get_stop (pr->gh); + pr->gh = NULL; + } + if (GNUNET_SCHEDULER_NO_TASK != pr->warn_task) + { + GNUNET_SCHEDULER_cancel (pr->warn_task); + pr->warn_task = GNUNET_SCHEDULER_NO_TASK; + } + return; + } + GNUNET_assert (GNUNET_YES == + clean_request (NULL, &pr->public_data.query, pr)); +} + + +/** + * Iterate over all pending requests. + * + * @param it function to call for each request + * @param cls closure for it + */ +void +GSF_iterate_pending_requests_ (GSF_PendingRequestIterator it, void *cls) +{ + GNUNET_CONTAINER_multihashmap_iterate (pr_map, + (GNUNET_CONTAINER_HashMapIterator) it, + cls); +} + + + + +/** + * Closure for "process_reply" function. + */ +struct ProcessReplyClosure +{ + /** + * The data for the reply. + */ + const void *data; + + /** + * Who gave us this reply? NULL for local host (or DHT) + */ + struct GSF_ConnectedPeer *sender; + + /** + * When the reply expires. + */ + struct GNUNET_TIME_Absolute expiration; + + /** + * Size of data. + */ + size_t size; + + /** + * Type of the block. + */ + enum GNUNET_BLOCK_Type type; + + /** + * How much was this reply worth to us? + */ + uint32_t priority; + + /** + * Anonymity requirements for this reply. + */ + uint32_t anonymity_level; + + /** + * Evaluation result (returned). + */ + enum GNUNET_BLOCK_EvaluationResult eval; + + /** + * Did we find a matching request? + */ + int request_found; +}; + + +/** + * Update the performance data for the sender (if any) since + * the sender successfully answered one of our queries. + * + * @param prq information about the sender + * @param pr request that was satisfied + */ +static void +update_request_performance_data (struct ProcessReplyClosure *prq, + struct GSF_PendingRequest *pr) +{ + if (prq->sender == NULL) + return; + GSF_peer_update_performance_ (prq->sender, pr->public_data.start_time, + prq->priority); +} + + +/** + * We have received a reply; handle it! + * + * @param cls response (struct ProcessReplyClosure) + * @param key our query + * @param value value in the hash map (info about the query) + * @return GNUNET_YES (we should continue to iterate) + */ +static int +process_reply (void *cls, const GNUNET_HashCode * key, void *value) +{ + struct ProcessReplyClosure *prq = cls; + struct GSF_PendingRequest *pr = value; + GNUNET_HashCode chash; + struct GNUNET_TIME_Absolute last_transmission; + + if (NULL == pr->rh) + return GNUNET_YES; + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Matched result (type %u) for query `%s' with pending request\n", + (unsigned int) prq->type, GNUNET_h2s (key)); + GNUNET_STATISTICS_update (GSF_stats, + gettext_noop ("# replies received and matched"), 1, + GNUNET_NO); + prq->eval = + GNUNET_BLOCK_evaluate (GSF_block_ctx, prq->type, key, &pr->bf, pr->mingle, + &pr->public_data.namespace, + (prq->type == + GNUNET_BLOCK_TYPE_FS_SBLOCK) ? + sizeof (GNUNET_HashCode) : 0, prq->data, + prq->size); + switch (prq->eval) + { + case GNUNET_BLOCK_EVALUATION_OK_MORE: + update_request_performance_data (prq, pr); + break; + case GNUNET_BLOCK_EVALUATION_OK_LAST: + /* short cut: stop processing early, no BF-update, etc. */ + update_request_performance_data (prq, pr); + GNUNET_LOAD_update (GSF_rt_entry_lifetime, + GNUNET_TIME_absolute_get_duration (pr-> + public_data.start_time).rel_value); + if (!GSF_request_plan_reference_get_last_transmission_ (pr->public_data.rpr_head, prq->sender, &last_transmission)) + last_transmission.abs_value = GNUNET_TIME_UNIT_FOREVER_ABS.abs_value; + /* pass on to other peers / local clients */ + pr->rh (pr->rh_cls, prq->eval, pr, prq->anonymity_level, prq->expiration, + last_transmission, prq->type, prq->data, prq->size); + return GNUNET_YES; + case GNUNET_BLOCK_EVALUATION_OK_DUPLICATE: + GNUNET_STATISTICS_update (GSF_stats, + gettext_noop + ("# duplicate replies discarded (bloomfilter)"), + 1, GNUNET_NO); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Duplicate response, discarding.\n"); + return GNUNET_YES; /* duplicate */ + case GNUNET_BLOCK_EVALUATION_RESULT_INVALID: + return GNUNET_YES; /* wrong namespace */ + case GNUNET_BLOCK_EVALUATION_REQUEST_VALID: + GNUNET_break (0); + return GNUNET_YES; + case GNUNET_BLOCK_EVALUATION_REQUEST_INVALID: + GNUNET_break (0); + return GNUNET_YES; + case GNUNET_BLOCK_EVALUATION_TYPE_NOT_SUPPORTED: + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, _("Unsupported block type %u\n"), + prq->type); + return GNUNET_NO; + } + /* update bloomfilter */ + GNUNET_CRYPTO_hash (prq->data, prq->size, &chash); + GSF_pending_request_update_ (pr, &chash, 1); + if (NULL == prq->sender) + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Found result for query `%s' in local datastore\n", + GNUNET_h2s (key)); + GNUNET_STATISTICS_update (GSF_stats, + gettext_noop ("# results found locally"), 1, + GNUNET_NO); + } + else + { + GSF_dht_lookup_ (pr); + } + prq->priority += pr->public_data.original_priority; + pr->public_data.priority = 0; + pr->public_data.original_priority = 0; + pr->public_data.results_found++; + prq->request_found = GNUNET_YES; + /* finally, pass on to other peer / local client */ + if (!GSF_request_plan_reference_get_last_transmission_ (pr->public_data.rpr_head, prq->sender, &last_transmission)) + last_transmission.abs_value = GNUNET_TIME_UNIT_FOREVER_ABS.abs_value; + pr->rh (pr->rh_cls, prq->eval, pr, prq->anonymity_level, prq->expiration, + last_transmission, prq->type, prq->data, prq->size); + return GNUNET_YES; +} + + +/** + * Context for the 'put_migration_continuation'. + */ +struct PutMigrationContext +{ + + /** + * Start time for the operation. + */ + struct GNUNET_TIME_Absolute start; + + /** + * Request origin. + */ + struct GNUNET_PeerIdentity origin; + + /** + * GNUNET_YES if we had a matching request for this block, + * GNUNET_NO if not. + */ + int requested; +}; + + +/** + * Continuation called to notify client about result of the + * operation. + * + * @param cls closure + * @param success GNUNET_SYSERR on failure + * @param min_expiration minimum expiration time required for content to be stored + * @param msg NULL on success, otherwise an error message + */ +static void +put_migration_continuation (void *cls, int success, + struct GNUNET_TIME_Absolute min_expiration, + const char *msg) +{ + struct PutMigrationContext *pmc = cls; + struct GSF_ConnectedPeer *cp; + struct GNUNET_TIME_Relative mig_pause; + struct GSF_PeerPerformanceData *ppd; + + if (NULL != datastore_put_load) + { + if (GNUNET_SYSERR != success) + { + GNUNET_LOAD_update (datastore_put_load, + GNUNET_TIME_absolute_get_duration (pmc->start).rel_value); + } + else + { + /* on queue failure / timeout, increase the put load dramatically */ + GNUNET_LOAD_update (datastore_put_load, + GNUNET_TIME_UNIT_MINUTES.rel_value); + } + } + cp = GSF_peer_get_ (&pmc->origin); + if (GNUNET_OK == success) + { + if (NULL != cp) + { + ppd = GSF_get_peer_performance_data_ (cp); + ppd->migration_delay.rel_value /= 2; + } + GNUNET_free (pmc); + return; + } + if ( (GNUNET_NO == success) && + (GNUNET_NO == pmc->requested) && + (NULL != cp) ) + { + ppd = GSF_get_peer_performance_data_ (cp); + if (min_expiration.abs_value > 0) + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Asking to stop migration for %llu ms because datastore is full\n", + (unsigned long long) GNUNET_TIME_absolute_get_remaining (min_expiration).rel_value); + GSF_block_peer_migration_ (cp, min_expiration); + } + else + { + ppd->migration_delay = GNUNET_TIME_relative_max (GNUNET_TIME_UNIT_SECONDS, + ppd->migration_delay); + ppd->migration_delay = GNUNET_TIME_relative_min (GNUNET_TIME_UNIT_HOURS, + ppd->migration_delay); + mig_pause.rel_value = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK, + ppd->migration_delay.rel_value); + ppd->migration_delay = GNUNET_TIME_relative_multiply (ppd->migration_delay, 2); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Replicated content already exists locally, asking to stop migration for %llu ms\n", + (unsigned long long) mig_pause.rel_value); + GSF_block_peer_migration_ (cp, GNUNET_TIME_relative_to_absolute (mig_pause)); + } + } + GNUNET_free (pmc); + GNUNET_STATISTICS_update (GSF_stats, + gettext_noop ("# Datastore `PUT' failures"), 1, + GNUNET_NO); +} + + +/** + * Test if the DATABASE (PUT) load on this peer is too high + * to even consider processing the query at + * all. + * + * @return GNUNET_YES if the load is too high to do anything (load high) + * GNUNET_NO to process normally (load normal or low) + */ +static int +test_put_load_too_high (uint32_t priority) +{ + double ld; + + if (NULL == datastore_put_load) + return GNUNET_NO; + if (GNUNET_LOAD_get_average (datastore_put_load) < 50) + return GNUNET_NO; /* very fast */ + ld = GNUNET_LOAD_get_load (datastore_put_load); + if (ld < 2.0 * (1 + priority)) + return GNUNET_NO; + GNUNET_STATISTICS_update (GSF_stats, + gettext_noop + ("# storage requests dropped due to high load"), 1, + GNUNET_NO); + return GNUNET_YES; +} + + +/** + * Iterator called on each result obtained for a DHT + * operation that expects a reply + * + * @param cls closure + * @param exp when will this value expire + * @param key key of the result + * @param get_path peers on reply path (or NULL if not recorded) + * @param get_path_length number of entries in get_path + * @param put_path peers on the PUT path (or NULL if not recorded) + * @param put_path_length number of entries in get_path + * @param type type of the result + * @param size number of bytes in data + * @param data pointer to the result data + */ +static void +handle_dht_reply (void *cls, struct GNUNET_TIME_Absolute exp, + const GNUNET_HashCode * key, + const struct GNUNET_PeerIdentity *get_path, + unsigned int get_path_length, + const struct GNUNET_PeerIdentity *put_path, + unsigned int put_path_length, enum GNUNET_BLOCK_Type type, + size_t size, const void *data) +{ + struct GSF_PendingRequest *pr = cls; + struct ProcessReplyClosure prq; + struct PutMigrationContext *pmc; + + GNUNET_STATISTICS_update (GSF_stats, + gettext_noop ("# Replies received from DHT"), 1, + GNUNET_NO); + memset (&prq, 0, sizeof (prq)); + prq.data = data; + prq.expiration = exp; + /* do not allow migrated content to live longer than 1 year */ + prq.expiration = GNUNET_TIME_absolute_min (GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_YEARS), + prq.expiration); + prq.size = size; + prq.type = type; + process_reply (&prq, key, pr); + if ((GNUNET_YES == active_to_migration) && + (GNUNET_NO == test_put_load_too_high (prq.priority))) + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Replicating result for query `%s' with priority %u\n", + GNUNET_h2s (key), prq.priority); + pmc = GNUNET_malloc (sizeof (struct PutMigrationContext)); + pmc->start = GNUNET_TIME_absolute_get (); + pmc->requested = GNUNET_YES; + if (NULL == + GNUNET_DATASTORE_put (GSF_dsh, 0, key, size, data, type, prq.priority, + 1 /* anonymity */ , + 0 /* replication */ , + exp, 1 + prq.priority, MAX_DATASTORE_QUEUE, + GNUNET_CONSTANTS_SERVICE_TIMEOUT, + &put_migration_continuation, pmc)) + { + put_migration_continuation (pmc, GNUNET_SYSERR, GNUNET_TIME_UNIT_ZERO_ABS, NULL); + } + } +} + + +/** + * Consider looking up the data in the DHT (anonymity-level permitting). + * + * @param pr the pending request to process + */ +void +GSF_dht_lookup_ (struct GSF_PendingRequest *pr) +{ + const void *xquery; + size_t xquery_size; + struct GNUNET_PeerIdentity pi; + char buf[sizeof (GNUNET_HashCode) * 2]; + + if (0 != pr->public_data.anonymity_level) + return; + if (NULL != pr->gh) + { + GNUNET_DHT_get_stop (pr->gh); + pr->gh = NULL; + } + xquery = NULL; + xquery_size = 0; + if (GNUNET_BLOCK_TYPE_FS_SBLOCK == pr->public_data.type) + { + xquery = buf; + memcpy (buf, &pr->public_data.namespace, sizeof (GNUNET_HashCode)); + xquery_size = sizeof (GNUNET_HashCode); + } + if (0 != (pr->public_data.options & GSF_PRO_FORWARD_ONLY)) + { + GNUNET_assert (0 != pr->sender_pid); + GNUNET_PEER_resolve (pr->sender_pid, &pi); + memcpy (&buf[xquery_size], &pi, sizeof (struct GNUNET_PeerIdentity)); + xquery_size += sizeof (struct GNUNET_PeerIdentity); + } + pr->gh = + GNUNET_DHT_get_start (GSF_dht, GNUNET_TIME_UNIT_FOREVER_REL, + pr->public_data.type, &pr->public_data.query, + 5 /* DEFAULT_GET_REPLICATION */ , + GNUNET_DHT_RO_DEMULTIPLEX_EVERYWHERE, + /* FIXME: can no longer pass pr->bf/pr->mingle... */ + xquery, xquery_size, &handle_dht_reply, pr); +} + + +/** + * Task that issues a warning if the datastore lookup takes too long. + * + * @param cls the 'struct GSF_PendingRequest' + * @param tc task context + */ +static void +warn_delay_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct GSF_PendingRequest *pr = cls; + + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _("Datastore lookup already took %llu ms!\n"), + (unsigned long long) + GNUNET_TIME_absolute_get_duration (pr->qe_start).rel_value); + pr->warn_task = + GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES, &warn_delay_task, + pr); +} + + +/** + * Task that issues a warning if the datastore lookup takes too long. + * + * @param cls the 'struct GSF_PendingRequest' + * @param tc task context + */ +static void +odc_warn_delay_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct GSF_PendingRequest *pr = cls; + + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _("On-demand lookup already took %llu ms!\n"), + (unsigned long long) + GNUNET_TIME_absolute_get_duration (pr->qe_start).rel_value); + pr->warn_task = + GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES, + &odc_warn_delay_task, pr); +} + + +/** + * We're processing (local) results for a search request + * from another peer. Pass applicable results to the + * peer and if we are done either clean up (operation + * complete) or forward to other peers (more results possible). + * + * @param cls our closure (struct PendingRequest) + * @param key key for the content + * @param size number of bytes in data + * @param data content stored + * @param type type of the content + * @param priority priority of the content + * @param anonymity anonymity-level for the content + * @param expiration expiration time for the content + * @param uid unique identifier for the datum; + * maybe 0 if no unique identifier is available + */ +static void +process_local_reply (void *cls, const GNUNET_HashCode * key, size_t size, + const void *data, enum GNUNET_BLOCK_Type type, + uint32_t priority, uint32_t anonymity, + struct GNUNET_TIME_Absolute expiration, uint64_t uid) +{ + struct GSF_PendingRequest *pr = cls; + GSF_LocalLookupContinuation cont; + struct ProcessReplyClosure prq; + GNUNET_HashCode query; + unsigned int old_rf; + + GNUNET_SCHEDULER_cancel (pr->warn_task); + pr->warn_task = GNUNET_SCHEDULER_NO_TASK; + if (NULL != pr->qe) + { + pr->qe = NULL; + if (NULL == key) + { + GNUNET_STATISTICS_update (GSF_stats, + gettext_noop + ("# Datastore lookups concluded (no results)"), + 1, GNUNET_NO); + } + if (GNUNET_NO == pr->have_first_uid) + { + pr->first_uid = uid; + pr->have_first_uid = 1; + } + else + { + if ((uid == pr->first_uid) && (key != NULL)) + { + GNUNET_STATISTICS_update (GSF_stats, + gettext_noop + ("# Datastore lookups concluded (seen all)"), + 1, GNUNET_NO); + key = NULL; /* all replies seen! */ + } + pr->have_first_uid++; + if ((pr->have_first_uid > MAX_RESULTS) && (key != NULL)) + { + GNUNET_STATISTICS_update (GSF_stats, + gettext_noop + ("# Datastore lookups aborted (more than MAX_RESULTS)"), + 1, GNUNET_NO); + key = NULL; /* all replies seen! */ + } + } + } + if (NULL == key) + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG | GNUNET_ERROR_TYPE_BULK, + "No further local responses available.\n"); + if ((pr->public_data.type == GNUNET_BLOCK_TYPE_FS_DBLOCK) || + (pr->public_data.type == GNUNET_BLOCK_TYPE_FS_IBLOCK)) + GNUNET_STATISTICS_update (GSF_stats, + gettext_noop + ("# requested DBLOCK or IBLOCK not found"), 1, + GNUNET_NO); + goto check_error_and_continue; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Received reply for `%s' of type %d with UID %llu from datastore.\n", + GNUNET_h2s (key), type, (unsigned long long) uid); + if (type == GNUNET_BLOCK_TYPE_FS_ONDEMAND) + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Found ONDEMAND block, performing on-demand encoding\n"); + GNUNET_STATISTICS_update (GSF_stats, + gettext_noop + ("# on-demand blocks matched requests"), 1, + GNUNET_NO); + pr->qe_start = GNUNET_TIME_absolute_get (); + pr->warn_task = + GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES, + &odc_warn_delay_task, pr); + if (GNUNET_OK == + GNUNET_FS_handle_on_demand_block (key, size, data, type, priority, + anonymity, expiration, uid, + &process_local_reply, pr)) + { + GNUNET_STATISTICS_update (GSF_stats, + gettext_noop + ("# on-demand lookups performed successfully"), + 1, GNUNET_NO); + return; /* we're done */ + } + GNUNET_STATISTICS_update (GSF_stats, + gettext_noop ("# on-demand lookups failed"), 1, + GNUNET_NO); + GNUNET_SCHEDULER_cancel (pr->warn_task); + pr->warn_task = + GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES, + &warn_delay_task, pr); + pr->qe = + GNUNET_DATASTORE_get_key (GSF_dsh, pr->local_result_offset - 1, + &pr->public_data.query, + pr->public_data.type == + GNUNET_BLOCK_TYPE_FS_DBLOCK ? + GNUNET_BLOCK_TYPE_ANY : pr->public_data.type, + (0 != + (GSF_PRO_PRIORITY_UNLIMITED & + pr->public_data.options)) ? UINT_MAX : 1 + /* queue priority */ , + (0 != + (GSF_PRO_PRIORITY_UNLIMITED & + pr->public_data.options)) ? UINT_MAX : + datastore_queue_size + /* max queue size */ , + GNUNET_TIME_UNIT_FOREVER_REL, + &process_local_reply, pr); + if (NULL != pr->qe) + return; /* we're done */ + GNUNET_STATISTICS_update (GSF_stats, + gettext_noop + ("# Datastore lookups concluded (error queueing)"), + 1, GNUNET_NO); + goto check_error_and_continue; + } + old_rf = pr->public_data.results_found; + memset (&prq, 0, sizeof (prq)); + prq.data = data; + prq.expiration = expiration; + prq.size = size; + if (GNUNET_OK != + GNUNET_BLOCK_get_key (GSF_block_ctx, type, data, size, &query)) + { + GNUNET_break (0); + GNUNET_DATASTORE_remove (GSF_dsh, key, size, data, -1, -1, + GNUNET_TIME_UNIT_FOREVER_REL, NULL, NULL); + pr->qe_start = GNUNET_TIME_absolute_get (); + pr->warn_task = + GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES, + &warn_delay_task, pr); + pr->qe = + GNUNET_DATASTORE_get_key (GSF_dsh, pr->local_result_offset - 1, + &pr->public_data.query, + pr->public_data.type == + GNUNET_BLOCK_TYPE_FS_DBLOCK ? + GNUNET_BLOCK_TYPE_ANY : pr->public_data.type, + (0 != + (GSF_PRO_PRIORITY_UNLIMITED & + pr->public_data.options)) ? UINT_MAX : 1 + /* queue priority */ , + (0 != + (GSF_PRO_PRIORITY_UNLIMITED & + pr->public_data.options)) ? UINT_MAX : + datastore_queue_size + /* max queue size */ , + GNUNET_TIME_UNIT_FOREVER_REL, + &process_local_reply, pr); + if (pr->qe == NULL) + { + GNUNET_STATISTICS_update (GSF_stats, + gettext_noop + ("# Datastore lookups concluded (error queueing)"), + 1, GNUNET_NO); + goto check_error_and_continue; + } + return; + } + prq.type = type; + prq.priority = priority; + prq.request_found = GNUNET_NO; + prq.anonymity_level = anonymity; + if ((old_rf == 0) && (pr->public_data.results_found == 0)) + GSF_update_datastore_delay_ (pr->public_data.start_time); + process_reply (&prq, key, pr); + pr->local_result = prq.eval; + if (prq.eval == GNUNET_BLOCK_EVALUATION_OK_LAST) + { + GNUNET_STATISTICS_update (GSF_stats, + gettext_noop + ("# Datastore lookups concluded (found last result)"), + 1, GNUNET_NO); + goto check_error_and_continue; + } + if ((0 == (GSF_PRO_PRIORITY_UNLIMITED & pr->public_data.options)) && + ((GNUNET_YES == GSF_test_get_load_too_high_ (0)) || + (pr->public_data.results_found > 5 + 2 * pr->public_data.priority))) + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Load too high, done with request\n"); + GNUNET_STATISTICS_update (GSF_stats, + gettext_noop + ("# Datastore lookups concluded (load too high)"), + 1, GNUNET_NO); + goto check_error_and_continue; + } + pr->qe_start = GNUNET_TIME_absolute_get (); + pr->warn_task = + GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES, &warn_delay_task, + pr); + pr->qe = + GNUNET_DATASTORE_get_key (GSF_dsh, pr->local_result_offset++, + &pr->public_data.query, + pr->public_data.type == + GNUNET_BLOCK_TYPE_FS_DBLOCK ? + GNUNET_BLOCK_TYPE_ANY : pr->public_data.type, + (0 != + (GSF_PRO_PRIORITY_UNLIMITED & pr-> + public_data.options)) ? UINT_MAX : 1 + /* queue priority */ , + (0 != + (GSF_PRO_PRIORITY_UNLIMITED & pr-> + public_data.options)) ? UINT_MAX : + datastore_queue_size + /* max queue size */ , + GNUNET_TIME_UNIT_FOREVER_REL, + &process_local_reply, pr); + /* check if we successfully queued another datastore request; + * if so, return, otherwise call our continuation (if we have + * any) */ +check_error_and_continue: + if (NULL != pr->qe) + return; + if (GNUNET_SCHEDULER_NO_TASK != pr->warn_task) + { + GNUNET_SCHEDULER_cancel (pr->warn_task); + pr->warn_task = GNUNET_SCHEDULER_NO_TASK; + } + if (NULL == (cont = pr->llc_cont)) + return; /* no continuation */ + pr->llc_cont = NULL; + cont (pr->llc_cont_cls, pr, pr->local_result); +} + + +/** + * Is the given target a legitimate peer for forwarding the given request? + * + * @param pr request + * @param target + * @return GNUNET_YES if this request could be forwarded to the given peer + */ +int +GSF_pending_request_test_target_ (struct GSF_PendingRequest *pr, + const struct GNUNET_PeerIdentity *target) +{ + struct GNUNET_PeerIdentity pi; + + if (0 == pr->origin_pid) + return GNUNET_YES; + GNUNET_PEER_resolve (pr->origin_pid, &pi); + return (0 == + memcmp (&pi, target, + sizeof (struct GNUNET_PeerIdentity))) ? GNUNET_NO : + GNUNET_YES; +} + + +/** + * Look up the request in the local datastore. + * + * @param pr the pending request to process + * @param cont function to call at the end + * @param cont_cls closure for cont + */ +void +GSF_local_lookup_ (struct GSF_PendingRequest *pr, + GSF_LocalLookupContinuation cont, void *cont_cls) +{ + GNUNET_assert (NULL == pr->gh); + GNUNET_assert (NULL == pr->llc_cont); + pr->llc_cont = cont; + pr->llc_cont_cls = cont_cls; + pr->qe_start = GNUNET_TIME_absolute_get (); + pr->warn_task = + GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES, &warn_delay_task, + pr); + GNUNET_STATISTICS_update (GSF_stats, + gettext_noop ("# Datastore lookups initiated"), 1, + GNUNET_NO); + pr->qe = + GNUNET_DATASTORE_get_key (GSF_dsh, pr->local_result_offset++, + &pr->public_data.query, + pr->public_data.type == + GNUNET_BLOCK_TYPE_FS_DBLOCK ? + GNUNET_BLOCK_TYPE_ANY : pr->public_data.type, + (0 != + (GSF_PRO_PRIORITY_UNLIMITED & pr-> + public_data.options)) ? UINT_MAX : 1 + /* queue priority */ , + (0 != + (GSF_PRO_PRIORITY_UNLIMITED & pr-> + public_data.options)) ? UINT_MAX : + datastore_queue_size + /* max queue size */ , + GNUNET_TIME_UNIT_FOREVER_REL, + &process_local_reply, pr); + if (NULL != pr->qe) + return; + GNUNET_STATISTICS_update (GSF_stats, + gettext_noop + ("# Datastore lookups concluded (error queueing)"), + 1, GNUNET_NO); + GNUNET_SCHEDULER_cancel (pr->warn_task); + pr->warn_task = GNUNET_SCHEDULER_NO_TASK; + pr->llc_cont = NULL; + if (NULL != cont) + cont (cont_cls, pr, pr->local_result); +} + + + +/** + * Handle P2P "CONTENT" message. Checks that the message is + * well-formed and then checks if there are any pending requests for + * this content and possibly passes it on (to local clients or other + * peers). Does NOT perform migration (content caching at this peer). + * + * @param cp the other peer involved (sender or receiver, NULL + * for loopback messages where we are both sender and receiver) + * @param message the actual message + * @return GNUNET_OK if the message was well-formed, + * GNUNET_SYSERR if the message was malformed (close connection, + * do not cache under any circumstances) + */ +int +GSF_handle_p2p_content_ (struct GSF_ConnectedPeer *cp, + const struct GNUNET_MessageHeader *message) +{ + const struct PutMessage *put; + uint16_t msize; + size_t dsize; + enum GNUNET_BLOCK_Type type; + struct GNUNET_TIME_Absolute expiration; + GNUNET_HashCode query; + struct ProcessReplyClosure prq; + struct GNUNET_TIME_Relative block_time; + double putl; + struct PutMigrationContext *pmc; + + msize = ntohs (message->size); + if (msize < sizeof (struct PutMessage)) + { + GNUNET_break_op (0); + return GNUNET_SYSERR; + } + put = (const struct PutMessage *) message; + dsize = msize - sizeof (struct PutMessage); + type = ntohl (put->type); + expiration = GNUNET_TIME_absolute_ntoh (put->expiration); + /* do not allow migrated content to live longer than 1 year */ + expiration = GNUNET_TIME_absolute_min (GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_YEARS), + expiration); + if (type == GNUNET_BLOCK_TYPE_FS_ONDEMAND) + return GNUNET_SYSERR; + if (GNUNET_OK != + GNUNET_BLOCK_get_key (GSF_block_ctx, type, &put[1], dsize, &query)) + { + GNUNET_break_op (0); + return GNUNET_SYSERR; + } + GNUNET_STATISTICS_update (GSF_stats, + gettext_noop ("# GAP PUT messages received"), 1, + GNUNET_NO); + /* now, lookup 'query' */ + prq.data = (const void *) &put[1]; + if (NULL != cp) + prq.sender = cp; + else + prq.sender = NULL; + prq.size = dsize; + prq.type = type; + prq.expiration = expiration; + prq.priority = 0; + prq.anonymity_level = UINT32_MAX; + prq.request_found = GNUNET_NO; + GNUNET_CONTAINER_multihashmap_get_multiple (pr_map, &query, &process_reply, + &prq); + if (NULL != cp) + { + GSF_connected_peer_change_preference_ (cp, + CONTENT_BANDWIDTH_VALUE + + 1000 * prq.priority); + GSF_get_peer_performance_data_ (cp)->trust += prq.priority; + } + if ((GNUNET_YES == active_to_migration) && + (GNUNET_NO == test_put_load_too_high (prq.priority))) + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Replicating result for query `%s' with priority %u\n", + GNUNET_h2s (&query), prq.priority); + pmc = GNUNET_malloc (sizeof (struct PutMigrationContext)); + pmc->start = GNUNET_TIME_absolute_get (); + pmc->requested = prq.request_found; + GNUNET_assert (0 != GSF_get_peer_performance_data_ (cp)->pid); + GNUNET_PEER_resolve (GSF_get_peer_performance_data_ (cp)->pid, + &pmc->origin); + if (NULL == + GNUNET_DATASTORE_put (GSF_dsh, 0, &query, dsize, &put[1], type, + prq.priority, 1 /* anonymity */ , + 0 /* replication */ , + expiration, 1 + prq.priority, MAX_DATASTORE_QUEUE, + GNUNET_CONSTANTS_SERVICE_TIMEOUT, + &put_migration_continuation, pmc)) + { + put_migration_continuation (pmc, GNUNET_SYSERR, GNUNET_TIME_UNIT_ZERO_ABS, NULL); + } + } + else + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Choosing not to keep content `%s' (%d/%d)\n", + GNUNET_h2s (&query), active_to_migration, + test_put_load_too_high (prq.priority)); + } + putl = GNUNET_LOAD_get_load (datastore_put_load); + if ((NULL != (cp = prq.sender)) && (GNUNET_NO == prq.request_found) && + ((GNUNET_YES != active_to_migration) || + (putl > 2.5 * (1 + prq.priority)))) + { + if (GNUNET_YES != active_to_migration) + putl = 1.0 + GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 5); + block_time = + GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MILLISECONDS, + 5000 + + GNUNET_CRYPTO_random_u32 + (GNUNET_CRYPTO_QUALITY_WEAK, + (unsigned int) (60000 * putl * putl))); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Asking to stop migration for %llu ms because of load %f and events %d/%d\n", + (unsigned long long) block_time.rel_value, + putl, + active_to_migration, + (GNUNET_NO == prq.request_found)); + GSF_block_peer_migration_ (cp, GNUNET_TIME_relative_to_absolute (block_time)); + } + return GNUNET_OK; +} + + +/** + * Setup the subsystem. + */ +void +GSF_pending_request_init_ () +{ + unsigned long long bps; + + if (GNUNET_OK != + GNUNET_CONFIGURATION_get_value_number (GSF_cfg, "fs", + "MAX_PENDING_REQUESTS", + &max_pending_requests)) + { + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + _ + ("Configuration fails to specify `%s', assuming default value."), + "MAX_PENDING_REQUESTS"); + } + if (GNUNET_OK != + GNUNET_CONFIGURATION_get_value_size (GSF_cfg, "ats", "WAN_QUOTA_OUT", + &bps)) + { + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + _ + ("Configuration fails to specify `%s', assuming default value."), + "WAN_QUOTA_OUT"); + bps = 65536; + } + /* queue size should be #queries we can have pending and satisfy within + * a carry interval: */ + datastore_queue_size = + bps * GNUNET_CONSTANTS_MAX_BANDWIDTH_CARRY_S / DBLOCK_SIZE; + + active_to_migration = + GNUNET_CONFIGURATION_get_value_yesno (GSF_cfg, "FS", "CONTENT_CACHING"); + datastore_put_load = GNUNET_LOAD_value_init (DATASTORE_LOAD_AUTODECLINE); + pr_map = GNUNET_CONTAINER_multihashmap_create (32 * 1024); + requests_by_expiration_heap = + GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN); +} + + +/** + * Shutdown the subsystem. + */ +void +GSF_pending_request_done_ () +{ + GNUNET_CONTAINER_multihashmap_iterate (pr_map, &clean_request, NULL); + GNUNET_CONTAINER_multihashmap_destroy (pr_map); + pr_map = NULL; + GNUNET_CONTAINER_heap_destroy (requests_by_expiration_heap); + requests_by_expiration_heap = NULL; + GNUNET_LOAD_value_free (datastore_put_load); + datastore_put_load = NULL; +} + + +/* end of gnunet-service-fs_pr.c */ diff --git a/src/fs/gnunet-service-fs_pr.h b/src/fs/gnunet-service-fs_pr.h new file mode 100644 index 0000000..92827f7 --- /dev/null +++ b/src/fs/gnunet-service-fs_pr.h @@ -0,0 +1,403 @@ +/* + This file is part of GNUnet. + (C) 2009, 2010, 2011 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/gnunet-service-fs_pr.h + * @brief API to handle pending requests + * @author Christian Grothoff + */ +#ifndef GNUNET_SERVICE_FS_PR_H +#define GNUNET_SERVICE_FS_PR_H + +#include "gnunet-service-fs.h" + + +/** + * Options for pending requests (bits to be ORed). + */ +enum GSF_PendingRequestOptions +{ + + /** + * No special options (P2P-default). + */ + GSF_PRO_DEFAULTS = 0, + + /** + * Request must only be processed locally. + */ + GSF_PRO_LOCAL_ONLY = 1, + + /** + * Request must only be forwarded (no routing) + */ + GSF_PRO_FORWARD_ONLY = 2, + + /** + * Request persists indefinitely (no expiration). + */ + GSF_PRO_REQUEST_NEVER_EXPIRES = 4, + + /** + * Request is allowed to refresh bloomfilter and change mingle value. + */ + GSF_PRO_BLOOMFILTER_FULL_REFRESH = 8, + + /** + * Request priority is allowed to be exceeded. + */ + GSF_PRO_PRIORITY_UNLIMITED = 16, + + /** + * Option mask for typical local requests. + */ + GSF_PRO_LOCAL_REQUEST = + (GSF_PRO_BLOOMFILTER_FULL_REFRESH | GSF_PRO_PRIORITY_UNLIMITED | GSF_PRO_REQUEST_NEVER_EXPIRES) +}; + + +/** + * Public data (in the sense of not encapsulated within + * 'gnunet-service-fs_pr', not in the sense of network-wide + * known) associated with each pending request. + */ +struct GSF_PendingRequestData +{ + + /** + * Primary query hash for this request. + */ + GNUNET_HashCode query; + + /** + * Namespace to query, only set if the type is SBLOCK. + */ + GNUNET_HashCode namespace; + + /** + * Identity of a peer hosting the content, only set if + * 'has_target' is GNUNET_YES. + */ + struct GNUNET_PeerIdentity target; + + /** + * Fields for the plan module to track a DLL with the request. + */ + struct GSF_RequestPlanReference *rpr_head; + + /** + * Fields for the plan module to track a DLL with the request. + */ + struct GSF_RequestPlanReference *rpr_tail; + + /** + * Current TTL for the request. + */ + struct GNUNET_TIME_Absolute ttl; + + /** + * When did we start with the request. + */ + struct GNUNET_TIME_Absolute start_time; + + /** + * Desired anonymity level. + */ + uint32_t anonymity_level; + + /** + * Priority that this request (still) has for us. + */ + uint32_t priority; + + /** + * Priority that this request (originally) had for us. + */ + uint32_t original_priority; + + /** + * Options for the request. + */ + enum GSF_PendingRequestOptions options; + + /** + * Type of the requested block. + */ + enum GNUNET_BLOCK_Type type; + + /** + * Number of results we have found for this request so far. + */ + unsigned int results_found; + + /** + * Is the 'target' value set to a valid peer identity? + */ + int has_target; + + /** + * Has this request been started yet (local/p2p operations)? Or are + * we still constructing it? + */ + int has_started; + +}; + + +/** + * Handle a reply to a pending request. Also called if a request + * expires (then with data == NULL). The handler may be called + * many times (depending on the request type), but will not be + * called during or after a call to GSF_pending_request_cancel + * and will also not be called anymore after a call signalling + * expiration. + * + * @param cls user-specified closure + * @param eval evaluation of the result + * @param pr handle to the original pending request + * @param reply_anonymity_level anonymity level for the reply, UINT32_MAX for "unknown" + * @param expiration when does 'data' expire? + * @param last_transmission the last time we've tried to get this block (FOREVER if unknown) + * @param type type of the block + * @param data response data, NULL on request expiration + * @param data_len number of bytes in data + */ +typedef void (*GSF_PendingRequestReplyHandler) (void *cls, + enum + GNUNET_BLOCK_EvaluationResult + eval, + struct GSF_PendingRequest * pr, + uint32_t reply_anonymity_level, + struct GNUNET_TIME_Absolute + expiration, + struct GNUNET_TIME_Absolute + last_transmission, + enum GNUNET_BLOCK_Type type, + const void *data, + size_t data_len); + + +/** + * Create a new pending request. + * + * @param options request options + * @param type type of the block that is being requested + * @param query key for the lookup + * @param namespace namespace to lookup, NULL for no namespace + * @param target preferred target for the request, NULL for none + * @param bf_data raw data for bloom filter for known replies, can be NULL + * @param bf_size number of bytes in bf_data + * @param mingle mingle value for bf + * @param anonymity_level desired anonymity level + * @param priority maximum outgoing cummulative request priority to use + * @param ttl current time-to-live for the request + * @param sender_pid peer ID to use for the sender when forwarding, 0 for none; + * reference counter is taken over by this function + * @param origin_pid peer ID of origin of query (do not loop back) + * @param replies_seen hash codes of known local replies + * @param replies_seen_count size of the 'replies_seen' array + * @param rh handle to call when we get a reply + * @param rh_cls closure for rh + * @return handle for the new pending request + */ +struct GSF_PendingRequest * +GSF_pending_request_create_ (enum GSF_PendingRequestOptions options, + enum GNUNET_BLOCK_Type type, + const GNUNET_HashCode * query, + const GNUNET_HashCode * namespace, + const struct GNUNET_PeerIdentity *target, + const char *bf_data, size_t bf_size, + uint32_t mingle, uint32_t anonymity_level, + uint32_t priority, int32_t ttl, + GNUNET_PEER_Id sender_pid, + GNUNET_PEER_Id origin_pid, + const GNUNET_HashCode * replies_seen, + unsigned int replies_seen_count, + GSF_PendingRequestReplyHandler rh, void *rh_cls); + + +/** + * Update a given pending request with additional replies + * that have been seen. + * + * @param pr request to update + * @param replies_seen hash codes of replies that we've seen + * @param replies_seen_count size of the replies_seen array + */ +void +GSF_pending_request_update_ (struct GSF_PendingRequest *pr, + const GNUNET_HashCode * replies_seen, + unsigned int replies_seen_count); + + +/** + * Obtain the public data associated with a pending request + * + * @param pr pending request + * @return associated public data + */ +struct GSF_PendingRequestData * +GSF_pending_request_get_data_ (struct GSF_PendingRequest *pr); + + +/** + * Test if two pending requests are compatible (would generate + * the same query modulo filters and should thus be processed + * jointly). + * + * @param pra a pending request + * @param prb another pending request + * @return GNUNET_OK if the requests are compatible + */ +int +GSF_pending_request_is_compatible_ (struct GSF_PendingRequest *pra, + struct GSF_PendingRequest *prb); + + +/** + * Generate the message corresponding to the given pending request for + * transmission to other peers (or at least determine its size). + * + * @param pr request to generate the message for + * @param buf_size number of bytes available in buf + * @param buf where to copy the message (can be NULL) + * @return number of bytes needed (if buf_size too small) or used + */ +size_t +GSF_pending_request_get_message_ (struct GSF_PendingRequest *pr, + size_t buf_size, void *buf); + + +/** + * Explicitly cancel a pending request. + * + * @param pr request to cancel + * @param full_cleanup fully purge the request + */ +void +GSF_pending_request_cancel_ (struct GSF_PendingRequest *pr, int full_cleanup); + + +/** + * Signature of function called on each request. + * (Note: 'subtype' of GNUNET_CONTAINER_HashMapIterator). + * + * @param cls closure + * @param key query for the request + * @param pr handle to the pending request + * @return GNUNET_YES to continue to iterate + */ +typedef int (*GSF_PendingRequestIterator) (void *cls, + const GNUNET_HashCode * key, + struct GSF_PendingRequest * pr); + + +/** + * Iterate over all pending requests. + * + * @param it function to call for each request + * @param cls closure for it + */ +void +GSF_iterate_pending_requests_ (GSF_PendingRequestIterator it, void *cls); + + +/** + * Handle P2P "CONTENT" message. Checks that the message is + * well-formed and then checks if there are any pending requests for + * this content and possibly passes it on (to local clients or other + * peers). Does NOT perform migration (content caching at this peer). + * + * @param cp the other peer involved (sender or receiver, NULL + * for loopback messages where we are both sender and receiver) + * @param message the actual message + * @return GNUNET_OK if the message was well-formed, + * GNUNET_SYSERR if the message was malformed (close connection, + * do not cache under any circumstances) + */ +int +GSF_handle_p2p_content_ (struct GSF_ConnectedPeer *cp, + const struct GNUNET_MessageHeader *message); + + +/** + * Consider looking up the data in the DHT (anonymity-level permitting). + * + * @param pr the pending request to process + */ +void +GSF_dht_lookup_ (struct GSF_PendingRequest *pr); + + +/** + * Function to be called after we're done processing + * replies from the local lookup. + * + * @param cls closure + * @param pr the pending request we were processing + * @param result final datastore lookup result + */ +typedef void (*GSF_LocalLookupContinuation) (void *cls, + struct GSF_PendingRequest * pr, + enum GNUNET_BLOCK_EvaluationResult + result); + + +/** + * Look up the request in the local datastore. + * + * @param pr the pending request to process + * @param cont function to call at the end + * @param cont_cls closure for cont + */ +void +GSF_local_lookup_ (struct GSF_PendingRequest *pr, + GSF_LocalLookupContinuation cont, void *cont_cls); + + +/** + * Is the given target a legitimate peer for forwarding the given request? + * + * @param pr request + * @param target + * @return GNUNET_YES if this request could be forwarded to the given peer + */ +int +GSF_pending_request_test_target_ (struct GSF_PendingRequest *pr, + const struct GNUNET_PeerIdentity *target); + + + +/** + * Setup the subsystem. + */ +void +GSF_pending_request_init_ (void); + + +/** + * Shutdown the subsystem. + */ +void +GSF_pending_request_done_ (void); + + +#endif +/* end of gnunet-service-fs_pr.h */ diff --git a/src/fs/gnunet-service-fs_push.c b/src/fs/gnunet-service-fs_push.c new file mode 100644 index 0000000..22a76f3 --- /dev/null +++ b/src/fs/gnunet-service-fs_push.c @@ -0,0 +1,658 @@ +/* + This file is part of GNUnet. + (C) 2011 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/gnunet-service-fs_push.c + * @brief API to push content from our datastore to other peers + * ('anonymous'-content P2P migration) + * @author Christian Grothoff + */ +#include "platform.h" +#include "gnunet-service-fs.h" +#include "gnunet-service-fs_cp.h" +#include "gnunet-service-fs_indexing.h" +#include "gnunet-service-fs_push.h" + + +/** + * Maximum number of blocks we keep in memory for migration. + */ +#define MAX_MIGRATION_QUEUE 8 + +/** + * Blocks are at most migrated to this number of peers + * plus one, each time they are fetched from the database. + */ +#define MIGRATION_LIST_SIZE 2 + +/** + * How long must content remain valid for us to consider it for migration? + * If content will expire too soon, there is clearly no point in pushing + * it to other peers. This value gives the threshold for migration. Note + * that if this value is increased, the migration testcase may need to be + * adjusted as well (especially the CONTENT_LIFETIME in fs_test_lib.c). + */ +#define MIN_MIGRATION_CONTENT_LIFETIME GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 30) + + +/** + * Block that is ready for migration to other peers. Actual data is at the end of the block. + */ +struct MigrationReadyBlock +{ + + /** + * This is a doubly-linked list. + */ + struct MigrationReadyBlock *next; + + /** + * This is a doubly-linked list. + */ + struct MigrationReadyBlock *prev; + + /** + * Query for the block. + */ + GNUNET_HashCode query; + + /** + * When does this block expire? + */ + struct GNUNET_TIME_Absolute expiration; + + /** + * Peers we already forwarded this + * block to. Zero for empty entries. + */ + GNUNET_PEER_Id target_list[MIGRATION_LIST_SIZE]; + + /** + * Size of the block. + */ + size_t size; + + /** + * Number of targets already used. + */ + unsigned int used_targets; + + /** + * Type of the block. + */ + enum GNUNET_BLOCK_Type type; +}; + + +/** + * Information about a peer waiting for + * migratable data. + */ +struct MigrationReadyPeer +{ + /** + * This is a doubly-linked list. + */ + struct MigrationReadyPeer *next; + + /** + * This is a doubly-linked list. + */ + struct MigrationReadyPeer *prev; + + /** + * Handle to peer. + */ + struct GSF_ConnectedPeer *peer; + + /** + * Handle for current transmission request, + * or NULL for none. + */ + struct GSF_PeerTransmitHandle *th; + + /** + * Message we are trying to push right now (or NULL) + */ + struct PutMessage *msg; +}; + + +/** + * Head of linked list of blocks that can be migrated. + */ +static struct MigrationReadyBlock *mig_head; + +/** + * Tail of linked list of blocks that can be migrated. + */ +static struct MigrationReadyBlock *mig_tail; + +/** + * Head of linked list of peers. + */ +static struct MigrationReadyPeer *peer_head; + +/** + * Tail of linked list of peers. + */ +static struct MigrationReadyPeer *peer_tail; + +/** + * Request to datastore for migration (or NULL). + */ +static struct GNUNET_DATASTORE_QueueEntry *mig_qe; + +/** + * ID of task that collects blocks for migration. + */ +static GNUNET_SCHEDULER_TaskIdentifier mig_task; + +/** + * What is the maximum frequency at which we are allowed to + * poll the datastore for migration content? + */ +static struct GNUNET_TIME_Relative min_migration_delay; + +/** + * Size of the doubly-linked list of migration blocks. + */ +static unsigned int mig_size; + +/** + * Is this module enabled? + */ +static int enabled; + + +/** + * Delete the given migration block. + * + * @param mb block to delete + */ +static void +delete_migration_block (struct MigrationReadyBlock *mb) +{ + GNUNET_CONTAINER_DLL_remove (mig_head, mig_tail, mb); + GNUNET_PEER_decrement_rcs (mb->target_list, MIGRATION_LIST_SIZE); + mig_size--; + GNUNET_free (mb); +} + + +/** + * Find content for migration to this peer. + */ +static void +find_content (struct MigrationReadyPeer *mrp); + + +/** + * Transmit the message currently scheduled for + * transmission. + * + * @param cls the 'struct MigrationReadyPeer' + * @param buf_size number of bytes available in buf + * @param buf where to copy the message, NULL on error (peer disconnect) + * @return number of bytes copied to 'buf', can be 0 (without indicating an error) + */ +static size_t +transmit_message (void *cls, size_t buf_size, void *buf) +{ + struct MigrationReadyPeer *peer = cls; + struct PutMessage *msg; + uint16_t msize; + + peer->th = NULL; + msg = peer->msg; + peer->msg = NULL; + if (buf == NULL) + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Failed to migrate content to another peer (disconnect)\n"); + GNUNET_free (msg); + return 0; + } + msize = ntohs (msg->header.size); + GNUNET_assert (msize <= buf_size); + memcpy (buf, msg, msize); + GNUNET_free (msg); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Pushing %u bytes to another peer\n", + msize); + find_content (peer); + return msize; +} + + +/** + * Send the given block to the given peer. + * + * @param peer target peer + * @param block the block + * @return GNUNET_YES if the block was deleted (!) + */ +static int +transmit_content (struct MigrationReadyPeer *peer, + struct MigrationReadyBlock *block) +{ + size_t msize; + struct PutMessage *msg; + unsigned int i; + struct GSF_PeerPerformanceData *ppd; + int ret; + + ppd = GSF_get_peer_performance_data_ (peer->peer); + GNUNET_assert (NULL == peer->th); + msize = sizeof (struct PutMessage) + block->size; + msg = GNUNET_malloc (msize); + msg->header.type = htons (GNUNET_MESSAGE_TYPE_FS_PUT); + msg->header.size = htons (msize); + msg->type = htonl (block->type); + msg->expiration = GNUNET_TIME_absolute_hton (block->expiration); + memcpy (&msg[1], &block[1], block->size); + peer->msg = msg; + for (i = 0; i < MIGRATION_LIST_SIZE; i++) + { + if (block->target_list[i] == 0) + { + block->target_list[i] = ppd->pid; + GNUNET_PEER_change_rc (block->target_list[i], 1); + break; + } + } + if (MIGRATION_LIST_SIZE == i) + { + delete_migration_block (block); + ret = GNUNET_YES; + } + else + { + ret = GNUNET_NO; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Asking for transmission of %u bytes for migration\n", msize); + peer->th = GSF_peer_transmit_ (peer->peer, GNUNET_NO, 0 /* priority */ , + GNUNET_TIME_UNIT_FOREVER_REL, msize, + &transmit_message, peer); + return ret; +} + + +/** + * Count the number of peers this block has + * already been forwarded to. + * + * @param block the block + * @return number of times block was forwarded + */ +static unsigned int +count_targets (struct MigrationReadyBlock *block) +{ + unsigned int i; + + for (i = 0; i < MIGRATION_LIST_SIZE; i++) + if (block->target_list[i] == 0) + return i; + return i; +} + + +/** + * Check if sending this block to this peer would + * be a good idea. + * + * @param peer target peer + * @param block the block + * @return score (>= 0: feasible, negative: infeasible) + */ +static long +score_content (struct MigrationReadyPeer *peer, + struct MigrationReadyBlock *block) +{ + unsigned int i; + struct GSF_PeerPerformanceData *ppd; + struct GNUNET_PeerIdentity id; + uint32_t dist; + + ppd = GSF_get_peer_performance_data_ (peer->peer); + for (i = 0; i < MIGRATION_LIST_SIZE; i++) + if (block->target_list[i] == ppd->pid) + return -1; + GNUNET_assert (0 != ppd->pid); + GNUNET_PEER_resolve (ppd->pid, &id); + dist = GNUNET_CRYPTO_hash_distance_u32 (&block->query, &id.hashPubKey); + /* closer distance, higher score: */ + return UINT32_MAX - dist; +} + + +/** + * If the migration task is not currently running, consider + * (re)scheduling it with the appropriate delay. + */ +static void +consider_gathering (void); + + +/** + * Find content for migration to this peer. + * + * @param mrp peer to find content for + */ +static void +find_content (struct MigrationReadyPeer *mrp) +{ + struct MigrationReadyBlock *pos; + long score; + long best_score; + struct MigrationReadyBlock *best; + + GNUNET_assert (NULL == mrp->th); + best = NULL; + best_score = -1; + pos = mig_head; + while (NULL != pos) + { + score = score_content (mrp, pos); + if (score > best_score) + { + best_score = score; + best = pos; + } + pos = pos->next; + } + if (NULL == best) + { + if (mig_size < MAX_MIGRATION_QUEUE) + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "No content found for pushing, waiting for queue to fill\n"); + return; /* will fill up eventually... */ + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "No suitable content found, purging content from full queue\n"); + /* failed to find migration target AND + * queue is full, purge most-forwarded + * block from queue to make room for more */ + pos = mig_head; + while (NULL != pos) + { + score = count_targets (pos); + if (score >= best_score) + { + best_score = score; + best = pos; + } + pos = pos->next; + } + GNUNET_assert (NULL != best); + delete_migration_block (best); + consider_gathering (); + return; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Preparing to push best content to peer\n"); + transmit_content (mrp, best); +} + + +/** + * Task that is run periodically to obtain blocks for content + * migration + * + * @param cls unused + * @param tc scheduler context (also unused) + */ +static void +gather_migration_blocks (void *cls, + const struct GNUNET_SCHEDULER_TaskContext *tc); + + +/** + * If the migration task is not currently running, consider + * (re)scheduling it with the appropriate delay. + */ +static void +consider_gathering () +{ + struct GNUNET_TIME_Relative delay; + + if (GSF_dsh == NULL) + return; + if (mig_qe != NULL) + return; + if (mig_task != GNUNET_SCHEDULER_NO_TASK) + return; + if (mig_size >= MAX_MIGRATION_QUEUE) + return; + delay = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, mig_size); + delay = GNUNET_TIME_relative_divide (delay, MAX_MIGRATION_QUEUE); + delay = GNUNET_TIME_relative_max (delay, min_migration_delay); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Scheduling gathering task (queue size: %u)\n", mig_size); + mig_task = + GNUNET_SCHEDULER_add_delayed (delay, &gather_migration_blocks, NULL); +} + + +/** + * Process content offered for migration. + * + * @param cls closure + * @param key key for the content + * @param size number of bytes in data + * @param data content stored + * @param type type of the content + * @param priority priority of the content + * @param anonymity anonymity-level for the content + * @param expiration expiration time for the content + * @param uid unique identifier for the datum; + * maybe 0 if no unique identifier is available + */ +static void +process_migration_content (void *cls, const GNUNET_HashCode * key, size_t size, + const void *data, enum GNUNET_BLOCK_Type type, + uint32_t priority, uint32_t anonymity, + struct GNUNET_TIME_Absolute expiration, uint64_t uid) +{ + struct MigrationReadyBlock *mb; + struct MigrationReadyPeer *pos; + + mig_qe = NULL; + if (key == NULL) + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "No content found for migration...\n"); + consider_gathering (); + return; + } + if (GNUNET_TIME_absolute_get_remaining (expiration).rel_value < + MIN_MIGRATION_CONTENT_LIFETIME.rel_value) + { + /* content will expire soon, don't bother */ + consider_gathering (); + return; + } + if (type == GNUNET_BLOCK_TYPE_FS_ONDEMAND) + { + if (GNUNET_OK != + GNUNET_FS_handle_on_demand_block (key, size, data, type, priority, + anonymity, expiration, uid, + &process_migration_content, NULL)) + consider_gathering (); + return; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Retrieved block `%s' of type %u for migration (queue size: %u/%u)\n", + GNUNET_h2s (key), type, mig_size + 1, MAX_MIGRATION_QUEUE); + mb = GNUNET_malloc (sizeof (struct MigrationReadyBlock) + size); + mb->query = *key; + mb->expiration = expiration; + mb->size = size; + mb->type = type; + memcpy (&mb[1], data, size); + GNUNET_CONTAINER_DLL_insert_after (mig_head, mig_tail, mig_tail, mb); + mig_size++; + pos = peer_head; + while (pos != NULL) + { + if (NULL == pos->th) + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Preparing to push best content to peer\n"); + if (GNUNET_YES == transmit_content (pos, mb)) + break; /* 'mb' was freed! */ + } + pos = pos->next; + } + consider_gathering (); +} + + +/** + * Task that is run periodically to obtain blocks for content + * migration + * + * @param cls unused + * @param tc scheduler context (also unused) + */ +static void +gather_migration_blocks (void *cls, + const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + mig_task = GNUNET_SCHEDULER_NO_TASK; + if (mig_size >= MAX_MIGRATION_QUEUE) + return; + if (GSF_dsh != NULL) + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Asking datastore for content for replication (queue size: %u)\n", + mig_size); + mig_qe = + GNUNET_DATASTORE_get_for_replication (GSF_dsh, 0, UINT_MAX, + GNUNET_TIME_UNIT_FOREVER_REL, + &process_migration_content, NULL); + if (NULL == mig_qe) + consider_gathering (); + } +} + + +/** + * A peer connected to us. Start pushing content + * to this peer. + * + * @param peer handle for the peer that connected + */ +void +GSF_push_start_ (struct GSF_ConnectedPeer *peer) +{ + struct MigrationReadyPeer *mrp; + + if (GNUNET_YES != enabled) + return; + mrp = GNUNET_malloc (sizeof (struct MigrationReadyPeer)); + mrp->peer = peer; + find_content (mrp); + GNUNET_CONTAINER_DLL_insert (peer_head, peer_tail, mrp); +} + + +/** + * A peer disconnected from us. Stop pushing content + * to this peer. + * + * @param peer handle for the peer that disconnected + */ +void +GSF_push_stop_ (struct GSF_ConnectedPeer *peer) +{ + struct MigrationReadyPeer *pos; + + pos = peer_head; + while (pos != NULL) + { + if (pos->peer == peer) + { + GNUNET_CONTAINER_DLL_remove (peer_head, peer_tail, pos); + if (NULL != pos->th) + { + GSF_peer_transmit_cancel_ (pos->th); + pos->th = NULL; + } + if (NULL != pos->msg) + { + GNUNET_free (pos->msg); + pos->msg = NULL; + } + GNUNET_free (pos); + return; + } + pos = pos->next; + } +} + + +/** + * Setup the module. + */ +void +GSF_push_init_ () +{ + enabled = + GNUNET_CONFIGURATION_get_value_yesno (GSF_cfg, "FS", "CONTENT_PUSHING"); + if (GNUNET_YES != enabled) + return; + + if (GNUNET_OK != + GNUNET_CONFIGURATION_get_value_time (GSF_cfg, "fs", "MIN_MIGRATION_DELAY", + &min_migration_delay)) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _ + ("Invalid value specified for option `%s' in section `%s', content pushing disabled\n"), + "MIN_MIGRATION_DELAY", "fs"); + return; + } + consider_gathering (); +} + + +/** + * Shutdown the module. + */ +void +GSF_push_done_ () +{ + if (GNUNET_SCHEDULER_NO_TASK != mig_task) + { + GNUNET_SCHEDULER_cancel (mig_task); + mig_task = GNUNET_SCHEDULER_NO_TASK; + } + if (NULL != mig_qe) + { + GNUNET_DATASTORE_cancel (mig_qe); + mig_qe = NULL; + } + while (NULL != mig_head) + delete_migration_block (mig_head); + GNUNET_assert (0 == mig_size); +} + +/* end of gnunet-service-fs_push.c */ diff --git a/src/fs/gnunet-service-fs_push.h b/src/fs/gnunet-service-fs_push.h new file mode 100644 index 0000000..7967b04 --- /dev/null +++ b/src/fs/gnunet-service-fs_push.h @@ -0,0 +1,66 @@ +/* + This file is part of GNUnet. + (C) 2011 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/gnunet-service-fs_push.h + * @brief support for pushing out content + * @author Christian Grothoff + */ +#ifndef GNUNET_SERVICE_FS_PUSH_H +#define GNUNET_SERVICE_FS_PUSH_H + +#include "gnunet-service-fs.h" + + +/** + * Setup the module. + */ +void +GSF_push_init_ (void); + + +/** + * Shutdown the module. + */ +void +GSF_push_done_ (void); + + +/** + * A peer connected to us or we are now again allowed to push content. + * Start pushing content to this peer. + * + * @param peer handle for the peer that connected + */ +void +GSF_push_start_ (struct GSF_ConnectedPeer *peer); + + +/** + * A peer disconnected from us or asked us to stop pushing content for + * a while. Stop pushing content to this peer. + * + * @param peer handle for the peer that disconnected + */ +void +GSF_push_stop_ (struct GSF_ConnectedPeer *peer); + + +#endif diff --git a/src/fs/gnunet-service-fs_put.c b/src/fs/gnunet-service-fs_put.c new file mode 100644 index 0000000..3ac6713 --- /dev/null +++ b/src/fs/gnunet-service-fs_put.c @@ -0,0 +1,238 @@ +/* + This file is part of GNUnet. + (C) 2011 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/gnunet-service-fs_put.c + * @brief API to PUT zero-anonymity index data from our datastore into the DHT + * @author Christian Grothoff + */ +#include "platform.h" +#include "gnunet-service-fs.h" +#include "gnunet-service-fs_put.h" + + +/** + * How often do we at most PUT content into the DHT? + */ +#define MAX_DHT_PUT_FREQ GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 5) + + +/** + * Context for each zero-anonymity iterator. + */ +struct PutOperator +{ + + /** + * Request to datastore for DHT PUTs (or NULL). + */ + struct GNUNET_DATASTORE_QueueEntry *dht_qe; + + /** + * Type we request from the datastore. + */ + enum GNUNET_BLOCK_Type dht_put_type; + + /** + * ID of task that collects blocks for DHT PUTs. + */ + GNUNET_SCHEDULER_TaskIdentifier dht_task; + + /** + * How many entires with zero anonymity of our type do we currently + * estimate to have in the database? + */ + uint64_t zero_anonymity_count_estimate; + + /** + * Current offset when iterating the database. + */ + uint64_t current_offset; +}; + + +/** + * ANY-terminated list of our operators (one per type + * of block that we're putting into the DHT). + */ +static struct PutOperator operators[] = { + {NULL, GNUNET_BLOCK_TYPE_FS_KBLOCK, 0, 0, 0}, + {NULL, GNUNET_BLOCK_TYPE_FS_SBLOCK, 0, 0, 0}, + {NULL, GNUNET_BLOCK_TYPE_FS_NBLOCK, 0, 0, 0}, + {NULL, GNUNET_BLOCK_TYPE_ANY, 0, 0, 0} +}; + + +/** + * Task that is run periodically to obtain blocks for DHT PUTs. + * + * @param cls type of blocks to gather + * @param tc scheduler context (unused) + */ +static void +gather_dht_put_blocks (void *cls, + const struct GNUNET_SCHEDULER_TaskContext *tc); + + +/** + * Task that is run periodically to obtain blocks for DHT PUTs. + * + * @param cls type of blocks to gather + * @param tc scheduler context (unused) + */ +static void +delay_dht_put_blocks (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct PutOperator *po = cls; + struct GNUNET_TIME_Relative delay; + + po->dht_task = GNUNET_SCHEDULER_NO_TASK; + if (tc != NULL && 0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN)) + return; + if (po->zero_anonymity_count_estimate > 0) + { + delay = + GNUNET_TIME_relative_divide (GNUNET_DHT_DEFAULT_REPUBLISH_FREQUENCY, + po->zero_anonymity_count_estimate); + delay = GNUNET_TIME_relative_min (delay, MAX_DHT_PUT_FREQ); + } + else + { + /* if we have NO zero-anonymity content yet, wait 5 minutes for some to + * (hopefully) appear */ + delay = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 5); + } + po->dht_task = + GNUNET_SCHEDULER_add_delayed (delay, &gather_dht_put_blocks, po); +} + + +/** + * Store content in DHT. + * + * @param cls closure + * @param key key for the content + * @param size number of bytes in data + * @param data content stored + * @param type type of the content + * @param priority priority of the content + * @param anonymity anonymity-level for the content + * @param expiration expiration time for the content + * @param uid unique identifier for the datum; + * maybe 0 if no unique identifier is available + */ +static void +process_dht_put_content (void *cls, const GNUNET_HashCode * key, size_t size, + const void *data, enum GNUNET_BLOCK_Type type, + uint32_t priority, uint32_t anonymity, + struct GNUNET_TIME_Absolute expiration, uint64_t uid) +{ + struct PutOperator *po = cls; + + po->dht_qe = NULL; + if (key == NULL) + { + po->zero_anonymity_count_estimate = po->current_offset - 1; + po->current_offset = 0; + po->dht_task = GNUNET_SCHEDULER_add_now (&delay_dht_put_blocks, po); + return; + } + po->zero_anonymity_count_estimate = + GNUNET_MAX (po->current_offset, po->zero_anonymity_count_estimate); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Retrieved block `%s' of type %u for DHT PUT\n", GNUNET_h2s (key), + type); + GNUNET_DHT_put (GSF_dht, key, 5 /* DEFAULT_PUT_REPLICATION */ , + GNUNET_DHT_RO_DEMULTIPLEX_EVERYWHERE, type, size, data, + expiration, GNUNET_TIME_UNIT_FOREVER_REL, + &delay_dht_put_blocks, po); +} + + +/** + * Task that is run periodically to obtain blocks for DHT PUTs. + * + * @param cls type of blocks to gather + * @param tc scheduler context (unused) + */ +static void +gather_dht_put_blocks (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct PutOperator *po = cls; + + po->dht_task = GNUNET_SCHEDULER_NO_TASK; + if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN)) + return; + po->dht_qe = + GNUNET_DATASTORE_get_zero_anonymity (GSF_dsh, po->current_offset++, 0, + UINT_MAX, + GNUNET_TIME_UNIT_FOREVER_REL, + po->dht_put_type, + &process_dht_put_content, po); + if (NULL == po->dht_qe) + po->dht_task = GNUNET_SCHEDULER_add_now (&delay_dht_put_blocks, po); +} + + +/** + * Setup the module. + */ +void +GSF_put_init_ () +{ + unsigned int i; + + i = 0; + while (operators[i].dht_put_type != GNUNET_BLOCK_TYPE_ANY) + { + operators[i].dht_task = + GNUNET_SCHEDULER_add_now (&gather_dht_put_blocks, &operators[i]); + i++; + } +} + + +/** + * Shutdown the module. + */ +void +GSF_put_done_ () +{ + struct PutOperator *po; + unsigned int i; + + i = 0; + while ((po = &operators[i])->dht_put_type != GNUNET_BLOCK_TYPE_ANY) + { + if (GNUNET_SCHEDULER_NO_TASK != po->dht_task) + { + GNUNET_SCHEDULER_cancel (po->dht_task); + po->dht_task = GNUNET_SCHEDULER_NO_TASK; + } + if (NULL != po->dht_qe) + { + GNUNET_DATASTORE_cancel (po->dht_qe); + po->dht_qe = NULL; + } + i++; + } +} + +/* end of gnunet-service-fs_put.c */ diff --git a/src/fs/gnunet-service-fs_put.h b/src/fs/gnunet-service-fs_put.h new file mode 100644 index 0000000..59b1f83 --- /dev/null +++ b/src/fs/gnunet-service-fs_put.h @@ -0,0 +1,46 @@ +/* + This file is part of GNUnet. + (C) 2011 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/gnunet-service-fs_put.h + * @brief support for putting content into the DHT + * @author Christian Grothoff + */ +#ifndef GNUNET_SERVICE_FS_PUT_H +#define GNUNET_SERVICE_FS_PUT_H + +#include "gnunet-service-fs.h" + + +/** + * Setup the module. + */ +void +GSF_put_init_ (void); + + +/** + * Shutdown the module. + */ +void +GSF_put_done_ (void); + + +#endif diff --git a/src/fs/gnunet-unindex.c b/src/fs/gnunet-unindex.c new file mode 100644 index 0000000..3e8308d --- /dev/null +++ b/src/fs/gnunet-unindex.c @@ -0,0 +1,180 @@ +/* + This file is part of GNUnet. + (C) 2001, 2002, 2004, 2005, 2006, 2007, 2009 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ +/** + * @file fs/gnunet-unindex.c + * @brief unindex files published on GNUnet + * @author Christian Grothoff + * @author Krista Bennett + * @author James Blackwell + * @author Igor Wronsky + */ +#include "platform.h" +#include "gnunet_fs_service.h" + +static int ret; + +static int verbose; + +static const struct GNUNET_CONFIGURATION_Handle *cfg; + +static struct GNUNET_FS_Handle *ctx; + +static struct GNUNET_FS_UnindexContext *uc; + + +static void +cleanup_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + GNUNET_FS_stop (ctx); + ctx = NULL; +} + + +static void +shutdown_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct GNUNET_FS_UnindexContext *u; + + if (uc != NULL) + { + u = uc; + uc = NULL; + GNUNET_FS_unindex_stop (u); + } +} + +/** + * Called by FS client to give information about the progress of an + * operation. + * + * @param cls closure + * @param info details about the event, specifying the event type + * and various bits about the event + * @return client-context (for the next progress call + * for this operation; should be set to NULL for + * SUSPEND and STOPPED events). The value returned + * will be passed to future callbacks in the respective + * field in the GNUNET_FS_ProgressInfo struct. + */ +static void * +progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *info) +{ + char *s; + + switch (info->status) + { + case GNUNET_FS_STATUS_UNINDEX_START: + break; + case GNUNET_FS_STATUS_UNINDEX_PROGRESS: + if (verbose) + { + s = GNUNET_STRINGS_relative_time_to_string (info->value.unindex.eta); + FPRINTF (stdout, _("Unindexing at %llu/%llu (%s remaining)\n"), + (unsigned long long) info->value.unindex.completed, + (unsigned long long) info->value.unindex.size, s); + GNUNET_free (s); + } + break; + case GNUNET_FS_STATUS_UNINDEX_ERROR: + FPRINTF (stderr, _("Error unindexing: %s.\n"), + info->value.unindex.specifics.error.message); + GNUNET_SCHEDULER_shutdown (); + break; + case GNUNET_FS_STATUS_UNINDEX_COMPLETED: + FPRINTF (stdout, "%s", _("Unindexing done.\n")); + GNUNET_SCHEDULER_shutdown (); + break; + case GNUNET_FS_STATUS_UNINDEX_STOPPED: + GNUNET_SCHEDULER_add_continuation (&cleanup_task, NULL, + GNUNET_SCHEDULER_REASON_PREREQ_DONE); + break; + default: + FPRINTF (stderr, _("Unexpected status: %d\n"), info->status); + break; + } + return NULL; +} + + +/** + * Main function that will be run by the scheduler. + * + * @param cls closure + * @param args remaining command-line arguments + * @param cfgfile name of the configuration file used (for saving, can be NULL!) + * @param c configuration + */ +static void +run (void *cls, char *const *args, const char *cfgfile, + const struct GNUNET_CONFIGURATION_Handle *c) +{ + /* check arguments */ + if ((args[0] == NULL) || (args[1] != NULL)) + { + printf (_("You must specify one and only one filename for unindexing.\n")); + ret = -1; + return; + } + cfg = c; + ctx = + GNUNET_FS_start (cfg, "gnunet-unindex", &progress_cb, NULL, + GNUNET_FS_FLAGS_NONE, GNUNET_FS_OPTIONS_END); + if (NULL == ctx) + { + FPRINTF (stderr, _("Could not initialize `%s' subsystem.\n"), "FS"); + ret = 1; + return; + } + uc = GNUNET_FS_unindex_start (ctx, args[0], NULL); + if (NULL == uc) + { + FPRINTF (stderr, "%s", _("Could not start unindex operation.\n")); + GNUNET_FS_stop (ctx); + return; + } + GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_FOREVER_REL, &shutdown_task, + NULL); +} + + +/** + * The main function to unindex content. + * + * @param argc number of arguments from the command line + * @param argv command line arguments + * @return 0 ok, 1 on error + */ +int +main (int argc, char *const *argv) +{ + static const struct GNUNET_GETOPT_CommandLineOption options[] = { + {'V', "verbose", NULL, + gettext_noop ("be verbose (print progress information)"), + 0, &GNUNET_GETOPT_set_one, &verbose}, + GNUNET_GETOPT_OPTION_END + }; + return (GNUNET_OK == + GNUNET_PROGRAM_run (argc, argv, "gnunet-unindex [OPTIONS] FILENAME", + gettext_noop + ("Unindex a file that was previously indexed with gnunet-publish."), + options, &run, NULL)) ? ret : 1; +} + +/* end of gnunet-unindex.c */ diff --git a/src/fs/perf_gnunet_service_fs_p2p.c b/src/fs/perf_gnunet_service_fs_p2p.c new file mode 100644 index 0000000..32dcffa --- /dev/null +++ b/src/fs/perf_gnunet_service_fs_p2p.c @@ -0,0 +1,337 @@ +/* + This file is part of GNUnet. + (C) 2010 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/perf_gnunet_service_fs_p2p.c + * @brief profile P2P routing using simple publish + download operation + * @author Christian Grothoff + */ +#include "platform.h" +#include "fs_test_lib.h" +#include "gnunet_testing_lib.h" + +#define VERBOSE GNUNET_NO + +/** + * File-size we use for testing. + */ +#define FILESIZE (1024 * 1024 * 10) + +/** + * How long until we give up on transmitting the message? + */ +#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 30) + +#define NUM_DAEMONS 2 + +#define SEED 42 + +static struct GNUNET_FS_TestDaemon *daemons[NUM_DAEMONS]; + +static int ok; + +static struct GNUNET_TIME_Absolute start_time; + +static const char *progname; + +static void +do_stop (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + GNUNET_FS_TEST_daemons_stop (NUM_DAEMONS, daemons); +} + + +/** + * Master context for 'stat_run'. + */ +struct StatMaster +{ + struct GNUNET_STATISTICS_Handle *stat; + unsigned int daemon; + unsigned int value; +}; + +struct StatValues +{ + const char *subsystem; + const char *name; +}; + +/** + * Statistics we print out. + */ +static struct StatValues stats[] = { + {"fs", "# artificial delays introduced (ms)"}, + {"fs", "# queries forwarded"}, + {"fs", "# replies received and matched"}, + {"fs", "# results found locally"}, + {"fs", "# requests forwarded due to high load"}, + {"fs", "# requests done for free (low load)"}, + {"fs", "# requests dropped, priority insufficient"}, + {"fs", "# requests done for a price (normal load)"}, + {"fs", "# requests dropped by datastore (queue length limit)"}, + {"fs", "# P2P searches received"}, + {"fs", "# P2P searches discarded (queue length bound)"}, + {"fs", "# replies received for local clients"}, + {"fs", "# queries retransmitted to same target"}, + {"core", "# bytes decrypted"}, + {"core", "# bytes encrypted"}, + {"core", "# discarded CORE_SEND requests"}, + {"core", "# discarded CORE_SEND request bytes"}, + {"core", "# discarded lower priority CORE_SEND requests"}, + {"core", "# discarded lower priority CORE_SEND request bytes"}, + {"transport", "# bytes received via TCP"}, + {"transport", "# bytes transmitted via TCP"}, + {"datacache", "# bytes stored"}, + {NULL, NULL} +}; + + +/** + * Callback function to process statistic values. + * + * @param cls closure + * @param subsystem name of subsystem that created the statistic + * @param name the name of the datum + * @param value the current value + * @param is_persistent GNUNET_YES if the value is persistent, GNUNET_NO if not + * @return GNUNET_OK to continue, GNUNET_SYSERR to abort iteration + */ +static int +print_stat (void *cls, const char *subsystem, const char *name, uint64_t value, + int is_persistent) +{ + struct StatMaster *sm = cls; + + FPRINTF (stderr, "Peer %2u: %12s/%50s = %12llu\n", sm->daemon, subsystem, + name, (unsigned long long) value); + return GNUNET_OK; +} + + +/** + * Function that gathers stats from all daemons. + */ +static void +stat_run (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc); + + +/** + * Function called when GET operation on stats is done. + */ +static void +get_done (void *cls, int success) +{ + struct StatMaster *sm = cls; + + GNUNET_break (GNUNET_OK == success); + sm->value++; + GNUNET_SCHEDULER_add_now (&stat_run, sm); +} + + +/** + * Function that gathers stats from all daemons. + */ +static void +stat_run (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct StatMaster *sm = cls; + + if (stats[sm->value].name != NULL) + { + GNUNET_STATISTICS_get (sm->stat, +#if 0 + NULL, NULL, +#else + stats[sm->value].subsystem, stats[sm->value].name, +#endif + GNUNET_TIME_UNIT_FOREVER_REL, &get_done, &print_stat, + sm); + return; + } + GNUNET_STATISTICS_destroy (sm->stat, GNUNET_NO); + sm->value = 0; + sm->daemon++; + if (sm->daemon == NUM_DAEMONS) + { + GNUNET_free (sm); + GNUNET_SCHEDULER_add_now (&do_stop, NULL); + return; + } + sm->stat = + GNUNET_STATISTICS_create ("<driver>", + GNUNET_FS_TEST_get_configuration (daemons, + sm->daemon)); + GNUNET_SCHEDULER_add_now (&stat_run, sm); +} + + +static void +do_report (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct GNUNET_TIME_Relative del; + char *fancy; + struct StatMaster *sm; + + if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_PREREQ_DONE)) + { + del = GNUNET_TIME_absolute_get_duration (start_time); + if (del.rel_value == 0) + del.rel_value = 1; + fancy = + GNUNET_STRINGS_byte_size_fancy (((unsigned long long) FILESIZE) * + 1000LL / del.rel_value); + FPRINTF (stdout, "Download speed was %s/s\n", fancy); + GNUNET_free (fancy); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Finished download, shutting down\n", + (unsigned long long) FILESIZE); + sm = GNUNET_malloc (sizeof (struct StatMaster)); + sm->stat = + GNUNET_STATISTICS_create ("<driver>", + GNUNET_FS_TEST_get_configuration (daemons, + sm->daemon)); + GNUNET_SCHEDULER_add_now (&stat_run, sm); + } + else + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Timeout during download, shutting down with error\n"); + ok = 1; + GNUNET_SCHEDULER_add_now (&do_stop, NULL); + } +} + + +static void +do_download (void *cls, const struct GNUNET_FS_Uri *uri) +{ + int anonymity; + + if (NULL == uri) + { + GNUNET_FS_TEST_daemons_stop (NUM_DAEMONS, daemons); + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Timeout during upload attempt, shutting down with error\n"); + ok = 1; + return; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Downloading %llu bytes\n", + (unsigned long long) FILESIZE); + start_time = GNUNET_TIME_absolute_get (); + if (NULL != strstr (progname, "dht")) + anonymity = 0; + else + anonymity = 1; + GNUNET_FS_TEST_download (daemons[0], TIMEOUT, anonymity, SEED, uri, VERBOSE, + &do_report, NULL); +} + + +static void +do_publish (void *cls, const char *emsg) +{ + int do_index; + int anonymity; + + if (NULL != emsg) + { + GNUNET_FS_TEST_daemons_stop (NUM_DAEMONS, daemons); + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Error trying to connect: %s\n", emsg); + ok = 1; + return; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Publishing %llu bytes\n", + (unsigned long long) FILESIZE); + if (NULL != strstr (progname, "index")) + do_index = GNUNET_YES; + else + do_index = GNUNET_NO; + if (NULL != strstr (progname, "dht")) + anonymity = 0; + else + anonymity = 1; + + GNUNET_FS_TEST_publish (daemons[NUM_DAEMONS - 1], TIMEOUT, anonymity, + do_index, FILESIZE, SEED, VERBOSE, &do_download, + NULL); +} + + +static void +do_connect (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct GNUNET_TESTING_PeerGroup *pg; + + GNUNET_assert (0 != (tc->reason & GNUNET_SCHEDULER_REASON_PREREQ_DONE)); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Daemons started, will now try to connect them\n"); + pg = GNUNET_FS_TEST_get_group (daemons); + GNUNET_break ((NUM_DAEMONS - 1) * 2 == + (GNUNET_TESTING_create_topology + (pg, GNUNET_TESTING_TOPOLOGY_LINE, + GNUNET_TESTING_TOPOLOGY_NONE, NULL))); + GNUNET_TESTING_connect_topology (pg, GNUNET_TESTING_TOPOLOGY_LINE, + GNUNET_TESTING_TOPOLOGY_OPTION_NONE, 0.0, + TIMEOUT, NUM_DAEMONS, &do_publish, NULL); +} + + +static void +run (void *cls, char *const *args, const char *cfgfile, + const struct GNUNET_CONFIGURATION_Handle *cfg) +{ + GNUNET_FS_TEST_daemons_start ("fs_test_lib_data.conf", TIMEOUT, NUM_DAEMONS, + daemons, &do_connect, NULL); +} + + +int +main (int argc, char *argv[]) +{ + char *const argvx[] = { + "perf-gnunet-service-fs-p2p", + "-c", + "fs_test_lib_data.conf", +#if VERBOSE + "-L", "DEBUG", +#endif + NULL + }; + struct GNUNET_GETOPT_CommandLineOption options[] = { + GNUNET_GETOPT_OPTION_END + }; + progname = argv[0]; + GNUNET_DISK_directory_remove ("/tmp/gnunet-test-fs-lib/"); + GNUNET_log_setup ("perf_gnunet_service_fs_p2p_index", +#if VERBOSE + "DEBUG", +#else + "WARNING", +#endif + NULL); + GNUNET_PROGRAM_run ((sizeof (argvx) / sizeof (char *)) - 1, argvx, + "perf-gnunet-service-fs-p2p-index", "nohelp", options, + &run, NULL); + GNUNET_DISK_directory_remove ("/tmp/gnunet-test-fs-lib/"); + return ok; +} + +/* end of perf_gnunet_service_fs_p2p.c */ diff --git a/src/fs/perf_gnunet_service_fs_p2p_trust.c b/src/fs/perf_gnunet_service_fs_p2p_trust.c new file mode 100644 index 0000000..c412e84 --- /dev/null +++ b/src/fs/perf_gnunet_service_fs_p2p_trust.c @@ -0,0 +1,418 @@ +/* + This file is part of GNUnet. + (C) 2010, 2011 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/perf_gnunet_service_fs_p2p_trust.c + * @brief profile P2P routing trust mechanism. Creates + * a clique of NUM_DAEMONS (i.e. 3) where two + * peers share (seed) different files and download + * them from each other while all the other peers + * just "leach" those files. Ideally, the seeders + * "learn" that they contribute (to each other), + * and give the other seeder higher priority; + * naturally, this only happens nicely for larger + * files; finally, once the seeders are done, the + * leachers should see fast download rates as well. + * @author Christian Grothoff + * + * Sample output: + * - 10 MB, 3 peers, with delays: + * Download speed of type `seeder 1' was 757 KiB/s + * Download speed of type `seeder 2' was 613 KiB/s + * Download speed of type `leach` was 539 KiB/s + * + * - 10 MB, 3 peers, without delays: + * Download speed of type `seeder 1' was 1784 KiB/s + * Download speed of type `seeder 2' was 1604 KiB/s + * Download speed of type `leach` was 1384 KiB/s + */ +#include "platform.h" +#include "fs_test_lib.h" +#include "gnunet_testing_lib.h" + +#define VERBOSE GNUNET_NO + +/** + * File-size we use for testing. + */ +#define FILESIZE (1024 * 1024 * 1) + +/** + * How long until we give up on transmitting the message? + */ +#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 30) + +/** + * Number of daemons in clique, must be at least 3 (!). + */ +#define NUM_DAEMONS 3 + +/** + * Seed for first file on offer. + */ +#define SEED1 42 + +/** + * Seed for second file on offer. + */ +#define SEED2 43 + +static struct GNUNET_FS_TestDaemon *daemons[NUM_DAEMONS]; + +static int ok; + +static struct GNUNET_TIME_Absolute start_time; + +static const char *progname; + +static struct GNUNET_FS_Uri *uri1; + +static struct GNUNET_FS_Uri *uri2; + +static void +do_stop (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + GNUNET_FS_TEST_daemons_stop (NUM_DAEMONS, daemons); +} + + +/** + * Master context for 'stat_run'. + */ +struct StatMaster +{ + struct GNUNET_STATISTICS_Handle *stat; + unsigned int daemon; + unsigned int value; +}; + +struct StatValues +{ + const char *subsystem; + const char *name; +}; + +/** + * Statistics we print out. + */ +static struct StatValues stats[] = { + {"fs", "# artificial delays introduced (ms)"}, + {"fs", "# queries forwarded"}, + {"fs", "# replies received and matched"}, + {"fs", "# results found locally"}, + {"fs", "# requests forwarded due to high load"}, + {"fs", "# requests done for free (low load)"}, + {"fs", "# requests dropped, priority insufficient"}, + {"fs", "# requests done for a price (normal load)"}, + {"fs", "# requests dropped by datastore (queue length limit)"}, + {"fs", "# P2P searches received"}, + {"fs", "# P2P searches discarded (queue length bound)"}, + {"fs", "# replies received for local clients"}, + {"fs", "# queries retransmitted to same target"}, + {"core", "# bytes decrypted"}, + {"core", "# bytes encrypted"}, + {"core", "# discarded CORE_SEND requests"}, + {"core", "# discarded lower priority CORE_SEND requests"}, + {"transport", "# bytes received via TCP"}, + {"transport", "# bytes transmitted via TCP"}, + {"datacache", "# bytes stored"}, + {NULL, NULL} +}; + + +/** + * Callback function to process statistic values. + * + * @param cls closure + * @param subsystem name of subsystem that created the statistic + * @param name the name of the datum + * @param value the current value + * @param is_persistent GNUNET_YES if the value is persistent, GNUNET_NO if not + * @return GNUNET_OK to continue, GNUNET_SYSERR to abort iteration + */ +static int +print_stat (void *cls, const char *subsystem, const char *name, uint64_t value, + int is_persistent) +{ + struct StatMaster *sm = cls; + + FPRINTF (stderr, "Peer %2u: %12s/%50s = %12llu\n", sm->daemon, subsystem, + name, (unsigned long long) value); + return GNUNET_OK; +} + + +/** + * Function that gathers stats from all daemons. + */ +static void +stat_run (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc); + + +/** + * Function called when GET operation on stats is done. + */ +static void +get_done (void *cls, int success) +{ + struct StatMaster *sm = cls; + + GNUNET_break (GNUNET_OK == success); + sm->value++; + GNUNET_SCHEDULER_add_now (&stat_run, sm); +} + + +/** + * Function that gathers stats from all daemons. + */ +static void +stat_run (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct StatMaster *sm = cls; + + if (stats[sm->value].name != NULL) + { + GNUNET_STATISTICS_get (sm->stat, +#if 0 + NULL, NULL, +#else + stats[sm->value].subsystem, stats[sm->value].name, +#endif + GNUNET_TIME_UNIT_FOREVER_REL, &get_done, &print_stat, + sm); + return; + } + GNUNET_STATISTICS_destroy (sm->stat, GNUNET_NO); + sm->value = 0; + sm->daemon++; + if (sm->daemon == NUM_DAEMONS) + { + GNUNET_free (sm); + GNUNET_SCHEDULER_add_now (&do_stop, NULL); + return; + } + sm->stat = + GNUNET_STATISTICS_create ("<driver>", + GNUNET_FS_TEST_get_configuration (daemons, + sm->daemon)); + GNUNET_SCHEDULER_add_now (&stat_run, sm); +} + + +static void +do_report (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + static int download_counter; + const char *type = cls; + struct GNUNET_TIME_Relative del; + char *fancy; + struct StatMaster *sm; + + if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_PREREQ_DONE)) + { + del = GNUNET_TIME_absolute_get_duration (start_time); + if (del.rel_value == 0) + del.rel_value = 1; + fancy = + GNUNET_STRINGS_byte_size_fancy (((unsigned long long) FILESIZE) * + 1000LL / del.rel_value); + FPRINTF (stderr, "Download speed of type `%s' was %s/s\n", type, fancy); + GNUNET_free (fancy); + if (NUM_DAEMONS != ++download_counter) + return; /* more downloads to come */ + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Finished all downloads, shutting down\n", + (unsigned long long) FILESIZE); + sm = GNUNET_malloc (sizeof (struct StatMaster)); + sm->stat = + GNUNET_STATISTICS_create ("<driver>", + GNUNET_FS_TEST_get_configuration (daemons, + sm->daemon)); + GNUNET_SCHEDULER_add_now (&stat_run, sm); + } + else + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Timeout during download for type `%s', shutting down with error\n", + type); + ok = 1; + GNUNET_SCHEDULER_add_now (&do_stop, NULL); + } +} + + +static void +do_downloads (void *cls, const struct GNUNET_FS_Uri *u2) +{ + int anonymity; + unsigned int i; + + if (NULL == u2) + { + GNUNET_FS_TEST_daemons_stop (NUM_DAEMONS, daemons); + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Timeout during upload attempt, shutting down with error\n"); + ok = 1; + return; + } + uri2 = GNUNET_FS_uri_dup (u2); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Downloading %llu bytes\n", + (unsigned long long) FILESIZE); + start_time = GNUNET_TIME_absolute_get (); + if (NULL != strstr (progname, "dht")) + anonymity = 0; + else + anonymity = 1; + /* (semi) leach-download(s); not true leaches since + * these peers do participate in sharing, they just + * don't have to offer anything *initially*. */ + for (i = 0; i < NUM_DAEMONS - 2; i++) + GNUNET_FS_TEST_download (daemons[i], TIMEOUT, anonymity, + 0 == (i % 2) ? SEED1 : SEED2, + 0 == (i % 2) ? uri1 : uri2, VERBOSE, &do_report, + "leach"); + /* mutual downloads of (primary) sharing peers */ + GNUNET_FS_TEST_download (daemons[NUM_DAEMONS - 2], TIMEOUT, anonymity, SEED1, + uri1, VERBOSE, &do_report, "seeder 2"); + GNUNET_FS_TEST_download (daemons[NUM_DAEMONS - 1], TIMEOUT, anonymity, SEED2, + uri2, VERBOSE, &do_report, "seeder 1"); +} + + +static void +do_publish2 (void *cls, const struct GNUNET_FS_Uri *u1) +{ + int do_index; + int anonymity; + + if (NULL == u1) + { + GNUNET_FS_TEST_daemons_stop (NUM_DAEMONS, daemons); + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Timeout during upload attempt, shutting down with error\n"); + ok = 1; + return; + } + uri1 = GNUNET_FS_uri_dup (u1); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Publishing %llu bytes\n", + (unsigned long long) FILESIZE); + if (NULL != strstr (progname, "index")) + do_index = GNUNET_YES; + else + do_index = GNUNET_NO; + if (NULL != strstr (progname, "dht")) + anonymity = 0; + else + anonymity = 1; + + GNUNET_FS_TEST_publish (daemons[NUM_DAEMONS - 2], TIMEOUT, anonymity, + do_index, FILESIZE, SEED2, VERBOSE, &do_downloads, + NULL); +} + +static void +do_publish1 (void *cls, const char *emsg) +{ + int do_index; + int anonymity; + + if (NULL != emsg) + { + GNUNET_FS_TEST_daemons_stop (NUM_DAEMONS, daemons); + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Error trying to connect: %s\n", emsg); + ok = 1; + return; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Publishing %llu bytes\n", + (unsigned long long) FILESIZE); + if (NULL != strstr (progname, "index")) + do_index = GNUNET_YES; + else + do_index = GNUNET_NO; + if (NULL != strstr (progname, "dht")) + anonymity = 0; + else + anonymity = 1; + + GNUNET_FS_TEST_publish (daemons[NUM_DAEMONS - 1], TIMEOUT, anonymity, + do_index, FILESIZE, SEED1, VERBOSE, &do_publish2, + NULL); +} + + +static void +do_connect (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct GNUNET_TESTING_PeerGroup *pg; + + GNUNET_assert (0 != (tc->reason & GNUNET_SCHEDULER_REASON_PREREQ_DONE)); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Daemons started, will now try to connect them\n"); + pg = GNUNET_FS_TEST_get_group (daemons); + GNUNET_TESTING_create_topology (pg, GNUNET_TESTING_TOPOLOGY_CLIQUE, + GNUNET_TESTING_TOPOLOGY_NONE, NULL); + GNUNET_TESTING_connect_topology (pg, GNUNET_TESTING_TOPOLOGY_CLIQUE, + GNUNET_TESTING_TOPOLOGY_OPTION_NONE, 0.0, + TIMEOUT, NUM_DAEMONS, &do_publish1, NULL); +} + + +static void +run (void *cls, char *const *args, const char *cfgfile, + const struct GNUNET_CONFIGURATION_Handle *cfg) +{ + GNUNET_FS_TEST_daemons_start ("fs_test_lib_data.conf", TIMEOUT, NUM_DAEMONS, + daemons, &do_connect, NULL); +} + + +int +main (int argc, char *argv[]) +{ + char *const argvx[] = { + "perf-gnunet-service-fs-p2p", + "-c", + "fs_test_lib_data.conf", +#if VERBOSE + "-L", "DEBUG", +#endif + NULL + }; + struct GNUNET_GETOPT_CommandLineOption options[] = { + GNUNET_GETOPT_OPTION_END + }; + progname = argv[0]; + GNUNET_DISK_directory_remove ("/tmp/gnunet-test-fs-lib/"); + GNUNET_log_setup ("perf_gnunet_service_fs_p2p_trust", +#if VERBOSE + "DEBUG", +#else + "WARNING", +#endif + NULL); + GNUNET_PROGRAM_run ((sizeof (argvx) / sizeof (char *)) - 1, argvx, + "perf-gnunet-service-fs-p2p-trust", "nohelp", options, + &run, NULL); + GNUNET_DISK_directory_remove ("/tmp/gnunet-test-fs-lib/"); + return ok; +} + +/* end of perf_gnunet_service_fs_p2p_trust.c */ diff --git a/src/fs/plugin_block_fs.c b/src/fs/plugin_block_fs.c new file mode 100644 index 0000000..9b73f24 --- /dev/null +++ b/src/fs/plugin_block_fs.c @@ -0,0 +1,322 @@ +/* + This file is part of GNUnet + (C) 2010 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/plugin_block_fs.c + * @brief blocks used for file-sharing + * @author Christian Grothoff + */ + +#include "platform.h" +#include "gnunet_block_plugin.h" +#include "block_fs.h" +#include "gnunet_signatures.h" + +#define DEBUG_FS_BLOCK GNUNET_EXTRA_LOGGING + +/** + * Number of bits we set per entry in the bloomfilter. + * Do not change! + */ +#define BLOOMFILTER_K 16 + +/** + * Function called to validate a reply or a request. For + * request evaluation, simply pass "NULL" for the reply_block. + * Note that it is assumed that the reply has already been + * matched to the key (and signatures checked) as it would + * be done with the "get_key" function. + * + * @param cls closure + * @param type block type + * @param query original query (hash) + * @param bf pointer to bloom filter associated with query; possibly updated (!) + * @param bf_mutator mutation value for bf + * @param xquery extrended query data (can be NULL, depending on type) + * @param xquery_size number of bytes in xquery + * @param reply_block response to validate + * @param reply_block_size number of bytes in reply block + * @return characterization of result + */ +static enum GNUNET_BLOCK_EvaluationResult +block_plugin_fs_evaluate (void *cls, enum GNUNET_BLOCK_Type type, + const GNUNET_HashCode * query, + struct GNUNET_CONTAINER_BloomFilter **bf, + int32_t bf_mutator, const void *xquery, + size_t xquery_size, const void *reply_block, + size_t reply_block_size) +{ + const struct SBlock *sb; + GNUNET_HashCode chash; + GNUNET_HashCode mhash; + const GNUNET_HashCode *nsid; + GNUNET_HashCode sh; + + switch (type) + { + case GNUNET_BLOCK_TYPE_FS_DBLOCK: + case GNUNET_BLOCK_TYPE_FS_IBLOCK: + if (xquery_size != 0) + { + GNUNET_break_op (0); + return GNUNET_BLOCK_EVALUATION_REQUEST_INVALID; + } + if (reply_block == NULL) + return GNUNET_BLOCK_EVALUATION_REQUEST_VALID; + return GNUNET_BLOCK_EVALUATION_OK_LAST; + case GNUNET_BLOCK_TYPE_FS_KBLOCK: + case GNUNET_BLOCK_TYPE_FS_NBLOCK: + if (xquery_size != 0) + { + GNUNET_break_op (0); + return GNUNET_BLOCK_EVALUATION_REQUEST_INVALID; + } + if (reply_block == NULL) + return GNUNET_BLOCK_EVALUATION_REQUEST_VALID; + if (NULL != bf) + { + GNUNET_CRYPTO_hash (reply_block, reply_block_size, &chash); + GNUNET_BLOCK_mingle_hash (&chash, bf_mutator, &mhash); + if (NULL != *bf) + { + if (GNUNET_YES == GNUNET_CONTAINER_bloomfilter_test (*bf, &mhash)) + return GNUNET_BLOCK_EVALUATION_OK_DUPLICATE; + } + else + { + *bf = GNUNET_CONTAINER_bloomfilter_init (NULL, 8, BLOOMFILTER_K); + } + GNUNET_CONTAINER_bloomfilter_add (*bf, &mhash); + } + return GNUNET_BLOCK_EVALUATION_OK_MORE; + case GNUNET_BLOCK_TYPE_FS_SBLOCK: + if (xquery_size != sizeof (GNUNET_HashCode)) + { + GNUNET_break_op (0); + return GNUNET_BLOCK_EVALUATION_REQUEST_INVALID; + } + if (reply_block == NULL) + return GNUNET_BLOCK_EVALUATION_REQUEST_VALID; + nsid = xquery; + if (reply_block_size < sizeof (struct SBlock)) + { + GNUNET_break_op (0); + return GNUNET_BLOCK_EVALUATION_RESULT_INVALID; + } + sb = reply_block; + GNUNET_CRYPTO_hash (&sb->subspace, + sizeof (struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded), + &sh); + if (0 != memcmp (nsid, &sh, sizeof (GNUNET_HashCode))) + { + GNUNET_log_from (GNUNET_ERROR_TYPE_WARNING, "block-fs", + _ + ("Reply mismatched in terms of namespace. Discarded.\n")); + return GNUNET_BLOCK_EVALUATION_RESULT_INVALID; + } + if (NULL != bf) + { + GNUNET_CRYPTO_hash (reply_block, reply_block_size, &chash); + GNUNET_BLOCK_mingle_hash (&chash, bf_mutator, &mhash); + if (NULL != *bf) + { + if (GNUNET_YES == GNUNET_CONTAINER_bloomfilter_test (*bf, &mhash)) + return GNUNET_BLOCK_EVALUATION_OK_DUPLICATE; + } + else + { + *bf = GNUNET_CONTAINER_bloomfilter_init (NULL, 8, BLOOMFILTER_K); + } + GNUNET_CONTAINER_bloomfilter_add (*bf, &mhash); + } + return GNUNET_BLOCK_EVALUATION_OK_MORE; + default: + return GNUNET_BLOCK_EVALUATION_TYPE_NOT_SUPPORTED; + } +} + + +/** + * Function called to obtain the key for a block. + * + * @param cls closure + * @param type block type + * @param block block to get the key for + * @param block_size number of bytes in block + * @param key set to the key (query) for the given block + * @return GNUNET_OK on success, GNUNET_SYSERR if type not supported + * (or if extracting a key from a block of this type does not work) + */ +static int +block_plugin_fs_get_key (void *cls, enum GNUNET_BLOCK_Type type, + const void *block, size_t block_size, + GNUNET_HashCode * key) +{ + const struct KBlock *kb; + const struct SBlock *sb; + const struct NBlock *nb; + + switch (type) + { + case GNUNET_BLOCK_TYPE_FS_DBLOCK: + case GNUNET_BLOCK_TYPE_FS_IBLOCK: + GNUNET_CRYPTO_hash (block, block_size, key); + return GNUNET_OK; + case GNUNET_BLOCK_TYPE_FS_KBLOCK: + if (block_size < sizeof (struct KBlock)) + { + GNUNET_break_op (0); + return GNUNET_NO; + } + kb = block; + if (block_size - sizeof (struct KBlock) != + ntohl (kb->purpose.size) - + sizeof (struct GNUNET_CRYPTO_RsaSignaturePurpose) - + sizeof (struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded)) + { + GNUNET_break_op (0); + return GNUNET_NO; + } + if (GNUNET_OK != + GNUNET_CRYPTO_rsa_verify (GNUNET_SIGNATURE_PURPOSE_FS_KBLOCK, + &kb->purpose, &kb->signature, &kb->keyspace)) + { + GNUNET_break_op (0); + return GNUNET_NO; + } + if (key != NULL) + GNUNET_CRYPTO_hash (&kb->keyspace, + sizeof (struct + GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded), + key); + return GNUNET_OK; + case GNUNET_BLOCK_TYPE_FS_SBLOCK: + if (block_size < sizeof (struct SBlock)) + { + GNUNET_break_op (0); + return GNUNET_NO; + } + sb = block; + if (block_size != + ntohl (sb->purpose.size) + sizeof (struct GNUNET_CRYPTO_RsaSignature)) + { + GNUNET_break_op (0); + return GNUNET_NO; + } + if (GNUNET_OK != + GNUNET_CRYPTO_rsa_verify (GNUNET_SIGNATURE_PURPOSE_FS_SBLOCK, + &sb->purpose, &sb->signature, &sb->subspace)) + { + GNUNET_break_op (0); + return GNUNET_NO; + } + if (key != NULL) + *key = sb->identifier; + return GNUNET_OK; + case GNUNET_BLOCK_TYPE_FS_NBLOCK: + if (block_size < sizeof (struct NBlock)) + { + GNUNET_break_op (0); + return GNUNET_NO; + } + nb = block; + if (block_size - sizeof (struct NBlock) != + ntohl (nb->ns_purpose.size) - + sizeof (struct GNUNET_CRYPTO_RsaSignaturePurpose) - + sizeof (struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded)) + { + GNUNET_break_op (0); + return GNUNET_NO; + } + if (block_size != + ntohl (nb->ksk_purpose.size) + + sizeof (struct GNUNET_CRYPTO_RsaSignature)) + { + GNUNET_break_op (0); + return GNUNET_NO; + } + if (GNUNET_OK != + GNUNET_CRYPTO_rsa_verify (GNUNET_SIGNATURE_PURPOSE_FS_NBLOCK_KSIG, + &nb->ksk_purpose, &nb->ksk_signature, + &nb->keyspace)) + { + GNUNET_break_op (0); + return GNUNET_NO; + } + if (GNUNET_OK != + GNUNET_CRYPTO_rsa_verify (GNUNET_SIGNATURE_PURPOSE_FS_NBLOCK, + &nb->ns_purpose, &nb->ns_signature, + &nb->subspace)) + { + GNUNET_break_op (0); + return GNUNET_NO; + } + /* FIXME: we used to xor ID with NSID, + * why not here? */ + if (key != NULL) + GNUNET_CRYPTO_hash (&nb->keyspace, + sizeof (struct + GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded), + key); + return GNUNET_OK; + default: + return GNUNET_SYSERR; + } +} + + +/** + * Entry point for the plugin. + */ +void * +libgnunet_plugin_block_fs_init (void *cls) +{ + static enum GNUNET_BLOCK_Type types[] = + { + GNUNET_BLOCK_TYPE_FS_DBLOCK, + GNUNET_BLOCK_TYPE_FS_IBLOCK, + GNUNET_BLOCK_TYPE_FS_KBLOCK, + GNUNET_BLOCK_TYPE_FS_SBLOCK, + GNUNET_BLOCK_TYPE_FS_NBLOCK, + GNUNET_BLOCK_TYPE_ANY /* end of list */ + }; + struct GNUNET_BLOCK_PluginFunctions *api; + + api = GNUNET_malloc (sizeof (struct GNUNET_BLOCK_PluginFunctions)); + api->evaluate = &block_plugin_fs_evaluate; + api->get_key = &block_plugin_fs_get_key; + api->types = types; + return api; +} + + +/** + * Exit point from the plugin. + */ +void * +libgnunet_plugin_block_fs_done (void *cls) +{ + struct GNUNET_TRANSPORT_PluginFunctions *api = cls; + + GNUNET_free (api); + return NULL; +} + +/* end of plugin_block_fs.c */ diff --git a/src/fs/test_fs_data.conf b/src/fs/test_fs_data.conf new file mode 100644 index 0000000..f2b4e1e --- /dev/null +++ b/src/fs/test_fs_data.conf @@ -0,0 +1,8 @@ +@INLINE@ test_fs_defaults.conf +[PATHS] +SERVICEHOME = /tmp/gnunet-test-fs-data/ +DEFAULTCONFIG = test_fs_data.conf + +[fs] +ACTIVEMIGRATION = NO + diff --git a/src/fs/test_fs_defaults.conf b/src/fs/test_fs_defaults.conf new file mode 100644 index 0000000..2bc3d26 --- /dev/null +++ b/src/fs/test_fs_defaults.conf @@ -0,0 +1,87 @@ +[PATHS] +SERVICEHOME = /tmp/gnunet-test-fs-lib/ +DEFAULTCONFIG = fs_test_lib_data.conf + +[gnunetd] +HOSTKEY = $SERVICEHOME/.hostkey + +[resolver] +PORT = 43464 +HOSTNAME = localhost + +[transport] +PORT = 43465 +PLUGINS = tcp + +[nat] +DISABLEV6 = YES +ENABLE_UPNP = NO +BEHIND_NAT = NO +ALLOW_NAT = NO +INTERNAL_ADDRESS = 127.0.0.1 +EXTERNAL_ADDRESS = 127.0.0.1 +USE_LOCALADDR = NO + +[arm] +PORT = 43466 +HOSTNAME = localhost +DEFAULTSERVICES = fs + +[datastore] +QUOTA = 100 MB + +[statistics] +PORT = 43467 +HOSTNAME = localhost + +[transport-tcp] +BINDTO = 127.0.0.1 +PORT = 43468 + +[peerinfo] +PORT = 43469 +HOSTNAME = localhost + +[ats] +WAN_QUOTA_IN = 65536 +WAN_QUOTA_OUT = 65536 + +[core] +PORT = 43470 +HOSTNAME = localhost + +[fs] +PORT = 43471 +HOSTNAME = localhost +CONTENT_CACHING = YES +CONTENT_PUSHING = YES +DELAY = YES + +[testing] +WEAKRANDOM = YES +HOSTKEYSFILE = ../../contrib/testing_hostkeys.dat + +[dhtcache] +QUOTA=65536 +DATABASE=sqlite + +[mesh] +AUTOSTART = NO + +[dns] +AUTOSTART = NO + +[nse] +AUTOSTART = NO + +[dv] +AUTOSTART = NO + +[chat] +AUTOSTART = NO + +[gns] +AUTOSTART = NO + +[vpn] +AUTOSTART = NO diff --git a/src/fs/test_fs_directory.c b/src/fs/test_fs_directory.c new file mode 100644 index 0000000..96ad29c --- /dev/null +++ b/src/fs/test_fs_directory.c @@ -0,0 +1,179 @@ +/* + This file is part of GNUnet. + (C) 2005, 2006, 2009 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/test_fs_directory.c + * @brief Test for fs_directory.c + * @author Christian Grothoff + */ + +#include "platform.h" +#include <extractor.h> +#include "gnunet_util_lib.h" +#include "gnunet_fs_service.h" +#include "fs_api.h" + +#define ABORT() { fprintf(stderr, "Error at %s:%d\n", __FILE__, __LINE__); return 1; } + +struct PCLS +{ + struct GNUNET_FS_Uri **uri; + struct GNUNET_CONTAINER_MetaData **md; + unsigned int pos; + unsigned int max; +}; + +static void +processor (void *cls, const char *filename, const struct GNUNET_FS_Uri *uri, + const struct GNUNET_CONTAINER_MetaData *md, size_t length, + const void *data) +{ + struct PCLS *p = cls; + int i; + + if (NULL == uri) + return; /* ignore directory's meta data */ + for (i = 0; i < p->max; i++) + { + if (GNUNET_CONTAINER_meta_data_test_equal (p->md[i], md) && + GNUNET_FS_uri_test_equal (p->uri[i], uri)) + { + p->pos++; + return; + } + } + FPRINTF (stderr, "Error at %s:%d\n", __FILE__, __LINE__); +} + +static int +testDirectory (unsigned int i) +{ + struct GNUNET_FS_DirectoryBuilder *db; + char *data; + size_t dlen; + struct GNUNET_FS_Uri **uris; + struct GNUNET_CONTAINER_MetaData **mds; + struct GNUNET_CONTAINER_MetaData *meta; + struct PCLS cls; + char *emsg; + int p; + int q; + char uri[512]; + char txt[128]; + int ret = 0; + struct GNUNET_TIME_Absolute start; + char *s; + + cls.max = i; + uris = GNUNET_malloc (sizeof (struct GNUNET_FS_Uri *) * i); + mds = GNUNET_malloc (sizeof (struct GNUNET_CONTAINER_MetaData *) * i); + meta = GNUNET_CONTAINER_meta_data_create (); + GNUNET_CONTAINER_meta_data_insert (meta, "<test>", EXTRACTOR_METATYPE_TITLE, + EXTRACTOR_METAFORMAT_UTF8, "text/plain", + "A title", strlen ("A title") + 1); + GNUNET_CONTAINER_meta_data_insert (meta, "<test>", + EXTRACTOR_METATYPE_AUTHOR_NAME, + EXTRACTOR_METAFORMAT_UTF8, "text/plain", + "An author", strlen ("An author") + 1); + for (p = 0; p < i; p++) + { + mds[p] = GNUNET_CONTAINER_meta_data_create (); + for (q = 0; q <= p; q++) + { + GNUNET_snprintf (txt, sizeof (txt), "%u -- %u\n", p, q); + GNUNET_CONTAINER_meta_data_insert (mds[p], "<test>", + q % EXTRACTOR_metatype_get_max (), + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", txt, strlen (txt) + 1); + } + GNUNET_snprintf (uri, sizeof (uri), + "gnunet://fs/chk/C282GG70GKK41O4551011DO413KFBVTVMQG1OG30I0K4045N0G41HAPB82G680A02JRVVFO8URVRU2F159011DO41000000022RG820.RNVVVVOOLCLK065B5D04HTNVNSIB2AI022RG8200HSLK1CO1000ATQ98824DMA2032LIMG50CG0K057NVUVG200000H000004400000.%u", + p); + emsg = NULL; + uris[p] = GNUNET_FS_uri_parse (uri, &emsg); + if (uris[p] == NULL) + { + GNUNET_CONTAINER_meta_data_destroy (mds[p]); + while (--p > 0) + { + GNUNET_CONTAINER_meta_data_destroy (mds[p]); + GNUNET_FS_uri_destroy (uris[p]); + } + GNUNET_free (mds); + GNUNET_free (uris); + GNUNET_free (emsg); + GNUNET_CONTAINER_meta_data_destroy (meta); + ABORT (); /* error in testcase */ + } + GNUNET_assert (emsg == NULL); + } + start = GNUNET_TIME_absolute_get (); + db = GNUNET_FS_directory_builder_create (meta); + for (p = 0; p < i; p++) + GNUNET_FS_directory_builder_add (db, uris[p], mds[p], NULL); + GNUNET_FS_directory_builder_finish (db, &dlen, (void **) &data); + s = GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_duration + (start)); + FPRINTF (stdout, + "Creating directory with %u entires and total size %llu took %s\n", + i, (unsigned long long) dlen, s); + GNUNET_free (s); + if (i < 100) + { + cls.pos = 0; + cls.uri = uris; + cls.md = mds; + GNUNET_FS_directory_list_contents (dlen, data, 0, &processor, &cls); + GNUNET_assert (cls.pos == i); + } + GNUNET_free (data); + GNUNET_CONTAINER_meta_data_destroy (meta); + for (p = 0; p < i; p++) + { + GNUNET_CONTAINER_meta_data_destroy (mds[p]); + GNUNET_FS_uri_destroy (uris[p]); + } + GNUNET_free (uris); + GNUNET_free (mds); + return ret; +} + + +int +main (int argc, char *argv[]) +{ + int failureCount = 0; + int i; + + GNUNET_log_setup ("test_fs_directory", +#if VERBOSE + "DEBUG", +#else + "WARNING", +#endif + NULL); + for (i = 17; i < 1000; i *= 2) + failureCount += testDirectory (i); + if (failureCount != 0) + return 1; + return 0; +} + +/* end of test_fs_directory.c */ diff --git a/src/fs/test_fs_download.c b/src/fs/test_fs_download.c new file mode 100644 index 0000000..570eab9 --- /dev/null +++ b/src/fs/test_fs_download.c @@ -0,0 +1,353 @@ +/* + This file is part of GNUnet. + (C) 2004, 2005, 2006, 2008, 2009, 2011 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/test_fs_download.c + * @brief simple testcase for simple publish + download operation + * @author Christian Grothoff + */ + +#include "platform.h" +#include "gnunet_util_lib.h" +#include "gnunet_arm_service.h" +#include "gnunet_fs_service.h" +#include <gauger.h> + +#define VERBOSE GNUNET_NO + +#define START_ARM GNUNET_YES + +/** + * File-size we use for testing. + */ +#define FILESIZE (1024 * 1024 * 2) + +/** + * How long until we give up on transmitting the message? + */ +#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 120) + +/** + * How long should our test-content live? + */ +#define LIFETIME GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15) + +struct PeerContext +{ + struct GNUNET_CONFIGURATION_Handle *cfg; +#if START_ARM + struct GNUNET_OS_Process *arm_proc; +#endif +}; + +static struct PeerContext p1; + +static struct GNUNET_TIME_Absolute start; + +static struct GNUNET_FS_Handle *fs; + +static struct GNUNET_FS_DownloadContext *download; + +static struct GNUNET_FS_PublishContext *publish; + +static GNUNET_SCHEDULER_TaskIdentifier timeout_kill; + +static char *fn; + +static int err; + +static void +timeout_kill_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + if (download != NULL) + { + GNUNET_FS_download_stop (download, GNUNET_YES); + download = NULL; + } + else if (publish != NULL) + { + GNUNET_FS_publish_stop (publish); + publish = NULL; + } + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Timeout downloading file\n"); + timeout_kill = GNUNET_SCHEDULER_NO_TASK; + err = 1; +} + +static void +abort_publish_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + if (publish != NULL) + { + GNUNET_FS_publish_stop (publish); + publish = NULL; + } +} + +static void +stop_fs_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + GNUNET_FS_stop (fs); + fs = NULL; +} + +static void +abort_download_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + uint64_t size; + + if (download != NULL) + { + GNUNET_FS_download_stop (download, GNUNET_YES); + download = NULL; + } + GNUNET_assert (GNUNET_OK == GNUNET_DISK_file_size (fn, &size, GNUNET_YES)); + GNUNET_assert (size == FILESIZE); + GNUNET_DISK_directory_remove (fn); + GNUNET_free (fn); + fn = NULL; + GNUNET_SCHEDULER_cancel (timeout_kill); + timeout_kill = GNUNET_SCHEDULER_NO_TASK; +} + + +static void * +progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *event) +{ + + switch (event->status) + { + case GNUNET_FS_STATUS_PUBLISH_PROGRESS: +#if VERBOSE + printf ("Publish is progressing (%llu/%llu at level %u off %llu)...\n", + (unsigned long long) event->value.publish.completed, + (unsigned long long) event->value.publish.size, + event->value.publish.specifics.progress.depth, + (unsigned long long) event->value.publish.specifics. + progress.offset); +#endif + break; + case GNUNET_FS_STATUS_PUBLISH_COMPLETED: + printf ("Publishing complete, %llu kb/s.\n", + (unsigned long long) (FILESIZE * 1000LL / + (1 + + GNUNET_TIME_absolute_get_duration + (start).rel_value) / 1024LL)); + GAUGER ("FS", "Publishing speed (insertion)", + (unsigned long long) (FILESIZE * 1000LL / + (1 + + GNUNET_TIME_absolute_get_duration + (start).rel_value) / 1024LL), "kb/s"); + fn = GNUNET_DISK_mktemp ("gnunet-download-test-dst"); + start = GNUNET_TIME_absolute_get (); + download = + GNUNET_FS_download_start (fs, + event->value.publish.specifics. + completed.chk_uri, NULL, fn, NULL, 0, + FILESIZE, 1, GNUNET_FS_DOWNLOAD_OPTION_NONE, + "download", NULL); + GNUNET_assert (download != NULL); + break; + case GNUNET_FS_STATUS_DOWNLOAD_COMPLETED: + printf ("Download complete, %llu kb/s.\n", + (unsigned long long) (FILESIZE * 1000LL / + (1 + + GNUNET_TIME_absolute_get_duration + (start).rel_value) / 1024LL)); + GAUGER ("FS", "Local download speed (inserted)", + (unsigned long long) (FILESIZE * 1000LL / + (1 + + GNUNET_TIME_absolute_get_duration + (start).rel_value) / 1024LL), "kb/s"); + GNUNET_SCHEDULER_add_now (&abort_download_task, NULL); + break; + case GNUNET_FS_STATUS_DOWNLOAD_PROGRESS: + GNUNET_assert (download == event->value.download.dc); +#if VERBOSE + printf ("Download is progressing (%llu/%llu at level %u off %llu)...\n", + (unsigned long long) event->value.download.completed, + (unsigned long long) event->value.download.size, + event->value.download.specifics.progress.depth, + (unsigned long long) event->value.download.specifics. + progress.offset); +#endif + break; + case GNUNET_FS_STATUS_PUBLISH_ERROR: + FPRINTF (stderr, "Error publishing file: %s\n", + event->value.publish.specifics.error.message); + GNUNET_break (0); + GNUNET_SCHEDULER_add_continuation (&abort_publish_task, NULL, + GNUNET_SCHEDULER_REASON_PREREQ_DONE); + break; + case GNUNET_FS_STATUS_DOWNLOAD_ERROR: + FPRINTF (stderr, "Error downloading file: %s\n", + event->value.download.specifics.error.message); + GNUNET_SCHEDULER_add_now (&abort_download_task, NULL); + break; + case GNUNET_FS_STATUS_DOWNLOAD_ACTIVE: + case GNUNET_FS_STATUS_DOWNLOAD_INACTIVE: + break; + case GNUNET_FS_STATUS_PUBLISH_START: + GNUNET_assert (0 == strcmp ("publish-context", event->value.publish.cctx)); + GNUNET_assert (NULL == event->value.publish.pctx); + GNUNET_assert (FILESIZE == event->value.publish.size); + GNUNET_assert (0 == event->value.publish.completed); + GNUNET_assert (1 == event->value.publish.anonymity); + break; + case GNUNET_FS_STATUS_PUBLISH_STOPPED: + GNUNET_assert (publish == event->value.publish.pc); + GNUNET_assert (FILESIZE == event->value.publish.size); + GNUNET_assert (1 == event->value.publish.anonymity); + GNUNET_SCHEDULER_add_now (&stop_fs_task, NULL); + break; + case GNUNET_FS_STATUS_DOWNLOAD_START: + GNUNET_assert (0 == strcmp ("download", event->value.download.cctx)); + GNUNET_assert (NULL == event->value.download.pctx); + GNUNET_assert (NULL != event->value.download.uri); + GNUNET_assert (0 == strcmp (fn, event->value.download.filename)); + GNUNET_assert (FILESIZE == event->value.download.size); + GNUNET_assert (0 == event->value.download.completed); + GNUNET_assert (1 == event->value.download.anonymity); + break; + case GNUNET_FS_STATUS_DOWNLOAD_STOPPED: + GNUNET_assert (download == event->value.download.dc); + GNUNET_SCHEDULER_add_continuation (&abort_publish_task, NULL, + GNUNET_SCHEDULER_REASON_PREREQ_DONE); + break; + default: + printf ("Unexpected event: %d\n", event->status); + break; + } + return NULL; +} + + +static void +setup_peer (struct PeerContext *p, const char *cfgname) +{ + p->cfg = GNUNET_CONFIGURATION_create (); +#if START_ARM + p->arm_proc = + GNUNET_OS_start_process (GNUNET_YES, NULL, NULL, "gnunet-service-arm", + "gnunet-service-arm", +#if VERBOSE + "-L", "DEBUG", +#endif + "-c", cfgname, NULL); +#endif + GNUNET_assert (GNUNET_OK == GNUNET_CONFIGURATION_load (p->cfg, cfgname)); +} + + +static void +stop_arm (struct PeerContext *p) +{ +#if START_ARM + if (NULL != p->arm_proc) + { + if (0 != GNUNET_OS_process_kill (p->arm_proc, SIGTERM)) + GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING, "kill"); + if (GNUNET_OS_process_wait (p->arm_proc) != GNUNET_OK) + GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING, "waitpid"); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "ARM process %u stopped\n", + GNUNET_OS_process_get_pid (p->arm_proc)); + GNUNET_OS_process_close (p->arm_proc); + p->arm_proc = NULL; + } +#endif + GNUNET_CONFIGURATION_destroy (p->cfg); +} + + +static void +run (void *cls, char *const *args, const char *cfgfile, + const struct GNUNET_CONFIGURATION_Handle *cfg) +{ + const char *keywords[] = { + "down_foo", + "down_bar", + }; + char *buf; + struct GNUNET_CONTAINER_MetaData *meta; + struct GNUNET_FS_Uri *kuri; + struct GNUNET_FS_FileInformation *fi; + size_t i; + struct GNUNET_FS_BlockOptions bo; + + setup_peer (&p1, "test_fs_download_data.conf"); + fs = GNUNET_FS_start (cfg, "test-fs-download", &progress_cb, NULL, + GNUNET_FS_FLAGS_NONE, GNUNET_FS_OPTIONS_END); + GNUNET_assert (NULL != fs); + buf = GNUNET_malloc (FILESIZE); + for (i = 0; i < FILESIZE; i++) + buf[i] = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 256); + meta = GNUNET_CONTAINER_meta_data_create (); + kuri = GNUNET_FS_uri_ksk_create_from_args (2, keywords); + bo.content_priority = 42; + bo.anonymity_level = 1; + bo.replication_level = 0; + bo.expiration_time = GNUNET_TIME_relative_to_absolute (LIFETIME); + fi = GNUNET_FS_file_information_create_from_data (fs, "publish-context", + FILESIZE, buf, kuri, meta, + GNUNET_NO, &bo); + GNUNET_FS_uri_destroy (kuri); + GNUNET_CONTAINER_meta_data_destroy (meta); + GNUNET_assert (NULL != fi); + timeout_kill = + GNUNET_SCHEDULER_add_delayed (TIMEOUT, &timeout_kill_task, NULL); + start = GNUNET_TIME_absolute_get (); + publish = + GNUNET_FS_publish_start (fs, fi, NULL, NULL, NULL, + GNUNET_FS_PUBLISH_OPTION_NONE); + GNUNET_assert (publish != NULL); +} + + +int +main (int argc, char *argv[]) +{ + char *const argvx[] = { + "test-fs-download", + "-c", + "test_fs_download_data.conf", +#if VERBOSE + "-L", "DEBUG", +#endif + NULL + }; + struct GNUNET_GETOPT_CommandLineOption options[] = { + GNUNET_GETOPT_OPTION_END + }; + + GNUNET_log_setup ("test_fs_download", +#if VERBOSE + "DEBUG", +#else + "WARNING", +#endif + NULL); + GNUNET_PROGRAM_run ((sizeof (argvx) / sizeof (char *)) - 1, argvx, + "test-fs-download", "nohelp", options, &run, NULL); + stop_arm (&p1); + GNUNET_DISK_directory_remove ("/tmp/gnunet-test-fs-download/"); + return err; +} + +/* end of test_fs_download.c */ diff --git a/src/fs/test_fs_download_data.conf b/src/fs/test_fs_download_data.conf new file mode 100644 index 0000000..25aad51 --- /dev/null +++ b/src/fs/test_fs_download_data.conf @@ -0,0 +1,5 @@ +@INLINE@ test_fs_defaults.conf +[PATHS] +SERVICEHOME = /tmp/gnunet-test-fs-download/ +DEFAULTCONFIG = test_fs_download_data.conf + diff --git a/src/fs/test_fs_download_indexed.c b/src/fs/test_fs_download_indexed.c new file mode 100644 index 0000000..e8504f1 --- /dev/null +++ b/src/fs/test_fs_download_indexed.c @@ -0,0 +1,372 @@ +/* + This file is part of GNUnet. + (C) 2004, 2005, 2006, 2008, 2009, 2011 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/test_fs_download_indexed.c + * @brief simple testcase for downloading of indexed file + * @author Christian Grothoff + */ + +#include "platform.h" +#include "gnunet_util_lib.h" +#include "gnunet_arm_service.h" +#include "gnunet_fs_service.h" +#include <gauger.h> + +#define VERBOSE GNUNET_NO + +#define START_ARM GNUNET_YES + +/** + * File-size we use for testing. + */ +#define FILESIZE (1024 * 1024 * 2) + +/** + * How long until we give up on transmitting the message? + */ +#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 60) + +/** + * How long should our test-content live? + */ +#define LIFETIME GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15) + +struct PeerContext +{ + struct GNUNET_CONFIGURATION_Handle *cfg; +#if START_ARM + struct GNUNET_OS_Process *arm_proc; +#endif +}; + +static struct PeerContext p1; + +static struct GNUNET_TIME_Absolute start; + +static struct GNUNET_FS_Handle *fs; + +static struct GNUNET_FS_DownloadContext *download; + +static struct GNUNET_FS_PublishContext *publish; + +static GNUNET_SCHEDULER_TaskIdentifier timeout_kill; + +static char *fn; + +static char *fn1; + +static int err; + +static void +timeout_kill_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + if (download != NULL) + { + GNUNET_FS_download_stop (download, GNUNET_YES); + download = NULL; + } + else if (publish != NULL) + { + GNUNET_FS_publish_stop (publish); + publish = NULL; + } + timeout_kill = GNUNET_SCHEDULER_NO_TASK; + err = 1; +} + +static void +abort_publish_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + if (publish != NULL) + { + GNUNET_FS_publish_stop (publish); + publish = NULL; + } +} + +static void +stop_fs_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + GNUNET_FS_stop (fs); + fs = NULL; +} + +static void +abort_download_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + uint64_t size; + + if (download != NULL) + { + GNUNET_FS_download_stop (download, GNUNET_YES); + download = NULL; + } + GNUNET_assert (GNUNET_OK == GNUNET_DISK_file_size (fn, &size, GNUNET_YES)); + GNUNET_assert (size == FILESIZE); + GNUNET_DISK_directory_remove (fn); + GNUNET_free (fn); + fn = NULL; + GNUNET_SCHEDULER_cancel (timeout_kill); + timeout_kill = GNUNET_SCHEDULER_NO_TASK; +} + + +static void * +progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *event) +{ + + switch (event->status) + { + case GNUNET_FS_STATUS_PUBLISH_PROGRESS: +#if VERBOSE + printf ("Publish is progressing (%llu/%llu at level %u off %llu)...\n", + (unsigned long long) event->value.publish.completed, + (unsigned long long) event->value.publish.size, + event->value.publish.specifics.progress.depth, + (unsigned long long) event->value.publish.specifics. + progress.offset); +#endif + break; + case GNUNET_FS_STATUS_PUBLISH_COMPLETED: + printf ("Publishing complete, %llu kbps.\n", + (unsigned long long) (FILESIZE * 1000LL / + (1 + + GNUNET_TIME_absolute_get_duration + (start).rel_value) / 1024LL)); + GAUGER ("FS", "Publishing speed (indexing)", + (unsigned long long) (FILESIZE * 1000LL / + (1 + + GNUNET_TIME_absolute_get_duration + (start).rel_value) / 1024LL), "kb/s"); + fn = GNUNET_DISK_mktemp ("gnunet-download-test-dst"); + start = GNUNET_TIME_absolute_get (); + download = + GNUNET_FS_download_start (fs, + event->value.publish.specifics. + completed.chk_uri, NULL, fn, NULL, 0, + FILESIZE, 1, GNUNET_FS_DOWNLOAD_OPTION_NONE, + "download", NULL); + GNUNET_assert (download != NULL); + break; + case GNUNET_FS_STATUS_DOWNLOAD_COMPLETED: + printf ("Download complete, %llu kbps.\n", + (unsigned long long) (FILESIZE * 1000LL / + (1 + + GNUNET_TIME_absolute_get_duration + (start).rel_value) / 1024LL)); + GAUGER ("FS", "Local download speed (indexed)", + (unsigned long long) (FILESIZE * 1000LL / + (1 + + GNUNET_TIME_absolute_get_duration + (start).rel_value) / 1024LL), "kb/s"); + GNUNET_SCHEDULER_add_now (&abort_download_task, NULL); + break; + case GNUNET_FS_STATUS_DOWNLOAD_PROGRESS: + GNUNET_assert (download == event->value.download.dc); +#if VERBOSE + printf ("Download is progressing (%llu/%llu at level %u off %llu)...\n", + (unsigned long long) event->value.download.completed, + (unsigned long long) event->value.download.size, + event->value.download.specifics.progress.depth, + (unsigned long long) event->value.download.specifics. + progress.offset); +#endif + break; + case GNUNET_FS_STATUS_PUBLISH_ERROR: + FPRINTF (stderr, "Error publishing file: %s\n", + event->value.publish.specifics.error.message); + GNUNET_break (0); + GNUNET_SCHEDULER_add_continuation (&abort_publish_task, NULL, + GNUNET_SCHEDULER_REASON_PREREQ_DONE); + break; + case GNUNET_FS_STATUS_DOWNLOAD_ERROR: + FPRINTF (stderr, "Error downloading file: %s\n", + event->value.download.specifics.error.message); + GNUNET_SCHEDULER_add_now (&abort_download_task, NULL); + break; + case GNUNET_FS_STATUS_DOWNLOAD_ACTIVE: + case GNUNET_FS_STATUS_DOWNLOAD_INACTIVE: + break; + case GNUNET_FS_STATUS_PUBLISH_START: + GNUNET_assert (0 == strcmp ("publish-context", event->value.publish.cctx)); + GNUNET_assert (NULL == event->value.publish.pctx); + GNUNET_assert (FILESIZE == event->value.publish.size); + GNUNET_assert (0 == event->value.publish.completed); + GNUNET_assert (1 == event->value.publish.anonymity); + break; + case GNUNET_FS_STATUS_PUBLISH_STOPPED: + GNUNET_assert (publish == event->value.publish.pc); + GNUNET_assert (FILESIZE == event->value.publish.size); + GNUNET_assert (1 == event->value.publish.anonymity); + GNUNET_SCHEDULER_add_now (&stop_fs_task, NULL); + break; + case GNUNET_FS_STATUS_DOWNLOAD_START: + GNUNET_assert (0 == strcmp ("download", event->value.download.cctx)); + GNUNET_assert (NULL == event->value.download.pctx); + GNUNET_assert (NULL != event->value.download.uri); + GNUNET_assert (0 == strcmp (fn, event->value.download.filename)); + GNUNET_assert (FILESIZE == event->value.download.size); + GNUNET_assert (0 == event->value.download.completed); + GNUNET_assert (1 == event->value.download.anonymity); + break; + case GNUNET_FS_STATUS_DOWNLOAD_STOPPED: + GNUNET_assert (download == event->value.download.dc); + GNUNET_SCHEDULER_add_continuation (&abort_publish_task, NULL, + GNUNET_SCHEDULER_REASON_PREREQ_DONE); + break; + default: + printf ("Unexpected event: %d\n", event->status); + break; + } + return NULL; +} + + +static void +setup_peer (struct PeerContext *p, const char *cfgname) +{ + p->cfg = GNUNET_CONFIGURATION_create (); +#if START_ARM + p->arm_proc = + GNUNET_OS_start_process (GNUNET_YES, NULL, NULL, "gnunet-service-arm", + "gnunet-service-arm", +#if VERBOSE + "-L", "DEBUG", +#endif + "-c", cfgname, NULL); +#endif + GNUNET_assert (GNUNET_OK == GNUNET_CONFIGURATION_load (p->cfg, cfgname)); +} + + +static void +stop_arm (struct PeerContext *p) +{ +#if START_ARM + if (NULL != p->arm_proc) + { + if (0 != GNUNET_OS_process_kill (p->arm_proc, SIGTERM)) + GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING, "kill"); + if (GNUNET_OS_process_wait (p->arm_proc) != GNUNET_OK) + GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING, "waitpid"); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "ARM process %u stopped\n", + GNUNET_OS_process_get_pid (p->arm_proc)); + GNUNET_OS_process_close (p->arm_proc); + p->arm_proc = NULL; + } +#endif + GNUNET_CONFIGURATION_destroy (p->cfg); +} + + +static void +run (void *cls, char *const *args, const char *cfgfile, + const struct GNUNET_CONFIGURATION_Handle *cfg) +{ + const char *keywords[] = { + "down_foo", + "down_bar", + }; + char *buf; + struct GNUNET_CONTAINER_MetaData *meta; + struct GNUNET_FS_Uri *kuri; + struct GNUNET_FS_FileInformation *fi; + struct GNUNET_FS_BlockOptions bo; + size_t i; + + setup_peer (&p1, "test_fs_download_data.conf"); + fs = GNUNET_FS_start (cfg, "test-fs-download-indexed", &progress_cb, NULL, + GNUNET_FS_FLAGS_NONE, GNUNET_FS_OPTIONS_END); + GNUNET_assert (NULL != fs); + + fn1 = GNUNET_DISK_mktemp ("gnunet-download-indexed-test"); + buf = GNUNET_malloc (FILESIZE); + for (i = 0; i < FILESIZE; i++) + buf[i] = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 256); + GNUNET_assert (FILESIZE == + GNUNET_DISK_fn_write (fn1, buf, FILESIZE, + GNUNET_DISK_PERM_USER_READ | + GNUNET_DISK_PERM_USER_WRITE)); + GNUNET_free (buf); + meta = GNUNET_CONTAINER_meta_data_create (); + kuri = GNUNET_FS_uri_ksk_create_from_args (2, keywords); + bo.content_priority = 42; + bo.anonymity_level = 1; + bo.replication_level = 0; + bo.expiration_time = GNUNET_TIME_relative_to_absolute (LIFETIME); + fi = GNUNET_FS_file_information_create_from_file (fs, "publish-context", fn1, + kuri, meta, GNUNET_YES, + &bo); + GNUNET_FS_uri_destroy (kuri); + GNUNET_CONTAINER_meta_data_destroy (meta); + GNUNET_assert (NULL != fi); + timeout_kill = + GNUNET_SCHEDULER_add_delayed (TIMEOUT, &timeout_kill_task, NULL); + start = GNUNET_TIME_absolute_get (); + publish = + GNUNET_FS_publish_start (fs, fi, NULL, NULL, NULL, + GNUNET_FS_PUBLISH_OPTION_NONE); + GNUNET_assert (publish != NULL); +} + + +int +main (int argc, char *argv[]) +{ + char *const argvx[] = { + "test-fs-download-indexed", + "-c", + "test_fs_download_data.conf", +#if VERBOSE + "-L", "DEBUG", +#endif + NULL + }; + struct GNUNET_GETOPT_CommandLineOption options[] = { + GNUNET_GETOPT_OPTION_END + }; + + GNUNET_log_setup ("test_fs_download_indexed", +#if VERBOSE + "DEBUG", +#else + "WARNING", +#endif + NULL); + GNUNET_PROGRAM_run ((sizeof (argvx) / sizeof (char *)) - 1, argvx, + "test-fs-download-indexed", "nohelp", options, &run, + NULL); + stop_arm (&p1); + if (fn1 != NULL) + { + GNUNET_DISK_directory_remove (fn1); + GNUNET_free (fn1); + } + if (fn != NULL) + { + GNUNET_DISK_directory_remove (fn); + GNUNET_free (fn); + } + GNUNET_DISK_directory_remove ("/tmp/gnunet-test-fs-download/"); + return err; +} + +/* end of test_fs_download_indexed.c */ diff --git a/src/fs/test_fs_download_persistence.c b/src/fs/test_fs_download_persistence.c new file mode 100644 index 0000000..bcb1c54 --- /dev/null +++ b/src/fs/test_fs_download_persistence.c @@ -0,0 +1,405 @@ +/* + This file is part of GNUnet. + (C) 2004, 2005, 2006, 2008, 2009, 2010 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/test_fs_download_persistence.c + * @brief simple testcase for persistence of simple download operation + * @author Christian Grothoff + */ + +#include "platform.h" +#include "gnunet_util_lib.h" +#include "gnunet_arm_service.h" +#include "gnunet_fs_service.h" + +#define VERBOSE GNUNET_NO + +#define START_ARM GNUNET_YES + +/** + * File-size we use for testing. + */ +#define FILESIZE (1024 * 1024 * 2) + +/** + * How long until we give up on transmitting the message? + */ +#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 60) + +/** + * How long should our test-content live? + */ +#define LIFETIME GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15) + +struct PeerContext +{ + struct GNUNET_CONFIGURATION_Handle *cfg; +#if START_ARM + struct GNUNET_OS_Process *arm_proc; +#endif +}; + +static struct PeerContext p1; + +static struct GNUNET_TIME_Absolute start; + +static const struct GNUNET_CONFIGURATION_Handle *cfg; + +static struct GNUNET_FS_Handle *fs; + +static struct GNUNET_FS_DownloadContext *download; + +static struct GNUNET_FS_PublishContext *publish; + +static GNUNET_SCHEDULER_TaskIdentifier timeout_kill; + +static char *fn; + +static int err; + +static void +timeout_kill_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Timeout downloading file\n"); + if (download != NULL) + { + GNUNET_FS_download_stop (download, GNUNET_YES); + download = NULL; + } + else if (publish != NULL) + { + GNUNET_FS_publish_stop (publish); + publish = NULL; + } + timeout_kill = GNUNET_SCHEDULER_NO_TASK; + err = 1; +} + +static void +abort_publish_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + if (publish != NULL) + { + GNUNET_FS_publish_stop (publish); + publish = NULL; + } +} + + +static void +abort_download_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + uint64_t size; + + if (download != NULL) + { + GNUNET_FS_download_stop (download, GNUNET_YES); + download = NULL; + } + GNUNET_assert (GNUNET_OK == GNUNET_DISK_file_size (fn, &size, GNUNET_YES)); + GNUNET_assert (size == FILESIZE); + GNUNET_DISK_directory_remove (fn); + GNUNET_free (fn); + fn = NULL; + GNUNET_SCHEDULER_cancel (timeout_kill); + timeout_kill = GNUNET_SCHEDULER_NO_TASK; +} + + +static void * +progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *event); + + +static void +restart_fs_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Restarting FS.\n"); + GNUNET_FS_stop (fs); + fs = GNUNET_FS_start (cfg, "test-fs-download-persistence", &progress_cb, NULL, + GNUNET_FS_FLAGS_PERSISTENCE, GNUNET_FS_OPTIONS_END); +} + + +/** + * Consider scheduling the restart-task. + * Only runs the restart task once per event + * category. + * + * @param ev type of the event to consider + */ +static void +consider_restart (int ev) +{ + static int prev[32]; + static int off; + int i; + + for (i = 0; i < off; i++) + if (prev[i] == ev) + return; + prev[off++] = ev; + GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_URGENT, + &restart_fs_task, NULL); +} + + +static void * +progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *event) +{ + switch (event->status) + { + case GNUNET_FS_STATUS_PUBLISH_PROGRESS: +#if VERBOSE + printf ("Publish is progressing (%llu/%llu at level %u off %llu)...\n", + (unsigned long long) event->value.publish.completed, + (unsigned long long) event->value.publish.size, + event->value.publish.specifics.progress.depth, + (unsigned long long) event->value.publish.specifics. + progress.offset); +#endif + break; + case GNUNET_FS_STATUS_PUBLISH_COMPLETED: + printf ("Publishing complete, %llu kbps.\n", + (unsigned long long) (FILESIZE * 1000LL / + (1 + + GNUNET_TIME_absolute_get_duration + (start).rel_value) / 1024LL)); + fn = GNUNET_DISK_mktemp ("gnunet-download-test-dst"); + start = GNUNET_TIME_absolute_get (); + GNUNET_assert (download == NULL); + GNUNET_FS_download_start (fs, + event->value.publish.specifics.completed.chk_uri, + NULL, fn, NULL, 0, FILESIZE, 1, + GNUNET_FS_DOWNLOAD_OPTION_NONE, "download", NULL); + break; + case GNUNET_FS_STATUS_DOWNLOAD_COMPLETED: + consider_restart (event->status); + printf ("Download complete, %llu kbps.\n", + (unsigned long long) (FILESIZE * 1000LL / + (1 + + GNUNET_TIME_absolute_get_duration + (start).rel_value) / 1024LL)); + GNUNET_SCHEDULER_add_now (&abort_download_task, NULL); + break; + case GNUNET_FS_STATUS_DOWNLOAD_PROGRESS: + consider_restart (event->status); + GNUNET_assert (download == event->value.download.dc); +#if VERBOSE + printf ("Download is progressing (%llu/%llu at level %u off %llu)...\n", + (unsigned long long) event->value.download.completed, + (unsigned long long) event->value.download.size, + event->value.download.specifics.progress.depth, + (unsigned long long) event->value.download.specifics. + progress.offset); +#endif + break; + case GNUNET_FS_STATUS_PUBLISH_ERROR: + FPRINTF (stderr, "Error publishing file: %s\n", + event->value.publish.specifics.error.message); + GNUNET_break (0); + GNUNET_SCHEDULER_add_continuation (&abort_publish_task, NULL, + GNUNET_SCHEDULER_REASON_PREREQ_DONE); + break; + case GNUNET_FS_STATUS_DOWNLOAD_ERROR: + FPRINTF (stderr, "Error downloading file: %s\n", + event->value.download.specifics.error.message); + GNUNET_SCHEDULER_add_now (&abort_download_task, NULL); + break; + case GNUNET_FS_STATUS_PUBLISH_SUSPEND: + GNUNET_assert (event->value.publish.pc == publish); + publish = NULL; + break; + case GNUNET_FS_STATUS_PUBLISH_RESUME: + GNUNET_assert (NULL == publish); + publish = event->value.publish.pc; + break; + case GNUNET_FS_STATUS_DOWNLOAD_SUSPEND: + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Download suspended.\n"); + GNUNET_assert (event->value.download.dc == download); + download = NULL; + break; + case GNUNET_FS_STATUS_DOWNLOAD_RESUME: + GNUNET_assert (NULL == download); + download = event->value.download.dc; + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Download resumed.\n"); + break; + case GNUNET_FS_STATUS_DOWNLOAD_ACTIVE: + consider_restart (event->status); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Download active.\n"); + break; + case GNUNET_FS_STATUS_DOWNLOAD_INACTIVE: + consider_restart (event->status); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Download inactive.\n"); + break; + case GNUNET_FS_STATUS_PUBLISH_START: + GNUNET_assert (0 == strcmp ("publish-context", event->value.publish.cctx)); + GNUNET_assert (NULL == event->value.publish.pctx); + GNUNET_assert (FILESIZE == event->value.publish.size); + GNUNET_assert (0 == event->value.publish.completed); + GNUNET_assert (1 == event->value.publish.anonymity); + break; + case GNUNET_FS_STATUS_PUBLISH_STOPPED: + GNUNET_assert (publish == event->value.publish.pc); + GNUNET_assert (FILESIZE == event->value.publish.size); + GNUNET_assert (1 == event->value.publish.anonymity); + GNUNET_FS_stop (fs); + fs = NULL; + break; + case GNUNET_FS_STATUS_DOWNLOAD_START: + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Download started.\n"); + consider_restart (event->status); + GNUNET_assert (download == NULL); + download = event->value.download.dc; + GNUNET_assert (0 == strcmp ("download", event->value.download.cctx)); + GNUNET_assert (NULL == event->value.download.pctx); + GNUNET_assert (NULL != event->value.download.uri); + GNUNET_assert (0 == strcmp (fn, event->value.download.filename)); + GNUNET_assert (FILESIZE == event->value.download.size); + GNUNET_assert (0 == event->value.download.completed); + GNUNET_assert (1 == event->value.download.anonymity); + break; + case GNUNET_FS_STATUS_DOWNLOAD_STOPPED: + GNUNET_assert (download == event->value.download.dc); + GNUNET_SCHEDULER_add_continuation (&abort_publish_task, NULL, + GNUNET_SCHEDULER_REASON_PREREQ_DONE); + download = NULL; + break; + default: + printf ("Unexpected event: %d\n", event->status); + break; + } + return NULL; +} + + +static void +setup_peer (struct PeerContext *p, const char *cfgname) +{ + p->cfg = GNUNET_CONFIGURATION_create (); +#if START_ARM + p->arm_proc = + GNUNET_OS_start_process (GNUNET_YES, NULL, NULL, "gnunet-service-arm", + "gnunet-service-arm", +#if VERBOSE + "-L", "DEBUG", +#endif + "-c", cfgname, NULL); +#endif + GNUNET_assert (GNUNET_OK == GNUNET_CONFIGURATION_load (p->cfg, cfgname)); +} + + +static void +stop_arm (struct PeerContext *p) +{ +#if START_ARM + if (NULL != p->arm_proc) + { + if (0 != GNUNET_OS_process_kill (p->arm_proc, SIGTERM)) + GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING, "kill"); + if (GNUNET_OS_process_wait (p->arm_proc) != GNUNET_OK) + GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING, "waitpid"); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "ARM process %u stopped\n", + GNUNET_OS_process_get_pid (p->arm_proc)); + GNUNET_OS_process_close (p->arm_proc); + p->arm_proc = NULL; + } +#endif + GNUNET_CONFIGURATION_destroy (p->cfg); +} + + +static void +run (void *cls, char *const *args, const char *cfgfile, + const struct GNUNET_CONFIGURATION_Handle *c) +{ + const char *keywords[] = { + "down_foo", + "down_bar", + }; + char *buf; + struct GNUNET_CONTAINER_MetaData *meta; + struct GNUNET_FS_Uri *kuri; + struct GNUNET_FS_FileInformation *fi; + size_t i; + struct GNUNET_FS_BlockOptions bo; + + cfg = c; + setup_peer (&p1, "test_fs_download_data.conf"); + fs = GNUNET_FS_start (cfg, "test-fs-download-persistence", &progress_cb, NULL, + GNUNET_FS_FLAGS_PERSISTENCE, GNUNET_FS_OPTIONS_END); + GNUNET_assert (NULL != fs); + buf = GNUNET_malloc (FILESIZE); + for (i = 0; i < FILESIZE; i++) + buf[i] = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 256); + meta = GNUNET_CONTAINER_meta_data_create (); + kuri = GNUNET_FS_uri_ksk_create_from_args (2, keywords); + bo.content_priority = 42; + bo.anonymity_level = 1; + bo.replication_level = 0; + bo.expiration_time = GNUNET_TIME_relative_to_absolute (LIFETIME); + fi = GNUNET_FS_file_information_create_from_data (fs, "publish-context", + FILESIZE, buf, kuri, meta, + GNUNET_NO, &bo); + GNUNET_FS_uri_destroy (kuri); + GNUNET_CONTAINER_meta_data_destroy (meta); + GNUNET_assert (NULL != fi); + timeout_kill = + GNUNET_SCHEDULER_add_delayed (TIMEOUT, &timeout_kill_task, NULL); + start = GNUNET_TIME_absolute_get (); + publish = + GNUNET_FS_publish_start (fs, fi, NULL, NULL, NULL, + GNUNET_FS_PUBLISH_OPTION_NONE); + GNUNET_assert (publish != NULL); +} + + +int +main (int argc, char *argv[]) +{ + char *const argvx[] = { + "test-fs-download-persistence", + "-c", + "test_fs_download_data.conf", +#if VERBOSE + "-L", "DEBUG", +#endif + NULL + }; + struct GNUNET_GETOPT_CommandLineOption options[] = { + GNUNET_GETOPT_OPTION_END + }; + GNUNET_log_setup ("test_fs_download_persistence", +#if VERBOSE + "DEBUG", +#else + "WARNING", +#endif + NULL); + GNUNET_DISK_directory_remove ("/tmp/gnunet-test-fs-download/"); + GNUNET_PROGRAM_run ((sizeof (argvx) / sizeof (char *)) - 1, argvx, + "test-fs-download-persistence", "nohelp", options, &run, + NULL); + stop_arm (&p1); + GNUNET_DISK_directory_remove ("/tmp/gnunet-test-fs-download/"); + return err; +} + +/* end of test_fs_download_persistence.c */ diff --git a/src/fs/test_fs_file_information.c b/src/fs/test_fs_file_information.c new file mode 100644 index 0000000..fb7de7d --- /dev/null +++ b/src/fs/test_fs_file_information.c @@ -0,0 +1,172 @@ +/* + This file is part of GNUnet. + (C) 2004, 2005, 2006, 2008, 2009 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/test_fs_file_information.c + * @brief simple testcase for file_information operations + * @author Christian Grothoff + * + * TODO: + * - test that metatdata, etc. are all correct (for example, + * there is a known bug with dirname never being set that is + * not detected!) + * - need to iterate over file-information structure + * - other API functions may not yet be tested (such as + * filedata-from-callback) + */ + +#include "platform.h" +#include "gnunet_util_lib.h" +#include "gnunet_fs_service.h" + +#define VERBOSE GNUNET_NO + +/** + * File-size we use for testing. + */ +#define FILESIZE (1024 * 1024 * 2) + +/** + * How long should our test-content live? + */ +#define LIFETIME GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15) + + +static int +mycleaner (void *cls, struct GNUNET_FS_FileInformation *fi, uint64_t length, + struct GNUNET_CONTAINER_MetaData *meta, struct GNUNET_FS_Uri **uri, + struct GNUNET_FS_BlockOptions *bo, int *do_index, void **client_info) +{ + return GNUNET_OK; +} + + +static void +run (void *cls, char *const *args, const char *cfgfile, + const struct GNUNET_CONFIGURATION_Handle *cfg) +{ + const char *keywords[] = { + "down_foo", + "down_bar", + }; + char *fn1; + char *fn2; + char *buf; + struct GNUNET_CONTAINER_MetaData *meta; + struct GNUNET_FS_Uri *kuri; + struct GNUNET_FS_FileInformation *fi1; + struct GNUNET_FS_FileInformation *fi2; + struct GNUNET_FS_FileInformation *fidir; + struct GNUNET_FS_Handle *fs; + size_t i; + struct GNUNET_FS_BlockOptions bo; + + fs = GNUNET_FS_start (cfg, "test-fs-file-information", NULL, NULL, + GNUNET_FS_FLAGS_NONE, GNUNET_FS_OPTIONS_END); + fn1 = GNUNET_DISK_mktemp ("gnunet-file_information-test-dst"); + buf = GNUNET_malloc (FILESIZE); + for (i = 0; i < FILESIZE; i++) + buf[i] = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 256); + GNUNET_assert (FILESIZE == + GNUNET_DISK_fn_write (fn1, buf, FILESIZE, + GNUNET_DISK_PERM_USER_READ | + GNUNET_DISK_PERM_USER_WRITE)); + GNUNET_free (buf); + + fn2 = GNUNET_DISK_mktemp ("gnunet-file_information-test-dst"); + buf = GNUNET_malloc (FILESIZE); + for (i = 0; i < FILESIZE; i++) + buf[i] = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 256); + GNUNET_assert (FILESIZE == + GNUNET_DISK_fn_write (fn2, buf, FILESIZE, + GNUNET_DISK_PERM_USER_READ | + GNUNET_DISK_PERM_USER_WRITE)); + GNUNET_free (buf); + + meta = GNUNET_CONTAINER_meta_data_create (); + kuri = GNUNET_FS_uri_ksk_create_from_args (2, keywords); + bo.content_priority = 42; + bo.anonymity_level = 1; + bo.replication_level = 0; + bo.expiration_time = GNUNET_TIME_relative_to_absolute (LIFETIME); + fi1 = + GNUNET_FS_file_information_create_from_file (fs, + "file_information-context1", + fn1, kuri, meta, GNUNET_YES, + &bo); + GNUNET_assert (fi1 != NULL); + fi2 = + GNUNET_FS_file_information_create_from_file (fs, + "file_information-context2", + fn2, kuri, meta, GNUNET_YES, + &bo); + GNUNET_assert (fi2 != NULL); + fidir = + GNUNET_FS_file_information_create_empty_directory (fs, + "file_information-context-dir", + kuri, meta, &bo, NULL); + GNUNET_assert (GNUNET_OK == GNUNET_FS_file_information_add (fidir, fi1)); + GNUNET_assert (GNUNET_OK == GNUNET_FS_file_information_add (fidir, fi2)); + GNUNET_FS_uri_destroy (kuri); + GNUNET_CONTAINER_meta_data_destroy (meta); + GNUNET_assert (NULL != fidir); + /* FIXME: test more of API! */ + GNUNET_FS_file_information_destroy (fidir, &mycleaner, NULL); + GNUNET_DISK_directory_remove (fn1); + GNUNET_DISK_directory_remove (fn2); + GNUNET_free_non_null (fn1); + GNUNET_free_non_null (fn2); + GNUNET_FS_stop (fs); +} + + + + + +int +main (int argc, char *argv[]) +{ + char *const argvx[] = { + "test-fs-file_information", + "-c", + "test_fs_file_information_data.conf", +#if VERBOSE + "-L", "DEBUG", +#endif + NULL + }; + struct GNUNET_GETOPT_CommandLineOption options[] = { + GNUNET_GETOPT_OPTION_END + }; + + GNUNET_log_setup ("test_fs_file_information", +#if VERBOSE + "DEBUG", +#else + "WARNING", +#endif + NULL); + GNUNET_PROGRAM_run ((sizeof (argvx) / sizeof (char *)) - 1, argvx, + "test-fs-file_information", "nohelp", options, &run, + NULL); + return 0; +} + +/* end of test_fs_file_information.c */ diff --git a/src/fs/test_fs_file_information_data.conf b/src/fs/test_fs_file_information_data.conf new file mode 100644 index 0000000..09cedf8 --- /dev/null +++ b/src/fs/test_fs_file_information_data.conf @@ -0,0 +1,8 @@ +@INLINE@ test_fs_defaults.conf +[PATHS] +SERVICEHOME = /tmp/gnunet-test-fs-file-information/ +DEFAULTCONFIG = test_fs_file_information_data.conf + +[transport] +PLUGINS = + diff --git a/src/fs/test_fs_getopt.c b/src/fs/test_fs_getopt.c new file mode 100644 index 0000000..571346f --- /dev/null +++ b/src/fs/test_fs_getopt.c @@ -0,0 +1,40 @@ +/* + This file is part of GNUnet + (C) 2009 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ +/** + * @file fs/test_fs_getopt.c + * @brief test for fs_getopt.c + * @author Christian Grothoff + */ +#include "platform.h" +#include "gnunet_fs_service.h" + +int +main (int argc, char *argv[]) +{ + GNUNET_log_setup ("test_fs_directory", +#if VERBOSE + "DEBUG", +#else + "WARNING", +#endif + NULL); + FPRINTF (stderr, "%s", "WARNING: testcase not yet written.\n"); + return 0; /* testcase passed */ +} diff --git a/src/fs/test_fs_list_indexed.c b/src/fs/test_fs_list_indexed.c new file mode 100644 index 0000000..535f8ef --- /dev/null +++ b/src/fs/test_fs_list_indexed.c @@ -0,0 +1,338 @@ +/* + This file is part of GNUnet. + (C) 2004, 2005, 2006, 2008, 2009 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/test_fs_list_indexed.c + * @brief simple testcase for list_indexed operation (indexing, listing + * indexed) + * @author Christian Grothoff + * + * TODO: + * - actually call list_indexed API! + */ + +#include "platform.h" +#include "gnunet_util_lib.h" +#include "gnunet_arm_service.h" +#include "gnunet_fs_service.h" + +#define VERBOSE GNUNET_NO + +#define START_ARM GNUNET_YES + +/** + * File-size we use for testing. + */ +#define FILESIZE (1024 * 1024 * 2) + +/** + * How long until we give up on transmitting the message? + */ +#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 60) + +/** + * How long should our test-content live? + */ +#define LIFETIME GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15) + +struct PeerContext +{ + struct GNUNET_CONFIGURATION_Handle *cfg; +#if START_ARM + struct GNUNET_OS_Process *arm_proc; +#endif +}; + +static struct PeerContext p1; + +static struct GNUNET_TIME_Absolute start; + +static struct GNUNET_FS_Handle *fs; + +static struct GNUNET_FS_PublishContext *publish; + +static char *fn1; + +static char *fn2; + +static int err; + +static void +abort_publish_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + GNUNET_FS_publish_stop (publish); + publish = NULL; + GNUNET_DISK_directory_remove (fn1); + GNUNET_free (fn1); + fn1 = NULL; + GNUNET_DISK_directory_remove (fn2); + GNUNET_free (fn2); + fn2 = NULL; +} + + +static void +list_indexed_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + + GNUNET_SCHEDULER_add_continuation (&abort_publish_task, NULL, + GNUNET_SCHEDULER_REASON_PREREQ_DONE); +} + + +static void * +progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *event) +{ + void *ret; + + ret = NULL; + switch (event->status) + { + case GNUNET_FS_STATUS_PUBLISH_COMPLETED: + ret = event->value.publish.cctx; + printf ("Publish complete, %llu kbps.\n", + (unsigned long long) (FILESIZE * 1000 / + (1 + + GNUNET_TIME_absolute_get_duration + (start).rel_value) / 1024)); + if (0 == strcmp ("list_indexed-context-dir", event->value.publish.cctx)) + GNUNET_SCHEDULER_add_continuation (&list_indexed_task, NULL, + GNUNET_SCHEDULER_REASON_PREREQ_DONE); + + break; + case GNUNET_FS_STATUS_PUBLISH_PROGRESS: + ret = event->value.publish.cctx; + GNUNET_assert (publish == event->value.publish.pc); +#if VERBOSE + printf ("Publish is progressing (%llu/%llu at level %u off %llu)...\n", + (unsigned long long) event->value.publish.completed, + (unsigned long long) event->value.publish.size, + event->value.publish.specifics.progress.depth, + (unsigned long long) event->value.publish.specifics. + progress.offset); +#endif + break; + case GNUNET_FS_STATUS_PUBLISH_ERROR: + ret = event->value.publish.cctx; + FPRINTF (stderr, "Error publishing file: %s\n", + event->value.publish.specifics.error.message); + err = 1; + if (0 == strcmp ("list_indexed-context-dir", event->value.publish.cctx)) + GNUNET_SCHEDULER_add_continuation (&abort_publish_task, NULL, + GNUNET_SCHEDULER_REASON_PREREQ_DONE); + break; + case GNUNET_FS_STATUS_PUBLISH_START: + ret = event->value.publish.cctx; + if (0 == strcmp ("list_indexed-context1", event->value.publish.cctx)) + { + GNUNET_assert (0 == + strcmp ("list_indexed-context-dir", + event->value.publish.pctx)); + GNUNET_assert (FILESIZE == event->value.publish.size); + GNUNET_assert (0 == event->value.publish.completed); + GNUNET_assert (1 == event->value.publish.anonymity); + } + else if (0 == strcmp ("list_indexed-context2", event->value.publish.cctx)) + { + GNUNET_assert (0 == + strcmp ("list_indexed-context-dir", + event->value.publish.pctx)); + GNUNET_assert (FILESIZE == event->value.publish.size); + GNUNET_assert (0 == event->value.publish.completed); + GNUNET_assert (2 == event->value.publish.anonymity); + } + else if (0 == + strcmp ("list_indexed-context-dir", event->value.publish.cctx)) + { + GNUNET_assert (0 == event->value.publish.completed); + GNUNET_assert (3 == event->value.publish.anonymity); + } + else + GNUNET_assert (0); + break; + case GNUNET_FS_STATUS_PUBLISH_STOPPED: + if (0 == strcmp ("list_indexed-context-dir", event->value.publish.cctx)) + { + GNUNET_assert (publish == event->value.publish.pc); + publish = NULL; + } + break; + default: + printf ("Unexpected event: %d\n", event->status); + break; + } + return ret; +} + + +static void +setup_peer (struct PeerContext *p, const char *cfgname) +{ + p->cfg = GNUNET_CONFIGURATION_create (); +#if START_ARM + p->arm_proc = + GNUNET_OS_start_process (GNUNET_YES, NULL, NULL, "gnunet-service-arm", + "gnunet-service-arm", +#if VERBOSE + "-L", "DEBUG", +#endif + "-c", cfgname, NULL); +#endif + GNUNET_assert (GNUNET_OK == GNUNET_CONFIGURATION_load (p->cfg, cfgname)); +} + + +static void +stop_arm (struct PeerContext *p) +{ +#if START_ARM + if (NULL != p->arm_proc) + { + if (0 != GNUNET_OS_process_kill (p->arm_proc, SIGTERM)) + GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING, "kill"); + if (GNUNET_OS_process_wait (p->arm_proc) != GNUNET_OK) + GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING, "waitpid"); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "ARM process %u stopped\n", + GNUNET_OS_process_get_pid (p->arm_proc)); + GNUNET_OS_process_close (p->arm_proc); + p->arm_proc = NULL; + } +#endif + GNUNET_CONFIGURATION_destroy (p->cfg); +} + + +static void +run (void *cls, char *const *args, const char *cfgfile, + const struct GNUNET_CONFIGURATION_Handle *cfg) +{ + const char *keywords[] = { + "down_foo", + "down_bar", + }; + char *buf; + struct GNUNET_CONTAINER_MetaData *meta; + struct GNUNET_FS_Uri *kuri; + struct GNUNET_FS_FileInformation *fi1; + struct GNUNET_FS_FileInformation *fi2; + struct GNUNET_FS_FileInformation *fidir; + size_t i; + struct GNUNET_FS_BlockOptions bo; + + setup_peer (&p1, "test_fs_list_indexed_data.conf"); + fs = GNUNET_FS_start (cfg, "test-fs-list_indexed", &progress_cb, NULL, + GNUNET_FS_FLAGS_NONE, GNUNET_FS_OPTIONS_END); + GNUNET_assert (NULL != fs); + fn1 = GNUNET_DISK_mktemp ("gnunet-list_indexed-test-dst"); + buf = GNUNET_malloc (FILESIZE); + for (i = 0; i < FILESIZE; i++) + buf[i] = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 256); + GNUNET_assert (FILESIZE == + GNUNET_DISK_fn_write (fn1, buf, FILESIZE, + GNUNET_DISK_PERM_USER_READ | + GNUNET_DISK_PERM_USER_WRITE)); + GNUNET_free (buf); + + fn2 = GNUNET_DISK_mktemp ("gnunet-list_indexed-test-dst"); + buf = GNUNET_malloc (FILESIZE); + for (i = 0; i < FILESIZE; i++) + buf[i] = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 256); + GNUNET_assert (FILESIZE == + GNUNET_DISK_fn_write (fn2, buf, FILESIZE, + GNUNET_DISK_PERM_USER_READ | + GNUNET_DISK_PERM_USER_WRITE)); + GNUNET_free (buf); + + meta = GNUNET_CONTAINER_meta_data_create (); + kuri = GNUNET_FS_uri_ksk_create_from_args (2, keywords); + bo.content_priority = 42; + bo.anonymity_level = 1; + bo.replication_level = 0; + bo.expiration_time = GNUNET_TIME_relative_to_absolute (LIFETIME); + fi1 = + GNUNET_FS_file_information_create_from_file (fs, "list_indexed-context1", + fn1, kuri, meta, GNUNET_YES, + &bo); + GNUNET_assert (NULL != fi1); + bo.anonymity_level = 2; + fi2 = + GNUNET_FS_file_information_create_from_file (fs, "list_indexed-context2", + fn2, kuri, meta, GNUNET_YES, + &bo); + GNUNET_assert (NULL != fi2); + bo.anonymity_level = 3; + fidir = + GNUNET_FS_file_information_create_empty_directory (fs, + "list_indexed-context-dir", + kuri, meta, &bo, NULL); + GNUNET_assert (GNUNET_OK == GNUNET_FS_file_information_add (fidir, fi1)); + GNUNET_assert (GNUNET_OK == GNUNET_FS_file_information_add (fidir, fi2)); + GNUNET_FS_uri_destroy (kuri); + GNUNET_CONTAINER_meta_data_destroy (meta); + GNUNET_assert (NULL != fidir); + start = GNUNET_TIME_absolute_get (); + publish = + GNUNET_FS_publish_start (fs, fidir, NULL, NULL, NULL, + GNUNET_FS_PUBLISH_OPTION_NONE); + GNUNET_assert (publish != NULL); +} + + +int +main (int argc, char *argv[]) +{ + char *const argvx[] = { + "test-fs-list_indexed", + "-c", + "test_fs_list_indexed_data.conf", +#if VERBOSE + "-L", "DEBUG", +#endif + NULL + }; + struct GNUNET_GETOPT_CommandLineOption options[] = { + GNUNET_GETOPT_OPTION_END + }; + + GNUNET_log_setup ("test_fs_list_indexed", +#if VERBOSE + "DEBUG", +#else + "WARNING", +#endif + NULL); + GNUNET_PROGRAM_run ((sizeof (argvx) / sizeof (char *)) - 1, argvx, + "test-fs-list_indexed", "nohelp", options, &run, NULL); + stop_arm (&p1); + GNUNET_DISK_directory_remove ("/tmp/gnunet-test-fs-list-indexed/"); + if (fn1 != NULL) + { + GNUNET_DISK_directory_remove (fn1); + GNUNET_free (fn1); + } + if (fn2 != NULL) + { + GNUNET_DISK_directory_remove (fn2); + GNUNET_free (fn2); + } + return err; +} + +/* end of test_fs_list_indexed.c */ diff --git a/src/fs/test_fs_list_indexed_data.conf b/src/fs/test_fs_list_indexed_data.conf new file mode 100644 index 0000000..704ba4d --- /dev/null +++ b/src/fs/test_fs_list_indexed_data.conf @@ -0,0 +1,11 @@ +@INLINE@ test_fs_defaults.conf +[PATHS] +SERVICEHOME = /tmp/gnunet-test-fs-list-indexed/ +DEFAULTCONFIG = test_fs_list_indexed_data.conf + +[transport] +PLUGINS = + +[fs] +ACTIVEMIGRATION = NO + diff --git a/src/fs/test_fs_namespace.c b/src/fs/test_fs_namespace.c new file mode 100644 index 0000000..d25fd6f --- /dev/null +++ b/src/fs/test_fs_namespace.c @@ -0,0 +1,410 @@ +/* + This file is part of GNUnet. + (C) 2005, 2006, 2008, 2009 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/test_fs_namespace.c + * @brief Test for fs_namespace.c + * @author Christian Grothoff + */ +#include "platform.h" +#include "gnunet_util_lib.h" +#include "gnunet_arm_service.h" +#include "gnunet_fs_service.h" + +#define VERBOSE GNUNET_NO + +#define START_ARM GNUNET_YES + +static struct PeerContext p1; + +static GNUNET_HashCode nsid; + +static struct GNUNET_FS_Uri *sks_expect_uri; + +static struct GNUNET_FS_Uri *ksk_expect_uri; + +static struct GNUNET_FS_Handle *fs; + +static struct GNUNET_FS_SearchContext *sks_search; + +static struct GNUNET_FS_SearchContext *ksk_search; + +static GNUNET_SCHEDULER_TaskIdentifier kill_task; + +static int update_started; + +static int err; + +struct PeerContext +{ + struct GNUNET_CONFIGURATION_Handle *cfg; +#if START_ARM + struct GNUNET_OS_Process *arm_proc; +#endif +}; + + +static void +setup_peer (struct PeerContext *p, const char *cfgname) +{ + p->cfg = GNUNET_CONFIGURATION_create (); +#if START_ARM + p->arm_proc = + GNUNET_OS_start_process (GNUNET_YES, NULL, NULL, "gnunet-service-arm", + "gnunet-service-arm", +#if VERBOSE + "-L", "DEBUG", +#endif + "-c", cfgname, NULL); +#endif + GNUNET_assert (GNUNET_OK == GNUNET_CONFIGURATION_load (p->cfg, cfgname)); +} + + +static void +stop_arm (struct PeerContext *p) +{ +#if START_ARM + if (NULL != p->arm_proc) + { + if (0 != GNUNET_OS_process_kill (p->arm_proc, SIGTERM)) + GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING, "kill"); + if (GNUNET_OS_process_wait (p->arm_proc) != GNUNET_OK) + GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING, "waitpid"); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "ARM process %u stopped\n", + GNUNET_OS_process_get_pid (p->arm_proc)); + GNUNET_OS_process_close (p->arm_proc); + p->arm_proc = NULL; + } +#endif + GNUNET_CONFIGURATION_destroy (p->cfg); +} + + +static void +abort_ksk_search_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + if (ksk_search != NULL) + { + GNUNET_FS_search_stop (ksk_search); + ksk_search = NULL; + if (sks_search == NULL) + { + GNUNET_FS_stop (fs); + if (GNUNET_SCHEDULER_NO_TASK != kill_task) + GNUNET_SCHEDULER_cancel (kill_task); + } + } +} + + +static void +abort_sks_search_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct GNUNET_FS_Namespace *ns; + + if (sks_search == NULL) + return; + GNUNET_FS_search_stop (sks_search); + sks_search = NULL; + ns = GNUNET_FS_namespace_create (fs, "testNamespace"); + GNUNET_assert (NULL != ns); + GNUNET_assert (GNUNET_OK == GNUNET_FS_namespace_delete (ns, GNUNET_YES)); + if (ksk_search == NULL) + { + GNUNET_FS_stop (fs); + if (GNUNET_SCHEDULER_NO_TASK != kill_task) + GNUNET_SCHEDULER_cancel (kill_task); + } +} + + +static void +do_timeout (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + FPRINTF (stderr, "%s", "Operation timed out\n"); + kill_task = GNUNET_SCHEDULER_NO_TASK; + abort_sks_search_task (NULL, tc); + abort_ksk_search_task (NULL, tc); +} + + + +static void * +progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *event) +{ + switch (event->status) + { + case GNUNET_FS_STATUS_SEARCH_RESULT: + if (sks_search == event->value.search.sc) + { + if (!GNUNET_FS_uri_test_equal + (sks_expect_uri, event->value.search.specifics.result.uri)) + { + FPRINTF (stderr, "%s", "Wrong result for sks search!\n"); + err = 1; + } + /* give system 1ms to initiate update search! */ + GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MILLISECONDS, + &abort_sks_search_task, NULL); + } + else if (ksk_search == event->value.search.sc) + { + if (!GNUNET_FS_uri_test_equal + (ksk_expect_uri, event->value.search.specifics.result.uri)) + { + FPRINTF (stderr, "%s", "Wrong result for ksk search!\n"); + err = 1; + } + GNUNET_SCHEDULER_add_continuation (&abort_ksk_search_task, NULL, + GNUNET_SCHEDULER_REASON_PREREQ_DONE); + } + else + { + FPRINTF (stderr, "%s", "Unexpected search result received!\n"); + GNUNET_break (0); + } + break; + case GNUNET_FS_STATUS_SEARCH_ERROR: + FPRINTF (stderr, "Error searching file: %s\n", + event->value.search.specifics.error.message); + if (sks_search == event->value.search.sc) + GNUNET_SCHEDULER_add_continuation (&abort_sks_search_task, NULL, + GNUNET_SCHEDULER_REASON_PREREQ_DONE); + else if (ksk_search == event->value.search.sc) + GNUNET_SCHEDULER_add_continuation (&abort_ksk_search_task, NULL, + GNUNET_SCHEDULER_REASON_PREREQ_DONE); + else + GNUNET_break (0); + break; + case GNUNET_FS_STATUS_SEARCH_START: + GNUNET_assert ((NULL == event->value.search.cctx) || + (0 == strcmp ("sks_search", event->value.search.cctx)) || + (0 == strcmp ("ksk_search", event->value.search.cctx))); + if (NULL == event->value.search.cctx) + { + GNUNET_assert (0 == strcmp ("sks_search", event->value.search.pctx)); + update_started = GNUNET_YES; + } + GNUNET_assert (1 == event->value.search.anonymity); + break; + case GNUNET_FS_STATUS_SEARCH_RESULT_STOPPED: + return NULL; + case GNUNET_FS_STATUS_SEARCH_STOPPED: + return NULL; + default: + FPRINTF (stderr, "Unexpected event: %d\n", event->status); + break; + } + return event->value.search.cctx; +} + + +static void +publish_cont (void *cls, const struct GNUNET_FS_Uri *ksk_uri, const char *emsg) +{ + char *msg; + struct GNUNET_FS_Uri *sks_uri; + char sbuf[1024]; + struct GNUNET_CRYPTO_HashAsciiEncoded enc; + + if (NULL != emsg) + { + FPRINTF (stderr, "Error publishing: %s\n", emsg); + err = 1; + GNUNET_FS_stop (fs); + return; + } + GNUNET_CRYPTO_hash_to_enc (&nsid, &enc); + GNUNET_snprintf (sbuf, sizeof (sbuf), "gnunet://fs/sks/%s/this", &enc); + sks_uri = GNUNET_FS_uri_parse (sbuf, &msg); + if (NULL == sks_uri) + { + FPRINTF (stderr, "failed to parse URI `%s': %s\n", sbuf, msg); + err = 1; + GNUNET_FS_stop (fs); + GNUNET_free_non_null (msg); + return; + } + ksk_search = + GNUNET_FS_search_start (fs, ksk_uri, 1, GNUNET_FS_SEARCH_OPTION_NONE, + "ksk_search"); + sks_search = + GNUNET_FS_search_start (fs, sks_uri, 1, GNUNET_FS_SEARCH_OPTION_NONE, + "sks_search"); + GNUNET_FS_uri_destroy (sks_uri); +} + + +static void +sks_cont (void *cls, const struct GNUNET_FS_Uri *uri, const char *emsg) +{ + struct GNUNET_CONTAINER_MetaData *meta; + struct GNUNET_FS_Uri *ksk_uri; + char *msg; + struct GNUNET_FS_BlockOptions bo; + + meta = GNUNET_CONTAINER_meta_data_create (); + msg = NULL; + ksk_uri = GNUNET_FS_uri_parse ("gnunet://fs/ksk/ns-search", &msg); + GNUNET_assert (NULL == msg); + ksk_expect_uri = GNUNET_FS_uri_dup (uri); + bo.content_priority = 1; + bo.anonymity_level = 1; + bo.replication_level = 0; + bo.expiration_time = + GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_MINUTES); + GNUNET_FS_publish_ksk (fs, ksk_uri, meta, uri, &bo, + GNUNET_FS_PUBLISH_OPTION_NONE, &publish_cont, NULL); + GNUNET_FS_uri_destroy (ksk_uri); + GNUNET_CONTAINER_meta_data_destroy (meta); +} + + +static void +adv_cont (void *cls, const struct GNUNET_FS_Uri *uri, const char *emsg) +{ + struct GNUNET_CONTAINER_MetaData *meta; + struct GNUNET_FS_Namespace *ns; + struct GNUNET_FS_BlockOptions bo; + + if (NULL != emsg) + { + FPRINTF (stderr, "Error publishing: %s\n", emsg); + err = 1; + GNUNET_FS_stop (fs); + return; + } + ns = GNUNET_FS_namespace_create (fs, "testNamespace"); + GNUNET_assert (NULL != ns); + meta = GNUNET_CONTAINER_meta_data_create (); + GNUNET_assert (NULL == emsg); + sks_expect_uri = GNUNET_FS_uri_dup (uri); + bo.content_priority = 1; + bo.anonymity_level = 1; + bo.replication_level = 0; + bo.expiration_time = + GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_MINUTES); + GNUNET_FS_publish_sks (fs, ns, "this", "next", meta, uri, /* FIXME: this is non-sense (use CHK URI!?) */ + &bo, GNUNET_FS_PUBLISH_OPTION_NONE, &sks_cont, NULL); + GNUNET_CONTAINER_meta_data_destroy (meta); + GNUNET_FS_namespace_delete (ns, GNUNET_NO); +} + + +static void +ns_iterator (void *cls, const char *name, const GNUNET_HashCode * id) +{ + int *ok = cls; + + if (0 != strcmp (name, "testNamespace")) + return; + *ok = GNUNET_YES; + nsid = *id; +} + + +static void +testNamespace () +{ + struct GNUNET_FS_Namespace *ns; + struct GNUNET_FS_BlockOptions bo; + struct GNUNET_CONTAINER_MetaData *meta; + struct GNUNET_FS_Uri *ksk_uri; + int ok; + + ns = GNUNET_FS_namespace_create (fs, "testNamespace"); + GNUNET_assert (NULL != ns); + ok = GNUNET_NO; + GNUNET_FS_namespace_list (fs, &ns_iterator, &ok); + if (GNUNET_NO == ok) + { + FPRINTF (stderr, "%s", "namespace_list failed to find namespace!\n"); + GNUNET_FS_namespace_delete (ns, GNUNET_YES); + GNUNET_FS_stop (fs); + err = 1; + return; + } + meta = GNUNET_CONTAINER_meta_data_create (); + ksk_uri = GNUNET_FS_uri_parse ("gnunet://fs/ksk/testnsa", NULL); + bo.content_priority = 1; + bo.anonymity_level = 1; + bo.replication_level = 0; + bo.expiration_time = + GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_MINUTES); + GNUNET_FS_namespace_advertise (fs, ksk_uri, ns, meta, &bo, "root", &adv_cont, + NULL); + kill_task = + GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES, &do_timeout, + NULL); + GNUNET_FS_uri_destroy (ksk_uri); + GNUNET_FS_namespace_delete (ns, GNUNET_NO); + GNUNET_CONTAINER_meta_data_destroy (meta); +} + + +static void +run (void *cls, char *const *args, const char *cfgfile, + const struct GNUNET_CONFIGURATION_Handle *cfg) +{ + setup_peer (&p1, "test_fs_namespace_data.conf"); + fs = GNUNET_FS_start (cfg, "test-fs-namespace", &progress_cb, NULL, + GNUNET_FS_FLAGS_NONE, GNUNET_FS_OPTIONS_END); + testNamespace (); +} + + +int +main (int argc, char *argv[]) +{ + char *const argvx[] = { + "test-fs-namespace", + "-c", + "test_fs_namespace_data.conf", +#if VERBOSE + "-L", "DEBUG", +#endif + NULL + }; + struct GNUNET_GETOPT_CommandLineOption options[] = { + GNUNET_GETOPT_OPTION_END + }; + + GNUNET_log_setup ("test_fs_namespace", +#if VERBOSE + "DEBUG", +#else + "WARNING", +#endif + NULL); + GNUNET_PROGRAM_run ((sizeof (argvx) / sizeof (char *)) - 1, argvx, + "test-fs-namespace", "nohelp", options, &run, NULL); + stop_arm (&p1); + if (GNUNET_YES != update_started) + { + FPRINTF (stderr, "%s", "Update search never started!\n"); + err = 1; + } + GNUNET_DISK_directory_remove ("/tmp/gnunet-test-fs-namespace/"); + return err; +} + + +/* end of test_fs_namespace.c */ diff --git a/src/fs/test_fs_namespace_data.conf b/src/fs/test_fs_namespace_data.conf new file mode 100644 index 0000000..3cdd241 --- /dev/null +++ b/src/fs/test_fs_namespace_data.conf @@ -0,0 +1,8 @@ +@INLINE@ test_fs_defaults.conf +[PATHS] +SERVICEHOME = /tmp/gnunet-test-fs-namespace/ +DEFAULTCONFIG = test_fs_namespace_data.conf + +[transport] +PLUGINS = + diff --git a/src/fs/test_fs_namespace_list_updateable.c b/src/fs/test_fs_namespace_list_updateable.c new file mode 100644 index 0000000..44775ac --- /dev/null +++ b/src/fs/test_fs_namespace_list_updateable.c @@ -0,0 +1,244 @@ +/* + This file is part of GNUnet. + (C) 2005, 2006, 2008, 2009 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/test_fs_namespace_list_updateable.c + * @brief Test for fs_namespace_list_updateable.c + * @author Christian Grothoff + */ +#include "platform.h" +#include "gnunet_util_lib.h" +#include "gnunet_arm_service.h" +#include "gnunet_fs_service.h" + +#define VERBOSE GNUNET_NO + +#define START_ARM GNUNET_YES + +static struct PeerContext p1; + +static struct GNUNET_FS_Handle *fs; + +static int err; + +static struct GNUNET_FS_Namespace *ns; + +static struct GNUNET_CONTAINER_MetaData *meta; + +static struct GNUNET_FS_Uri *uri_this; + +static struct GNUNET_FS_Uri *uri_next; + +static struct GNUNET_FS_BlockOptions bo; + + +struct PeerContext +{ + struct GNUNET_CONFIGURATION_Handle *cfg; +#if START_ARM + struct GNUNET_OS_Process *arm_proc; +#endif +}; + + +static void * +progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *event) +{ + return NULL; +} + + +static void +setup_peer (struct PeerContext *p, const char *cfgname) +{ + p->cfg = GNUNET_CONFIGURATION_create (); +#if START_ARM + p->arm_proc = + GNUNET_OS_start_process (GNUNET_YES, NULL, NULL, "gnunet-service-arm", + "gnunet-service-arm", +#if VERBOSE + "-L", "DEBUG", +#endif + "-c", cfgname, NULL); +#endif + GNUNET_assert (GNUNET_OK == GNUNET_CONFIGURATION_load (p->cfg, cfgname)); +} + + +static void +stop_arm (struct PeerContext *p) +{ +#if START_ARM + if (NULL != p->arm_proc) + { + if (0 != GNUNET_OS_process_kill (p->arm_proc, SIGTERM)) + GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING, "kill"); + if (GNUNET_OS_process_wait (p->arm_proc) != GNUNET_OK) + GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING, "waitpid"); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "ARM process %u stopped\n", + GNUNET_OS_process_get_pid (p->arm_proc)); + GNUNET_OS_process_close (p->arm_proc); + p->arm_proc = NULL; + } +#endif + if (uri_this != NULL) + GNUNET_FS_uri_destroy (uri_this); + if (uri_next != NULL) + GNUNET_FS_uri_destroy (uri_next); + if (ns != NULL) + GNUNET_FS_namespace_delete (ns, GNUNET_NO); + if (meta != NULL) + GNUNET_CONTAINER_meta_data_destroy (meta); + GNUNET_CONFIGURATION_destroy (p->cfg); +} + + + +static void +check_next (void *cls, const char *last_id, + const struct GNUNET_FS_Uri *last_uri, + const struct GNUNET_CONTAINER_MetaData *last_meta, + const char *next_id) +{ + GNUNET_break (0 == strcmp (last_id, "next")); + GNUNET_break (0 == strcmp (next_id, "future")); + err -= 4; +} + + +static void +check_this_next (void *cls, const char *last_id, + const struct GNUNET_FS_Uri *last_uri, + const struct GNUNET_CONTAINER_MetaData *last_meta, + const char *next_id) +{ + GNUNET_break (0 == strcmp (last_id, "this")); + GNUNET_break (0 == strcmp (next_id, "next")); + err -= 2; + err += 4; + GNUNET_FS_namespace_list_updateable (ns, next_id, &check_next, NULL); +} + + +static void +sks_cont_next (void *cls, const struct GNUNET_FS_Uri *uri, const char *emsg) +{ + GNUNET_assert (NULL == emsg); + err += 2; + GNUNET_FS_namespace_list_updateable (ns, NULL, &check_this_next, NULL); + +} + + +static void +check_this (void *cls, const char *last_id, + const struct GNUNET_FS_Uri *last_uri, + const struct GNUNET_CONTAINER_MetaData *last_meta, + const char *next_id) +{ + GNUNET_break (0 == strcmp (last_id, "this")); + GNUNET_break (0 == strcmp (next_id, "next")); + err -= 1; +} + + +static void +sks_cont_this (void *cls, const struct GNUNET_FS_Uri *uri, const char *emsg) +{ + + GNUNET_assert (NULL == emsg); + err = 1; + GNUNET_FS_namespace_list_updateable (ns, NULL, &check_this, NULL); + GNUNET_FS_publish_sks (fs, ns, "next", "future", meta, uri_next, &bo, + GNUNET_FS_PUBLISH_OPTION_NONE, &sks_cont_next, NULL); + +} + + + +static void +testNamespace () +{ + + ns = GNUNET_FS_namespace_create (fs, "testNamespace"); + GNUNET_assert (NULL != ns); + bo.content_priority = 1; + bo.anonymity_level = 1; + bo.replication_level = 0; + bo.expiration_time = + GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_MINUTES); + meta = GNUNET_CONTAINER_meta_data_create (); + + uri_this = + GNUNET_FS_uri_parse + ("gnunet://fs/chk/C282GG70GKK41O4551011DO413KFBVTVMQG1OG30I0K4045N0G41HAPB82G680A02JRVVFO8URVRU2F159011DO41000000022RG820.RNVVVVOOLCLK065B5D04HTNVNSIB2AI022RG8200HSLK1CO1000ATQ98824DMA2032LIMG50CG0K057NVUVG200000H000004400000.42", + NULL); + uri_next = + GNUNET_FS_uri_parse + ("gnunet://fs/chk/C282GG70GKK41O4551011DO413KFBVTVMQG1OG30I0K4045N0G41HAPB82G680A02JRVVFO8URVRU2F159011DO41000000022RG820.RNVVVVOOLCLK065B5D04HTNVNSIB2AI022RG8200HSLK1CO1000ATQ98824DMA2032LIMG50CG0K057NVUVG200000H000004400000.43", + NULL); + GNUNET_FS_publish_sks (fs, ns, "this", "next", meta, uri_this, &bo, + GNUNET_FS_PUBLISH_OPTION_NONE, &sks_cont_this, NULL); +} + + +static void +run (void *cls, char *const *args, const char *cfgfile, + const struct GNUNET_CONFIGURATION_Handle *cfg) +{ + setup_peer (&p1, "test_fs_namespace_data.conf"); + fs = GNUNET_FS_start (cfg, "test-fs-namespace", &progress_cb, NULL, + GNUNET_FS_FLAGS_NONE, GNUNET_FS_OPTIONS_END); + testNamespace (); +} + + +int +main (int argc, char *argv[]) +{ + char *const argvx[] = { + "test-fs-namespace", + "-c", + "test_fs_namespace_data.conf", +#if VERBOSE + "-L", "DEBUG", +#endif + NULL + }; + struct GNUNET_GETOPT_CommandLineOption options[] = { + GNUNET_GETOPT_OPTION_END + }; + + GNUNET_log_setup ("test_fs_namespace_list_updateable", +#if VERBOSE + "DEBUG", +#else + "WARNING", +#endif + NULL); + GNUNET_PROGRAM_run ((sizeof (argvx) / sizeof (char *)) - 1, argvx, + "test-fs-namespace", "nohelp", options, &run, NULL); + stop_arm (&p1); + GNUNET_DISK_directory_remove ("/tmp/gnunet-test-fs-namespace/"); + return err; +} + + +/* end of test_fs_namespace_list_updateable.c */ diff --git a/src/fs/test_fs_publish.c b/src/fs/test_fs_publish.c new file mode 100644 index 0000000..e527438 --- /dev/null +++ b/src/fs/test_fs_publish.c @@ -0,0 +1,323 @@ +/* + This file is part of GNUnet. + (C) 2004, 2005, 2006, 2008, 2009 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/test_fs_publish.c + * @brief simple testcase for publish operation (indexing, listing + * indexed, directory structure) + * @author Christian Grothoff + */ + +#include "platform.h" +#include "gnunet_util_lib.h" +#include "gnunet_arm_service.h" +#include "gnunet_fs_service.h" + +#define VERBOSE GNUNET_NO + +#define START_ARM GNUNET_YES + +/** + * File-size we use for testing. + */ +#define FILESIZE (1024 * 1024 * 2) + +/** + * How long until we give up on transmitting the message? + */ +#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 60) + +/** + * How long should our test-content live? + */ +#define LIFETIME GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15) + +struct PeerContext +{ + struct GNUNET_CONFIGURATION_Handle *cfg; +#if START_ARM + struct GNUNET_OS_Process *arm_proc; +#endif +}; + +static struct PeerContext p1; + +static struct GNUNET_TIME_Absolute start; + +static struct GNUNET_FS_Handle *fs; + +static struct GNUNET_FS_PublishContext *publish; + +static char *fn1; + +static char *fn2; + +static int err; + +static void +abort_publish_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + GNUNET_FS_publish_stop (publish); + publish = NULL; + GNUNET_DISK_directory_remove (fn1); + GNUNET_free (fn1); + fn1 = NULL; + GNUNET_DISK_directory_remove (fn2); + GNUNET_free (fn2); + fn2 = NULL; +} + + +static void * +progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *event) +{ + void *ret; + + ret = NULL; + switch (event->status) + { + case GNUNET_FS_STATUS_PUBLISH_COMPLETED: + ret = event->value.publish.cctx; + printf ("Publish complete, %llu kbps.\n", + (unsigned long long) (FILESIZE * 1000 / + (1 + + GNUNET_TIME_absolute_get_duration + (start).rel_value) / 1024)); + if (0 == strcmp ("publish-context-dir", event->value.publish.cctx)) + GNUNET_SCHEDULER_add_continuation (&abort_publish_task, NULL, + GNUNET_SCHEDULER_REASON_PREREQ_DONE); + break; + case GNUNET_FS_STATUS_PUBLISH_PROGRESS: + ret = event->value.publish.cctx; + GNUNET_assert (publish == event->value.publish.pc); +#if VERBOSE + printf ("Publish is progressing (%llu/%llu at level %u off %llu)...\n", + (unsigned long long) event->value.publish.completed, + (unsigned long long) event->value.publish.size, + event->value.publish.specifics.progress.depth, + (unsigned long long) event->value.publish.specifics. + progress.offset); +#endif + break; + case GNUNET_FS_STATUS_PUBLISH_ERROR: + ret = event->value.publish.cctx; + FPRINTF (stderr, "Error publishing file: %s\n", + event->value.publish.specifics.error.message); + err = 1; + if (0 == strcmp ("publish-context-dir", event->value.publish.cctx)) + { + FPRINTF (stderr, "Scheduling abort task for error on `%s'\n", + (const char *) event->value.publish.cctx); + GNUNET_SCHEDULER_add_continuation (&abort_publish_task, NULL, + GNUNET_SCHEDULER_REASON_PREREQ_DONE); + } + break; + case GNUNET_FS_STATUS_PUBLISH_START: + ret = event->value.publish.cctx; + if (0 == strcmp ("publish-context1", event->value.publish.cctx)) + { + GNUNET_assert (0 == + strcmp ("publish-context-dir", event->value.publish.pctx)); + GNUNET_assert (FILESIZE == event->value.publish.size); + GNUNET_assert (0 == event->value.publish.completed); + GNUNET_assert (1 == event->value.publish.anonymity); + } + else if (0 == strcmp ("publish-context2", event->value.publish.cctx)) + { + GNUNET_assert (0 == + strcmp ("publish-context-dir", event->value.publish.pctx)); + GNUNET_assert (FILESIZE == event->value.publish.size); + GNUNET_assert (0 == event->value.publish.completed); + GNUNET_assert (2 == event->value.publish.anonymity); + } + else if (0 == strcmp ("publish-context-dir", event->value.publish.cctx)) + { + GNUNET_assert (0 == event->value.publish.completed); + GNUNET_assert (3 == event->value.publish.anonymity); + } + else + GNUNET_assert (0); + break; + case GNUNET_FS_STATUS_PUBLISH_STOPPED: + if (0 == strcmp ("publish-context-dir", event->value.publish.cctx)) + GNUNET_assert (publish == event->value.publish.pc); + break; + default: + printf ("Unexpected event: %d\n", event->status); + break; + } + return ret; +} + + +static void +setup_peer (struct PeerContext *p, const char *cfgname) +{ + p->cfg = GNUNET_CONFIGURATION_create (); +#if START_ARM + p->arm_proc = + GNUNET_OS_start_process (GNUNET_YES, NULL, NULL, "gnunet-service-arm", + "gnunet-service-arm", +#if VERBOSE + "-L", "DEBUG", +#endif + "-c", cfgname, NULL); +#endif + GNUNET_assert (GNUNET_OK == GNUNET_CONFIGURATION_load (p->cfg, cfgname)); +} + + +static void +stop_arm (struct PeerContext *p) +{ +#if START_ARM + if (NULL != p->arm_proc) + { + if (0 != GNUNET_OS_process_kill (p->arm_proc, SIGTERM)) + GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING, "kill"); + if (GNUNET_OS_process_wait (p->arm_proc) != GNUNET_OK) + GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING, "waitpid"); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "ARM process %u stopped\n", + GNUNET_OS_process_get_pid (p->arm_proc)); + GNUNET_OS_process_close (p->arm_proc); + p->arm_proc = NULL; + } +#endif + GNUNET_CONFIGURATION_destroy (p->cfg); +} + + +static void +run (void *cls, char *const *args, const char *cfgfile, + const struct GNUNET_CONFIGURATION_Handle *cfg) +{ + const char *keywords[] = { + "down_foo", + "down_bar", + }; + char *buf; + struct GNUNET_CONTAINER_MetaData *meta; + struct GNUNET_FS_Uri *kuri; + struct GNUNET_FS_FileInformation *fi1; + struct GNUNET_FS_FileInformation *fi2; + struct GNUNET_FS_FileInformation *fidir; + size_t i; + struct GNUNET_FS_BlockOptions bo; + + setup_peer (&p1, "test_fs_publish_data.conf"); + fs = GNUNET_FS_start (cfg, "test-fs-publish", &progress_cb, NULL, + GNUNET_FS_FLAGS_NONE, GNUNET_FS_OPTIONS_END); + GNUNET_assert (NULL != fs); + fn1 = GNUNET_DISK_mktemp ("gnunet-publish-test-dst"); + buf = GNUNET_malloc (FILESIZE); + for (i = 0; i < FILESIZE; i++) + buf[i] = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 256); + GNUNET_assert (FILESIZE == + GNUNET_DISK_fn_write (fn1, buf, FILESIZE, + GNUNET_DISK_PERM_USER_READ | + GNUNET_DISK_PERM_USER_WRITE)); + GNUNET_free (buf); + + fn2 = GNUNET_DISK_mktemp ("gnunet-publish-test-dst"); + buf = GNUNET_malloc (FILESIZE); + for (i = 0; i < FILESIZE; i++) + buf[i] = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 256); + GNUNET_assert (FILESIZE == + GNUNET_DISK_fn_write (fn2, buf, FILESIZE, + GNUNET_DISK_PERM_USER_READ | + GNUNET_DISK_PERM_USER_WRITE)); + GNUNET_free (buf); + + meta = GNUNET_CONTAINER_meta_data_create (); + kuri = GNUNET_FS_uri_ksk_create_from_args (2, keywords); + bo.content_priority = 42; + bo.anonymity_level = 1; + bo.replication_level = 0; + bo.expiration_time = GNUNET_TIME_relative_to_absolute (LIFETIME); + + fi1 = + GNUNET_FS_file_information_create_from_file (fs, "publish-context1", fn1, + kuri, meta, GNUNET_YES, &bo); + + GNUNET_assert (NULL != fi1); + bo.anonymity_level = 2; + fi2 = + GNUNET_FS_file_information_create_from_file (fs, "publish-context2", fn2, + kuri, meta, GNUNET_YES, &bo); + GNUNET_assert (NULL != fi2); + bo.anonymity_level = 3; + fidir = + GNUNET_FS_file_information_create_empty_directory (fs, + "publish-context-dir", + kuri, meta, &bo, NULL); + GNUNET_assert (GNUNET_OK == GNUNET_FS_file_information_add (fidir, fi1)); + GNUNET_assert (GNUNET_OK == GNUNET_FS_file_information_add (fidir, fi2)); + GNUNET_FS_uri_destroy (kuri); + GNUNET_CONTAINER_meta_data_destroy (meta); + GNUNET_assert (NULL != fidir); + start = GNUNET_TIME_absolute_get (); + publish = + GNUNET_FS_publish_start (fs, fidir, NULL, NULL, NULL, + GNUNET_FS_PUBLISH_OPTION_NONE); + GNUNET_assert (publish != NULL); +} + + +int +main (int argc, char *argv[]) +{ + char *const argvx[] = { + "test-fs-publish", + "-c", + "test_fs_publish_data.conf", +#if VERBOSE + "-L", "DEBUG", +#endif + NULL + }; + struct GNUNET_GETOPT_CommandLineOption options[] = { + GNUNET_GETOPT_OPTION_END + }; + + GNUNET_log_setup ("test_fs_publish", +#if VERBOSE + "DEBUG", +#else + "WARNING", +#endif + NULL); + GNUNET_PROGRAM_run ((sizeof (argvx) / sizeof (char *)) - 1, argvx, + "test-fs-publish", "nohelp", options, &run, NULL); + stop_arm (&p1); + GNUNET_DISK_directory_remove ("/tmp/gnunet-test-fs-publish/"); + if (fn1 != NULL) + { + GNUNET_DISK_directory_remove (fn1); + GNUNET_free (fn1); + } + if (fn2 != NULL) + { + GNUNET_DISK_directory_remove (fn2); + GNUNET_free (fn2); + } + return err; +} + +/* end of test_fs_publish.c */ diff --git a/src/fs/test_fs_publish_data.conf b/src/fs/test_fs_publish_data.conf new file mode 100644 index 0000000..234f2b0 --- /dev/null +++ b/src/fs/test_fs_publish_data.conf @@ -0,0 +1,11 @@ +@INLINE@ test_fs_defaults.conf +[PATHS] +SERVICEHOME = /tmp/gnunet-test-fs-publish/ +DEFAULTCONFIG = test_fs_publish_data.conf + +[transport] +PLUGINS = + +[fs] +ACTIVEMIGRATION = NO + diff --git a/src/fs/test_fs_publish_persistence.c b/src/fs/test_fs_publish_persistence.c new file mode 100644 index 0000000..7707eac --- /dev/null +++ b/src/fs/test_fs_publish_persistence.c @@ -0,0 +1,384 @@ +/* + This file is part of GNUnet. + (C) 2004, 2005, 2006, 2008, 2009, 2010 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/test_fs_publish_persistence.c + * @brief simple testcase for persistence of simple publish operation + * @author Christian Grothoff + */ + +#include "platform.h" +#include "gnunet_util_lib.h" +#include "gnunet_arm_service.h" +#include "gnunet_fs_service.h" + +#define VERBOSE GNUNET_NO + +#define START_ARM GNUNET_YES + +/** + * File-size we use for testing. + */ +#define FILESIZE (1024 * 1024 * 2) + +/** + * How long until we give up on transmitting the message? + */ +#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 60) + +/** + * How long should our test-content live? + */ +#define LIFETIME GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15) + +struct PeerContext +{ + struct GNUNET_CONFIGURATION_Handle *cfg; +#if START_ARM + struct GNUNET_OS_Process *arm_proc; +#endif +}; + +static struct PeerContext p1; + +static struct GNUNET_TIME_Absolute start; + +static struct GNUNET_FS_Handle *fs; + +static const struct GNUNET_CONFIGURATION_Handle *cfg; + +static struct GNUNET_FS_PublishContext *publish; + +static struct GNUNET_FS_PublishContext *publish; + +static char *fn1; + +static char *fn2; + +static int err; + +static GNUNET_SCHEDULER_TaskIdentifier rtask; + +static void +abort_publish_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + GNUNET_FS_publish_stop (publish); + publish = NULL; + GNUNET_DISK_directory_remove (fn1); + GNUNET_free (fn1); + fn1 = NULL; + GNUNET_DISK_directory_remove (fn2); + GNUNET_free (fn2); + fn2 = NULL; + GNUNET_FS_stop (fs); + fs = NULL; + if (GNUNET_SCHEDULER_NO_TASK != rtask) + { + GNUNET_SCHEDULER_cancel (rtask); + rtask = GNUNET_SCHEDULER_NO_TASK; + } +} + + +static void * +progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *event); + + +static void +restart_fs_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + rtask = GNUNET_SCHEDULER_NO_TASK; + GNUNET_FS_stop (fs); + fs = GNUNET_FS_start (cfg, "test-fs-publish-persistence", &progress_cb, NULL, + GNUNET_FS_FLAGS_PERSISTENCE, GNUNET_FS_OPTIONS_END); +} + + +/** + * Consider scheduling the restart-task. + * Only runs the restart task once per event + * category. + * + * @param ev type of the event to consider + */ +static void +consider_restart (int ev) +{ + static int prev[32]; + static int off; + int i; + + for (i = 0; i < off; i++) + if (prev[i] == ev) + return; + prev[off++] = ev; + rtask = + GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_URGENT, + &restart_fs_task, NULL); +} + + +static void * +progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *event) +{ + void *ret; + + ret = NULL; + switch (event->status) + { + case GNUNET_FS_STATUS_PUBLISH_COMPLETED: + consider_restart (event->status); + ret = event->value.publish.cctx; + printf ("Publish complete, %llu kbps.\n", + (unsigned long long) (FILESIZE * 1000LL / + (1 + + GNUNET_TIME_absolute_get_duration + (start).rel_value) / 1024)); + if (0 == strcmp ("publish-context-dir", event->value.publish.cctx)) + GNUNET_SCHEDULER_add_now (&abort_publish_task, NULL); + break; + case GNUNET_FS_STATUS_PUBLISH_PROGRESS: + consider_restart (event->status); + ret = event->value.publish.cctx; + GNUNET_assert (publish == event->value.publish.pc); +#if VERBOSE + printf ("Publish is progressing (%llu/%llu at level %u off %llu)...\n", + (unsigned long long) event->value.publish.completed, + (unsigned long long) event->value.publish.size, + event->value.publish.specifics.progress.depth, + (unsigned long long) event->value.publish.specifics. + progress.offset); +#endif + break; + case GNUNET_FS_STATUS_PUBLISH_SUSPEND: + if (event->value.publish.pc == publish) + publish = NULL; + break; + case GNUNET_FS_STATUS_PUBLISH_RESUME: + if (NULL == publish) + { + GNUNET_assert (GNUNET_YES == + GNUNET_FS_file_information_is_directory (event-> + value.publish. + fi)); + publish = event->value.publish.pc; + return "publish-context-dir"; + } + break; + case GNUNET_FS_STATUS_PUBLISH_ERROR: + ret = event->value.publish.cctx; + FPRINTF (stderr, "Error publishing file: %s\n", + event->value.publish.specifics.error.message); + err = 1; + GNUNET_SCHEDULER_add_now (&abort_publish_task, NULL); + break; + case GNUNET_FS_STATUS_PUBLISH_START: + consider_restart (event->status); + publish = event->value.publish.pc; + ret = event->value.publish.cctx; + if (0 == strcmp ("publish-context1", event->value.publish.cctx)) + { + GNUNET_assert (0 == + strcmp ("publish-context-dir", event->value.publish.pctx)); + GNUNET_assert (FILESIZE == event->value.publish.size); + GNUNET_assert (0 == event->value.publish.completed); + GNUNET_assert (1 == event->value.publish.anonymity); + } + else if (0 == strcmp ("publish-context2", event->value.publish.cctx)) + { + GNUNET_assert (0 == + strcmp ("publish-context-dir", event->value.publish.pctx)); + GNUNET_assert (FILESIZE == event->value.publish.size); + GNUNET_assert (0 == event->value.publish.completed); + GNUNET_assert (2 == event->value.publish.anonymity); + } + else if (0 == strcmp ("publish-context-dir", event->value.publish.cctx)) + { + GNUNET_assert (0 == event->value.publish.completed); + GNUNET_assert (3 == event->value.publish.anonymity); + } + else + GNUNET_assert (0); + break; + case GNUNET_FS_STATUS_PUBLISH_STOPPED: + consider_restart (event->status); + if (0 == strcmp ("publish-context-dir", event->value.publish.cctx)) + GNUNET_assert (publish == event->value.publish.pc); + break; + default: + printf ("Unexpected event: %d\n", event->status); + break; + } + return ret; +} + + +static void +setup_peer (struct PeerContext *p, const char *cfgname) +{ + p->cfg = GNUNET_CONFIGURATION_create (); +#if START_ARM + p->arm_proc = + GNUNET_OS_start_process (GNUNET_YES, NULL, NULL, "gnunet-service-arm", + "gnunet-service-arm", +#if VERBOSE + "-L", "DEBUG", +#endif + "-c", cfgname, NULL); +#endif + GNUNET_assert (GNUNET_OK == GNUNET_CONFIGURATION_load (p->cfg, cfgname)); +} + + +static void +stop_arm (struct PeerContext *p) +{ +#if START_ARM + if (NULL != p->arm_proc) + { + if (0 != GNUNET_OS_process_kill (p->arm_proc, SIGTERM)) + GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING, "kill"); + if (GNUNET_OS_process_wait (p->arm_proc) != GNUNET_OK) + GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING, "waitpid"); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "ARM process %u stopped\n", + GNUNET_OS_process_get_pid (p->arm_proc)); + GNUNET_OS_process_close (p->arm_proc); + p->arm_proc = NULL; + } +#endif + GNUNET_CONFIGURATION_destroy (p->cfg); +} + + +static void +run (void *cls, char *const *args, const char *cfgfile, + const struct GNUNET_CONFIGURATION_Handle *c) +{ + const char *keywords[] = { + "down_foo", + "down_bar", + }; + char *buf; + struct GNUNET_CONTAINER_MetaData *meta; + struct GNUNET_FS_Uri *kuri; + struct GNUNET_FS_FileInformation *fi1; + struct GNUNET_FS_FileInformation *fi2; + struct GNUNET_FS_FileInformation *fidir; + size_t i; + struct GNUNET_FS_BlockOptions bo; + + cfg = c; + setup_peer (&p1, "test_fs_publish_data.conf"); + fs = GNUNET_FS_start (cfg, "test-fs-publish-persistence", &progress_cb, NULL, + GNUNET_FS_FLAGS_PERSISTENCE, GNUNET_FS_OPTIONS_END); + GNUNET_assert (NULL != fs); + fn1 = GNUNET_DISK_mktemp ("gnunet-publish-test-dst"); + buf = GNUNET_malloc (FILESIZE); + for (i = 0; i < FILESIZE; i++) + buf[i] = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 256); + GNUNET_assert (FILESIZE == + GNUNET_DISK_fn_write (fn1, buf, FILESIZE, + GNUNET_DISK_PERM_USER_READ | + GNUNET_DISK_PERM_USER_WRITE)); + GNUNET_free (buf); + + fn2 = GNUNET_DISK_mktemp ("gnunet-publish-test-dst"); + buf = GNUNET_malloc (FILESIZE); + for (i = 0; i < FILESIZE; i++) + buf[i] = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 256); + GNUNET_assert (FILESIZE == + GNUNET_DISK_fn_write (fn2, buf, FILESIZE, + GNUNET_DISK_PERM_USER_READ | + GNUNET_DISK_PERM_USER_WRITE)); + GNUNET_free (buf); + + meta = GNUNET_CONTAINER_meta_data_create (); + kuri = GNUNET_FS_uri_ksk_create_from_args (2, keywords); + bo.content_priority = 42; + bo.anonymity_level = 1; + bo.replication_level = 0; + bo.expiration_time = GNUNET_TIME_relative_to_absolute (LIFETIME); + fi1 = + GNUNET_FS_file_information_create_from_file (fs, "publish-context1", fn1, + kuri, meta, GNUNET_YES, &bo); + GNUNET_assert (NULL != fi1); + bo.anonymity_level = 2; + fi2 = + GNUNET_FS_file_information_create_from_file (fs, "publish-context2", fn2, + kuri, meta, GNUNET_YES, &bo); + GNUNET_assert (NULL != fi2); + bo.anonymity_level = 3; + fidir = + GNUNET_FS_file_information_create_empty_directory (fs, + "publish-context-dir", + kuri, meta, &bo, NULL); + GNUNET_assert (GNUNET_OK == GNUNET_FS_file_information_add (fidir, fi1)); + GNUNET_assert (GNUNET_OK == GNUNET_FS_file_information_add (fidir, fi2)); + GNUNET_FS_uri_destroy (kuri); + GNUNET_CONTAINER_meta_data_destroy (meta); + GNUNET_assert (NULL != fidir); + start = GNUNET_TIME_absolute_get (); + GNUNET_FS_publish_start (fs, fidir, NULL, NULL, NULL, + GNUNET_FS_PUBLISH_OPTION_NONE); + GNUNET_assert (publish != NULL); +} + + +int +main (int argc, char *argv[]) +{ + char *const argvx[] = { + "test-fs-publish-persistence", + "-c", + "test_fs_publish_data.conf", +#if VERBOSE + "-L", "DEBUG", +#endif + NULL + }; + struct GNUNET_GETOPT_CommandLineOption options[] = { + GNUNET_GETOPT_OPTION_END + }; + + GNUNET_log_setup ("test_fs_publish_persistence", +#if VERBOSE + "DEBUG", +#else + "WARNING", +#endif + NULL); + GNUNET_PROGRAM_run ((sizeof (argvx) / sizeof (char *)) - 1, argvx, + "test-fs-publish", "nohelp", options, &run, NULL); + stop_arm (&p1); + GNUNET_DISK_directory_remove ("/tmp/gnunet-test-fs-publish/"); + if (fn1 != NULL) + { + GNUNET_DISK_directory_remove (fn1); + GNUNET_free (fn1); + } + if (fn2 != NULL) + { + GNUNET_DISK_directory_remove (fn2); + GNUNET_free (fn2); + } + return err; +} + +/* end of test_fs_publish_persistence.c */ diff --git a/src/fs/test_fs_search.c b/src/fs/test_fs_search.c new file mode 100644 index 0000000..f6c8f00 --- /dev/null +++ b/src/fs/test_fs_search.c @@ -0,0 +1,280 @@ +/* + This file is part of GNUnet. + (C) 2004, 2005, 2006, 2008, 2009 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/test_fs_search.c + * @brief simple testcase for simple publish + search operation + * @author Christian Grothoff + */ + +#include "platform.h" +#include "gnunet_util_lib.h" +#include "gnunet_arm_service.h" +#include "gnunet_fs_service.h" + +#define VERBOSE GNUNET_NO + +#define START_ARM GNUNET_YES + +/** + * File-size we use for testing. + */ +#define FILESIZE 1024 + +/** + * How long until we give up on transmitting the message? + */ +#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 60) + +/** + * How long should our test-content live? + */ +#define LIFETIME GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15) + +struct PeerContext +{ + struct GNUNET_CONFIGURATION_Handle *cfg; + struct GNUNET_PeerIdentity id; +#if START_ARM + struct GNUNET_OS_Process *arm_proc; +#endif +}; + +static struct PeerContext p1; + +static struct GNUNET_TIME_Absolute start; + +static struct GNUNET_FS_Handle *fs; + +static struct GNUNET_FS_SearchContext *search; + +static struct GNUNET_FS_PublishContext *publish; + + +static void +abort_publish_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + GNUNET_FS_publish_stop (publish); + publish = NULL; +} + + +static void +abort_search_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + if (search != NULL) + GNUNET_FS_search_stop (search); + search = NULL; +} + + +static void * +progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *event) +{ + const char *keywords[] = { + "down_foo" + }; + struct GNUNET_FS_Uri *kuri; + + switch (event->status) + { + case GNUNET_FS_STATUS_PUBLISH_PROGRESS: +#if VERBOSE + printf ("Publish is progressing (%llu/%llu at level %u off %llu)...\n", + (unsigned long long) event->value.publish.completed, + (unsigned long long) event->value.publish.size, + event->value.publish.specifics.progress.depth, + (unsigned long long) event->value.publish.specifics. + progress.offset); +#endif + break; + case GNUNET_FS_STATUS_PUBLISH_COMPLETED: + kuri = GNUNET_FS_uri_ksk_create_from_args (1, keywords); + start = GNUNET_TIME_absolute_get (); + search = + GNUNET_FS_search_start (fs, kuri, 1, GNUNET_FS_SEARCH_OPTION_NONE, + "search"); + GNUNET_FS_uri_destroy (kuri); + GNUNET_assert (search != NULL); + break; + case GNUNET_FS_STATUS_SEARCH_RESULT: +#if VERBOSE + printf ("Search complete.\n"); +#endif + GNUNET_SCHEDULER_add_continuation (&abort_search_task, NULL, + GNUNET_SCHEDULER_REASON_PREREQ_DONE); + break; + case GNUNET_FS_STATUS_PUBLISH_ERROR: + FPRINTF (stderr, "Error publishing file: %s\n", + event->value.publish.specifics.error.message); + GNUNET_break (0); + GNUNET_SCHEDULER_add_continuation (&abort_publish_task, NULL, + GNUNET_SCHEDULER_REASON_PREREQ_DONE); + break; + case GNUNET_FS_STATUS_SEARCH_ERROR: + FPRINTF (stderr, "Error searching file: %s\n", + event->value.search.specifics.error.message); + GNUNET_SCHEDULER_add_continuation (&abort_search_task, NULL, + GNUNET_SCHEDULER_REASON_PREREQ_DONE); + break; + case GNUNET_FS_STATUS_PUBLISH_START: + GNUNET_assert (0 == strcmp ("publish-context", event->value.publish.cctx)); + GNUNET_assert (NULL == event->value.publish.pctx); + GNUNET_assert (FILESIZE == event->value.publish.size); + GNUNET_assert (0 == event->value.publish.completed); + GNUNET_assert (1 == event->value.publish.anonymity); + break; + case GNUNET_FS_STATUS_PUBLISH_STOPPED: + GNUNET_assert (publish == event->value.publish.pc); + GNUNET_assert (FILESIZE == event->value.publish.size); + GNUNET_assert (1 == event->value.publish.anonymity); + GNUNET_FS_stop (fs); + fs = NULL; + break; + case GNUNET_FS_STATUS_SEARCH_START: + GNUNET_assert (search == NULL); + GNUNET_assert (0 == strcmp ("search", event->value.search.cctx)); + GNUNET_assert (1 == event->value.search.anonymity); + break; + case GNUNET_FS_STATUS_SEARCH_RESULT_STOPPED: + break; + case GNUNET_FS_STATUS_SEARCH_STOPPED: + GNUNET_assert (search == event->value.search.sc); + GNUNET_SCHEDULER_add_continuation (&abort_publish_task, NULL, + GNUNET_SCHEDULER_REASON_PREREQ_DONE); + break; + default: + FPRINTF (stderr, "Unexpected event: %d\n", event->status); + break; + } + return NULL; +} + + +static void +setup_peer (struct PeerContext *p, const char *cfgname) +{ + p->cfg = GNUNET_CONFIGURATION_create (); +#if START_ARM + p->arm_proc = + GNUNET_OS_start_process (GNUNET_YES, NULL, NULL, "gnunet-service-arm", + "gnunet-service-arm", +#if VERBOSE + "-L", "DEBUG", +#endif + "-c", cfgname, NULL); +#endif + GNUNET_assert (GNUNET_OK == GNUNET_CONFIGURATION_load (p->cfg, cfgname)); +} + + +static void +stop_arm (struct PeerContext *p) +{ +#if START_ARM + if (NULL != p->arm_proc) + { + if (0 != GNUNET_OS_process_kill (p->arm_proc, SIGTERM)) + GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING, "kill"); + if (GNUNET_OS_process_wait (p->arm_proc) != GNUNET_OK) + GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING, "waitpid"); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "ARM process %u stopped\n", + GNUNET_OS_process_get_pid (p->arm_proc)); + GNUNET_OS_process_close (p->arm_proc); + p->arm_proc = NULL; + } +#endif + GNUNET_CONFIGURATION_destroy (p->cfg); +} + + +static void +run (void *cls, char *const *args, const char *cfgfile, + const struct GNUNET_CONFIGURATION_Handle *cfg) +{ + const char *keywords[] = { + "down_foo", + "down_bar" + }; + char *buf; + struct GNUNET_CONTAINER_MetaData *meta; + struct GNUNET_FS_Uri *kuri; + struct GNUNET_FS_BlockOptions bo; + struct GNUNET_FS_FileInformation *fi; + size_t i; + + setup_peer (&p1, "test_fs_search_data.conf"); + fs = GNUNET_FS_start (cfg, "test-fs-search", &progress_cb, NULL, + GNUNET_FS_FLAGS_NONE, GNUNET_FS_OPTIONS_END); + GNUNET_assert (NULL != fs); + buf = GNUNET_malloc (FILESIZE); + for (i = 0; i < FILESIZE; i++) + buf[i] = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 256); + meta = GNUNET_CONTAINER_meta_data_create (); + kuri = GNUNET_FS_uri_ksk_create_from_args (2, keywords); + bo.content_priority = 42; + bo.anonymity_level = 1; + bo.replication_level = 0; + bo.expiration_time = GNUNET_TIME_relative_to_absolute (LIFETIME); + fi = GNUNET_FS_file_information_create_from_data (fs, "publish-context", + FILESIZE, buf, kuri, meta, + GNUNET_NO, &bo); + GNUNET_FS_uri_destroy (kuri); + GNUNET_CONTAINER_meta_data_destroy (meta); + GNUNET_assert (NULL != fi); + start = GNUNET_TIME_absolute_get (); + publish = + GNUNET_FS_publish_start (fs, fi, NULL, NULL, NULL, + GNUNET_FS_PUBLISH_OPTION_NONE); + GNUNET_assert (publish != NULL); +} + + +int +main (int argc, char *argv[]) +{ + char *const argvx[] = { + "test-fs-search", + "-c", + "test_fs_search_data.conf", +#if VERBOSE + "-L", "DEBUG", +#endif + NULL + }; + struct GNUNET_GETOPT_CommandLineOption options[] = { + GNUNET_GETOPT_OPTION_END + }; + + GNUNET_log_setup ("test_fs_search", +#if VERBOSE + "DEBUG", +#else + "WARNING", +#endif + NULL); + GNUNET_PROGRAM_run ((sizeof (argvx) / sizeof (char *)) - 1, argvx, + "test-fs-search", "nohelp", options, &run, NULL); + stop_arm (&p1); + GNUNET_DISK_directory_remove ("/tmp/gnunet-test-fs-search/"); + return 0; +} + +/* end of test_fs_search.c */ diff --git a/src/fs/test_fs_search_data.conf b/src/fs/test_fs_search_data.conf new file mode 100644 index 0000000..ea5e4c5 --- /dev/null +++ b/src/fs/test_fs_search_data.conf @@ -0,0 +1,8 @@ +@INLINE@ test_fs_defaults.conf +[PATHS] +SERVICEHOME = /tmp/gnunet-test-fs-search/ +DEFAULTCONFIG = test_fs_search_data.conf + +[transport] +PLUGINS = + diff --git a/src/fs/test_fs_search_persistence.c b/src/fs/test_fs_search_persistence.c new file mode 100644 index 0000000..38f88a8 --- /dev/null +++ b/src/fs/test_fs_search_persistence.c @@ -0,0 +1,345 @@ +/* + This file is part of GNUnet. + (C) 2004, 2005, 2006, 2008, 2009, 2010 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/test_fs_search_persistence.c + * @brief simple testcase for persistence of search operation + * @author Christian Grothoff + */ + +#include "platform.h" +#include "gnunet_util_lib.h" +#include "gnunet_arm_service.h" +#include "gnunet_fs_service.h" + +#define VERBOSE GNUNET_NO + +#define START_ARM GNUNET_YES + +/** + * File-size we use for testing. + */ +#define FILESIZE 1024 + +/** + * How long until we give up on transmitting the message? + */ +#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 60) + +/** + * How long should our test-content live? + */ +#define LIFETIME GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15) + +struct PeerContext +{ + struct GNUNET_CONFIGURATION_Handle *cfg; + struct GNUNET_PeerIdentity id; +#if START_ARM + struct GNUNET_OS_Process *arm_proc; +#endif +}; + +static struct PeerContext p1; + +static struct GNUNET_TIME_Absolute start; + +static struct GNUNET_FS_Handle *fs; + +static struct GNUNET_FS_SearchContext *search; + +static struct GNUNET_FS_PublishContext *publish; + +static const struct GNUNET_CONFIGURATION_Handle *cfg; + +static void +abort_publish_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + GNUNET_FS_publish_stop (publish); + publish = NULL; +} + + +static void +abort_search_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + if (search != NULL) + GNUNET_FS_search_stop (search); + search = NULL; +} + + +static void * +progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *event); + + +static void +restart_fs_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + GNUNET_FS_stop (fs); + fs = GNUNET_FS_start (cfg, "test-fs-search-persistence", &progress_cb, NULL, + GNUNET_FS_FLAGS_PERSISTENCE, GNUNET_FS_OPTIONS_END); +} + + + + +/** + * Consider scheduling the restart-task. + * Only runs the restart task once per event + * category. + * + * @param ev type of the event to consider + */ +static void +consider_restart (int ev) +{ + static int prev[32]; + static int off; + int i; + + for (i = 0; i < off; i++) + if (prev[i] == ev) + return; + prev[off++] = ev; + GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_URGENT, + &restart_fs_task, NULL); +} + + +static void * +progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *event) +{ + const char *keywords[] = { + "down_foo" + }; + struct GNUNET_FS_Uri *kuri; + + switch (event->status) + { + case GNUNET_FS_STATUS_PUBLISH_PROGRESS: +#if VERBOSE + printf ("Publish is progressing (%llu/%llu at level %u off %llu)...\n", + (unsigned long long) event->value.publish.completed, + (unsigned long long) event->value.publish.size, + event->value.publish.specifics.progress.depth, + (unsigned long long) event->value.publish.specifics. + progress.offset); +#endif + break; + case GNUNET_FS_STATUS_PUBLISH_COMPLETED: + kuri = GNUNET_FS_uri_ksk_create_from_args (1, keywords); + start = GNUNET_TIME_absolute_get (); + GNUNET_FS_search_start (fs, kuri, 1, GNUNET_FS_SEARCH_OPTION_NONE, + "search"); + GNUNET_FS_uri_destroy (kuri); + GNUNET_assert (search != NULL); + break; + case GNUNET_FS_STATUS_PUBLISH_SUSPEND: + if (event->value.publish.pc == publish) + publish = NULL; + break; + case GNUNET_FS_STATUS_PUBLISH_RESUME: + if (NULL == publish) + publish = event->value.publish.pc; + break; + case GNUNET_FS_STATUS_SEARCH_RESULT: + /* FIXME: consider_restart (event->status); cannot be tested with + * search result since we exit here after the first one... */ +#if VERBOSE + printf ("Search complete.\n"); +#endif + GNUNET_SCHEDULER_add_continuation (&abort_search_task, NULL, + GNUNET_SCHEDULER_REASON_PREREQ_DONE); + break; + case GNUNET_FS_STATUS_PUBLISH_ERROR: + FPRINTF (stderr, "Error publishing file: %s\n", + event->value.publish.specifics.error.message); + GNUNET_break (0); + GNUNET_SCHEDULER_add_continuation (&abort_publish_task, NULL, + GNUNET_SCHEDULER_REASON_PREREQ_DONE); + break; + case GNUNET_FS_STATUS_SEARCH_ERROR: + FPRINTF (stderr, "Error searching file: %s\n", + event->value.search.specifics.error.message); + GNUNET_SCHEDULER_add_continuation (&abort_search_task, NULL, + GNUNET_SCHEDULER_REASON_PREREQ_DONE); + break; + case GNUNET_FS_STATUS_SEARCH_SUSPEND: + if (event->value.search.sc == search) + search = NULL; + break; + case GNUNET_FS_STATUS_SEARCH_RESUME: + if (NULL == search) + { + search = event->value.search.sc; + return "search"; + } + break; + case GNUNET_FS_STATUS_PUBLISH_START: + GNUNET_assert (0 == strcmp ("publish-context", event->value.publish.cctx)); + GNUNET_assert (NULL == event->value.publish.pctx); + GNUNET_assert (FILESIZE == event->value.publish.size); + GNUNET_assert (0 == event->value.publish.completed); + GNUNET_assert (1 == event->value.publish.anonymity); + break; + case GNUNET_FS_STATUS_PUBLISH_STOPPED: + GNUNET_assert (publish == event->value.publish.pc); + GNUNET_assert (FILESIZE == event->value.publish.size); + GNUNET_assert (1 == event->value.publish.anonymity); + GNUNET_FS_stop (fs); + fs = NULL; + break; + case GNUNET_FS_STATUS_SEARCH_START: + consider_restart (event->status); + GNUNET_assert (search == NULL); + search = event->value.search.sc; + GNUNET_assert (0 == strcmp ("search", event->value.search.cctx)); + GNUNET_assert (1 == event->value.search.anonymity); + break; + case GNUNET_FS_STATUS_SEARCH_RESULT_STOPPED: + break; + case GNUNET_FS_STATUS_SEARCH_STOPPED: + GNUNET_assert (search == event->value.search.sc); + GNUNET_SCHEDULER_add_continuation (&abort_publish_task, NULL, + GNUNET_SCHEDULER_REASON_PREREQ_DONE); + search = NULL; + break; + default: + FPRINTF (stderr, "Unexpected event: %d\n", event->status); + break; + } + return NULL; +} + + +static void +setup_peer (struct PeerContext *p, const char *cfgname) +{ + p->cfg = GNUNET_CONFIGURATION_create (); +#if START_ARM + p->arm_proc = + GNUNET_OS_start_process (GNUNET_YES, NULL, NULL, "gnunet-service-arm", + "gnunet-service-arm", +#if VERBOSE + "-L", "DEBUG", +#endif + "-c", cfgname, NULL); +#endif + GNUNET_assert (GNUNET_OK == GNUNET_CONFIGURATION_load (p->cfg, cfgname)); +} + + +static void +stop_arm (struct PeerContext *p) +{ +#if START_ARM + if (NULL != p->arm_proc) + { + if (0 != GNUNET_OS_process_kill (p->arm_proc, SIGTERM)) + GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING, "kill"); + if (GNUNET_OS_process_wait (p->arm_proc) != GNUNET_OK) + GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING, "waitpid"); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "ARM process %u stopped\n", + GNUNET_OS_process_get_pid (p->arm_proc)); + GNUNET_OS_process_close (p->arm_proc); + p->arm_proc = NULL; + } +#endif + GNUNET_CONFIGURATION_destroy (p->cfg); +} + + +static void +run (void *cls, char *const *args, const char *cfgfile, + const struct GNUNET_CONFIGURATION_Handle *c) +{ + const char *keywords[] = { + "down_foo", + "down_bar" + }; + char *buf; + struct GNUNET_CONTAINER_MetaData *meta; + struct GNUNET_FS_Uri *kuri; + struct GNUNET_FS_FileInformation *fi; + size_t i; + struct GNUNET_FS_BlockOptions bo; + + cfg = c; + setup_peer (&p1, "test_fs_search_data.conf"); + fs = GNUNET_FS_start (cfg, "test-fs-search-persistence", &progress_cb, NULL, + GNUNET_FS_FLAGS_PERSISTENCE, GNUNET_FS_OPTIONS_END); + GNUNET_assert (NULL != fs); + buf = GNUNET_malloc (FILESIZE); + for (i = 0; i < FILESIZE; i++) + buf[i] = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 256); + meta = GNUNET_CONTAINER_meta_data_create (); + kuri = GNUNET_FS_uri_ksk_create_from_args (2, keywords); + bo.content_priority = 42; + bo.anonymity_level = 1; + bo.replication_level = 0; + bo.expiration_time = GNUNET_TIME_relative_to_absolute (LIFETIME); + fi = GNUNET_FS_file_information_create_from_data (fs, "publish-context", + FILESIZE, buf, kuri, meta, + GNUNET_NO, &bo); + GNUNET_FS_uri_destroy (kuri); + GNUNET_CONTAINER_meta_data_destroy (meta); + GNUNET_assert (NULL != fi); + start = GNUNET_TIME_absolute_get (); + publish = + GNUNET_FS_publish_start (fs, fi, NULL, NULL, NULL, + GNUNET_FS_PUBLISH_OPTION_NONE); + GNUNET_assert (publish != NULL); +} + + +int +main (int argc, char *argv[]) +{ + char *const argvx[] = { + "test-fs-search-persistence", + "-c", + "test_fs_search_data.conf", +#if VERBOSE + "-L", "DEBUG", +#endif + NULL + }; + struct GNUNET_GETOPT_CommandLineOption options[] = { + GNUNET_GETOPT_OPTION_END + }; + + GNUNET_DISK_directory_remove ("/tmp/gnunet-test-fs-search/"); + GNUNET_log_setup ("test_fs_search_persistence", +#if VERBOSE + "DEBUG", +#else + "WARNING", +#endif + NULL); + GNUNET_PROGRAM_run ((sizeof (argvx) / sizeof (char *)) - 1, argvx, + "test-fs-search-persistence", "nohelp", options, &run, + NULL); + stop_arm (&p1); + GNUNET_DISK_directory_remove ("/tmp/gnunet-test-fs-search/"); + return 0; +} + +/* end of test_fs_search_persistence.c */ diff --git a/src/fs/test_fs_start_stop.c b/src/fs/test_fs_start_stop.c new file mode 100644 index 0000000..0ef0723 --- /dev/null +++ b/src/fs/test_fs_start_stop.c @@ -0,0 +1,135 @@ +/* + This file is part of GNUnet. + (C) 2004, 2005, 2006, 2009 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/test_fs_start_stop.c + * @brief testcase for fs.c (start-stop only) + * @author Christian Grothoff + */ + +#include "platform.h" +#include "gnunet_util_lib.h" +#include "gnunet_arm_service.h" +#include "gnunet_fs_service.h" + +#define VERBOSE GNUNET_NO + +#define START_ARM GNUNET_YES + +static struct PeerContext p1; + +struct PeerContext +{ + struct GNUNET_CONFIGURATION_Handle *cfg; +#if START_ARM + struct GNUNET_OS_Process *arm_proc; +#endif +}; + + +static void * +progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *event) +{ + return NULL; +} + + +static void +setup_peer (struct PeerContext *p, const char *cfgname) +{ + p->cfg = GNUNET_CONFIGURATION_create (); +#if START_ARM + p->arm_proc = + GNUNET_OS_start_process (GNUNET_YES, NULL, NULL, "gnunet-service-arm", + "gnunet-service-arm", +#if VERBOSE + "-L", "DEBUG", +#endif + "-c", cfgname, NULL); +#endif + GNUNET_assert (GNUNET_OK == GNUNET_CONFIGURATION_load (p->cfg, cfgname)); +} + + +static void +stop_arm (struct PeerContext *p) +{ +#if START_ARM + if (NULL != p->arm_proc) + { + if (0 != GNUNET_OS_process_kill (p->arm_proc, SIGTERM)) + GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING, "kill"); + if (GNUNET_OS_process_wait (p->arm_proc) != GNUNET_OK) + GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING, "waitpid"); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "ARM process %u stopped\n", + GNUNET_OS_process_get_pid (p->arm_proc)); + GNUNET_OS_process_close (p->arm_proc); + p->arm_proc = NULL; + } +#endif + GNUNET_CONFIGURATION_destroy (p->cfg); +} + + +static void +run (void *cls, char *const *args, const char *cfgfile, + const struct GNUNET_CONFIGURATION_Handle *cfg) +{ + struct GNUNET_FS_Handle *fs; + + setup_peer (&p1, "test_fs_data.conf"); + fs = GNUNET_FS_start (cfg, "test-fs-start-stop", &progress_cb, NULL, + GNUNET_FS_FLAGS_NONE, GNUNET_FS_OPTIONS_END); + GNUNET_assert (NULL != fs); + GNUNET_FS_stop (fs); +} + + +int +main (int argc, char *argv[]) +{ + char *const argvx[] = { + "test-fs-start-stop", + "-c", + "test_fs_data.conf", +#if VERBOSE + "-L", "DEBUG", +#endif + NULL + }; + struct GNUNET_GETOPT_CommandLineOption options[] = { + GNUNET_GETOPT_OPTION_END + }; + + GNUNET_log_setup ("test_fs_start_stop", +#if VERBOSE + "DEBUG", +#else + "WARNING", +#endif + NULL); + GNUNET_PROGRAM_run ((sizeof (argvx) / sizeof (char *)) - 1, argvx, + "test-fs-start-stop", "nohelp", options, &run, NULL); + stop_arm (&p1); + GNUNET_DISK_directory_remove ("/tmp/gnunet-test-fs/"); + return 0; +} + +/* end of test_fs_start_stop.c */ diff --git a/src/fs/test_fs_test_lib.c b/src/fs/test_fs_test_lib.c new file mode 100644 index 0000000..589abb3 --- /dev/null +++ b/src/fs/test_fs_test_lib.c @@ -0,0 +1,164 @@ +/* + This file is part of GNUnet. + (C) 2010 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/test_fs_test_lib.c + * @brief test fs test library + * @author Christian Grothoff + */ +#include "platform.h" +#include "fs_test_lib.h" + +#define VERBOSE GNUNET_NO + +/** + * File-size we use for testing. + */ +#define FILESIZE (1024 * 1024 * 2) + +/** + * How long until we give up on transmitting the message? + */ +#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 300) + +#define NUM_DAEMONS 2 + +#define SEED 42 + +static struct GNUNET_FS_TestDaemon *daemons[NUM_DAEMONS]; + +static struct GNUNET_FS_TEST_ConnectContext *cc; + +static int ret; + +static void +do_stop (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + if (NULL != cc) + { + GNUNET_FS_TEST_daemons_connect_cancel (cc); + cc = NULL; + } + if (0 == (tc->reason & GNUNET_SCHEDULER_REASON_PREREQ_DONE)) + { + GNUNET_break (0); + ret = 1; + } + else + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Finished download, shutting down\n", + (unsigned long long) FILESIZE); + } + GNUNET_FS_TEST_daemons_stop (NUM_DAEMONS, daemons); +} + + +static void +do_download (void *cls, const struct GNUNET_FS_Uri *uri) +{ + if (NULL == uri) + { + GNUNET_break (0); + GNUNET_SCHEDULER_add_now (&do_stop, NULL); + ret = 1; + return; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Downloading %llu bytes\n", + (unsigned long long) FILESIZE); + GNUNET_FS_TEST_download (daemons[0], TIMEOUT, 1, SEED, uri, VERBOSE, &do_stop, + NULL); +} + + +static void +do_publish (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + cc = NULL; + if (0 == (tc->reason & GNUNET_SCHEDULER_REASON_PREREQ_DONE)) + { + GNUNET_break (0); + ret = 1; + GNUNET_SCHEDULER_add_now (&do_stop, NULL); + return; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Publishing %llu bytes\n", + (unsigned long long) FILESIZE); + GNUNET_FS_TEST_publish (daemons[0], TIMEOUT, 1, GNUNET_NO, FILESIZE, SEED, + VERBOSE, &do_download, NULL); +} + + +static void +do_connect (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + if (0 == (tc->reason & GNUNET_SCHEDULER_REASON_PREREQ_DONE)) + { + GNUNET_break (0); + ret = 1; + GNUNET_SCHEDULER_add_now (&do_stop, NULL); + return; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Daemons started, will now try to connect them\n"); + cc = GNUNET_FS_TEST_daemons_connect (daemons[0], daemons[1], TIMEOUT, + &do_publish, NULL); +} + + +static void +run (void *cls, char *const *args, const char *cfgfile, + const struct GNUNET_CONFIGURATION_Handle *cfg) +{ + GNUNET_FS_TEST_daemons_start ("fs_test_lib_data.conf", TIMEOUT, NUM_DAEMONS, + daemons, &do_connect, NULL); +} + + +int +main (int argc, char *argv[]) +{ + char *const argvx[] = { + "test-fs-test-lib", + "-c", + "fs_test_lib_data.conf", +#if VERBOSE + "-L", "DEBUG", +#endif + NULL + }; + struct GNUNET_GETOPT_CommandLineOption options[] = { + GNUNET_GETOPT_OPTION_END + }; + + GNUNET_DISK_directory_remove ("/tmp/gnunet-test-fs-lib/"); + GNUNET_log_setup ("test_fs_test_lib", +#if VERBOSE + "DEBUG", +#else + "WARNING", +#endif + NULL); + GNUNET_PROGRAM_run ((sizeof (argvx) / sizeof (char *)) - 1, argvx, + "test-fs-test-lib", "nohelp", options, &run, NULL); + GNUNET_DISK_directory_remove ("/tmp/gnunet-test-fs-lib/"); + return ret; +} + +/* end of test_fs_test_lib.c */ diff --git a/src/fs/test_fs_unindex.c b/src/fs/test_fs_unindex.c new file mode 100644 index 0000000..a8b68a3 --- /dev/null +++ b/src/fs/test_fs_unindex.c @@ -0,0 +1,304 @@ +/* + This file is part of GNUnet. + (C) 2004, 2005, 2006, 2008, 2009 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/test_fs_unindex.c + * @brief simple testcase for simple publish + unindex operation + * @author Christian Grothoff + */ + +#include "platform.h" +#include "gnunet_util_lib.h" +#include "gnunet_arm_service.h" +#include "gnunet_fs_service.h" + +#define VERBOSE GNUNET_NO + +#define START_ARM GNUNET_YES + +/** + * File-size we use for testing. + */ +#define FILESIZE (1024 * 1024 * 2) + +/** + * How long until we give up on transmitting the message? + */ +#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 60) + +/** + * How long should our test-content live? + */ +#define LIFETIME GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15) + +struct PeerContext +{ + struct GNUNET_CONFIGURATION_Handle *cfg; +#if START_ARM + struct GNUNET_OS_Process *arm_proc; +#endif +}; + +static struct PeerContext p1; + +static struct GNUNET_TIME_Absolute start; + +static struct GNUNET_FS_Handle *fs; + +static struct GNUNET_FS_UnindexContext *unindex; + +static struct GNUNET_FS_PublishContext *publish; + +static char *fn; + + +static void +abort_publish_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + GNUNET_FS_publish_stop (publish); + publish = NULL; +} + + +static void +abort_unindex_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + GNUNET_FS_unindex_stop (unindex); + unindex = NULL; + GNUNET_DISK_directory_remove (fn); + GNUNET_free (fn); + fn = NULL; +} + + +static void * +progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *event) +{ + + switch (event->status) + { + case GNUNET_FS_STATUS_PUBLISH_PROGRESS: +#if VERBOSE + printf ("Publish is progressing (%llu/%llu at level %u off %llu)...\n", + (unsigned long long) event->value.publish.completed, + (unsigned long long) event->value.publish.size, + event->value.publish.specifics.progress.depth, + (unsigned long long) event->value.publish.specifics. + progress.offset); +#endif + break; + case GNUNET_FS_STATUS_PUBLISH_COMPLETED: + printf ("Publishing complete, %llu kbps.\n", + (unsigned long long) (FILESIZE * 1000 / + (1 + + GNUNET_TIME_absolute_get_duration + (start).rel_value) / 1024)); + start = GNUNET_TIME_absolute_get (); + unindex = GNUNET_FS_unindex_start (fs, fn, "unindex"); + GNUNET_assert (unindex != NULL); + break; + case GNUNET_FS_STATUS_UNINDEX_COMPLETED: + printf ("Unindex complete, %llu kbps.\n", + (unsigned long long) (FILESIZE * 1000 / + (1 + + GNUNET_TIME_absolute_get_duration + (start).rel_value) / 1024)); + GNUNET_SCHEDULER_add_continuation (&abort_unindex_task, NULL, + GNUNET_SCHEDULER_REASON_PREREQ_DONE); + break; + case GNUNET_FS_STATUS_UNINDEX_PROGRESS: + GNUNET_assert (unindex == event->value.unindex.uc); +#if VERBOSE + printf ("Unindex is progressing (%llu/%llu at level %u off %llu)...\n", + (unsigned long long) event->value.unindex.completed, + (unsigned long long) event->value.unindex.size, + event->value.unindex.specifics.progress.depth, + (unsigned long long) event->value.unindex.specifics. + progress.offset); +#endif + break; + case GNUNET_FS_STATUS_PUBLISH_ERROR: + FPRINTF (stderr, "Error publishing file: %s\n", + event->value.publish.specifics.error.message); + GNUNET_break (0); + GNUNET_SCHEDULER_add_continuation (&abort_publish_task, NULL, + GNUNET_SCHEDULER_REASON_PREREQ_DONE); + break; + case GNUNET_FS_STATUS_UNINDEX_ERROR: + FPRINTF (stderr, "Error unindexing file: %s\n", + event->value.unindex.specifics.error.message); + GNUNET_SCHEDULER_add_continuation (&abort_unindex_task, NULL, + GNUNET_SCHEDULER_REASON_PREREQ_DONE); + break; + case GNUNET_FS_STATUS_PUBLISH_START: + GNUNET_assert (0 == strcmp ("publish-context", event->value.publish.cctx)); + GNUNET_assert (NULL == event->value.publish.pctx); + GNUNET_assert (FILESIZE == event->value.publish.size); + GNUNET_assert (0 == event->value.publish.completed); + GNUNET_assert (1 == event->value.publish.anonymity); + break; + case GNUNET_FS_STATUS_PUBLISH_STOPPED: + GNUNET_assert (publish == event->value.publish.pc); + GNUNET_assert (FILESIZE == event->value.publish.size); + GNUNET_assert (1 == event->value.publish.anonymity); + GNUNET_FS_stop (fs); + fs = NULL; + break; + case GNUNET_FS_STATUS_UNINDEX_START: + GNUNET_assert (unindex == NULL); + GNUNET_assert (0 == strcmp ("unindex", event->value.unindex.cctx)); + GNUNET_assert (0 == strcmp (fn, event->value.unindex.filename)); + GNUNET_assert (FILESIZE == event->value.unindex.size); + GNUNET_assert (0 == event->value.unindex.completed); + break; + case GNUNET_FS_STATUS_UNINDEX_STOPPED: + GNUNET_assert (unindex == event->value.unindex.uc); + GNUNET_SCHEDULER_add_continuation (&abort_publish_task, NULL, + GNUNET_SCHEDULER_REASON_PREREQ_DONE); + break; + default: + printf ("Unexpected event: %d\n", event->status); + break; + } + return NULL; +} + + +static void +setup_peer (struct PeerContext *p, const char *cfgname) +{ + p->cfg = GNUNET_CONFIGURATION_create (); +#if START_ARM + p->arm_proc = + GNUNET_OS_start_process (GNUNET_YES, NULL, NULL, "gnunet-service-arm", + "gnunet-service-arm", +#if VERBOSE + "-L", "DEBUG", +#endif + "-c", cfgname, NULL); +#endif + GNUNET_assert (GNUNET_OK == GNUNET_CONFIGURATION_load (p->cfg, cfgname)); +} + + +static void +stop_arm (struct PeerContext *p) +{ +#if START_ARM + if (NULL != p->arm_proc) + { + if (0 != GNUNET_OS_process_kill (p->arm_proc, SIGTERM)) + GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING, "kill"); + if (GNUNET_OS_process_wait (p->arm_proc) != GNUNET_OK) + GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING, "waitpid"); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "ARM process %u stopped\n", + GNUNET_OS_process_get_pid (p->arm_proc)); + GNUNET_OS_process_close (p->arm_proc); + p->arm_proc = NULL; + } +#endif + GNUNET_CONFIGURATION_destroy (p->cfg); +} + + +static void +run (void *cls, char *const *args, const char *cfgfile, + const struct GNUNET_CONFIGURATION_Handle *cfg) +{ + const char *keywords[] = { + "down_foo", + "down_bar", + }; + char *buf; + struct GNUNET_CONTAINER_MetaData *meta; + struct GNUNET_FS_Uri *kuri; + struct GNUNET_FS_FileInformation *fi; + size_t i; + struct GNUNET_FS_BlockOptions bo; + + setup_peer (&p1, "test_fs_unindex_data.conf"); + fn = GNUNET_DISK_mktemp ("gnunet-unindex-test-dst"); + fs = GNUNET_FS_start (cfg, "test-fs-unindex", &progress_cb, NULL, + GNUNET_FS_FLAGS_NONE, GNUNET_FS_OPTIONS_END); + GNUNET_assert (NULL != fs); + buf = GNUNET_malloc (FILESIZE); + for (i = 0; i < FILESIZE; i++) + buf[i] = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 256); + GNUNET_assert (FILESIZE == + GNUNET_DISK_fn_write (fn, buf, FILESIZE, + GNUNET_DISK_PERM_USER_READ | + GNUNET_DISK_PERM_USER_WRITE)); + GNUNET_free (buf); + meta = GNUNET_CONTAINER_meta_data_create (); + kuri = GNUNET_FS_uri_ksk_create_from_args (2, keywords); + bo.content_priority = 42; + bo.anonymity_level = 1; + bo.replication_level = 0; + bo.expiration_time = GNUNET_TIME_relative_to_absolute (LIFETIME); + fi = GNUNET_FS_file_information_create_from_file (fs, "publish-context", fn, + kuri, meta, GNUNET_YES, + &bo); + GNUNET_FS_uri_destroy (kuri); + GNUNET_CONTAINER_meta_data_destroy (meta); + GNUNET_assert (NULL != fi); + start = GNUNET_TIME_absolute_get (); + publish = + GNUNET_FS_publish_start (fs, fi, NULL, NULL, NULL, + GNUNET_FS_PUBLISH_OPTION_NONE); + GNUNET_assert (publish != NULL); +} + + +int +main (int argc, char *argv[]) +{ + char *const argvx[] = { + "test-fs-unindex", + "-c", + "test_fs_unindex_data.conf", +#if VERBOSE + "-L", "DEBUG", +#endif + NULL + }; + struct GNUNET_GETOPT_CommandLineOption options[] = { + GNUNET_GETOPT_OPTION_END + }; + + GNUNET_log_setup ("test_fs_unindex", +#if VERBOSE + "DEBUG", +#else + "WARNING", +#endif + NULL); + GNUNET_PROGRAM_run ((sizeof (argvx) / sizeof (char *)) - 1, argvx, + "test-fs-unindex", "nohelp", options, &run, NULL); + stop_arm (&p1); + GNUNET_DISK_directory_remove ("/tmp/gnunet-test-fs-unindex/"); + if (NULL != fn) + { + GNUNET_DISK_directory_remove (fn); + GNUNET_free (fn); + } + return 0; +} + +/* end of test_fs_unindex.c */ diff --git a/src/fs/test_fs_unindex_data.conf b/src/fs/test_fs_unindex_data.conf new file mode 100644 index 0000000..977d6e7 --- /dev/null +++ b/src/fs/test_fs_unindex_data.conf @@ -0,0 +1,8 @@ +@INLINE@ test_fs_defaults.conf +[PATHS] +SERVICEHOME = /tmp/gnunet-test-fs-unindex/ +DEFAULTCONFIG = test_fs_unindex_data.conf + +[transport] +PLUGINS = + diff --git a/src/fs/test_fs_unindex_persistence.c b/src/fs/test_fs_unindex_persistence.c new file mode 100644 index 0000000..575e171 --- /dev/null +++ b/src/fs/test_fs_unindex_persistence.c @@ -0,0 +1,367 @@ +/* + This file is part of GNUnet. + (C) 2004, 2005, 2006, 2008, 2009, 2010 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/test_fs_unindex_persistence.c + * @brief simple testcase for simple publish + unindex operation + * @author Christian Grothoff + */ +#include "platform.h" +#include "gnunet_util_lib.h" +#include "gnunet_arm_service.h" +#include "gnunet_fs_service.h" + +#define VERBOSE GNUNET_NO + +#define START_ARM GNUNET_YES + +/** + * File-size we use for testing. + */ +#define FILESIZE (1024 * 1024 * 2) + +/** + * How long until we give up on transmitting the message? + */ +#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 60) + +/** + * How long should our test-content live? + */ +#define LIFETIME GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15) + +struct PeerContext +{ + struct GNUNET_CONFIGURATION_Handle *cfg; +#if START_ARM + struct GNUNET_OS_Process *arm_proc; +#endif +}; + +static struct PeerContext p1; + +static struct GNUNET_TIME_Absolute start; + +static struct GNUNET_FS_Handle *fs; + +static struct GNUNET_FS_UnindexContext *unindex; + +static struct GNUNET_FS_PublishContext *publish; + +static char *fn; + +static const struct GNUNET_CONFIGURATION_Handle *cfg; + +static void +abort_publish_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + GNUNET_FS_publish_stop (publish); + publish = NULL; +} + + +static void +abort_unindex_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + if (unindex != NULL) + { + GNUNET_FS_unindex_stop (unindex); + unindex = NULL; + } + if (fn != NULL) + { + GNUNET_DISK_directory_remove (fn); + GNUNET_free (fn); + fn = NULL; + } +} + + +static void * +progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *event); + + +static void +restart_fs_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + GNUNET_FS_stop (fs); + fs = GNUNET_FS_start (cfg, "test-fs-unindex-persistence", &progress_cb, NULL, + GNUNET_FS_FLAGS_PERSISTENCE, GNUNET_FS_OPTIONS_END); +} + + +/** + * Consider scheduling the restart-task. + * Only runs the restart task once per event + * category. + * + * @param ev type of the event to consider + */ +static void +consider_restart (int ev) +{ + static int prev[32]; + static int off; + int i; + + for (i = 0; i < off; i++) + if (prev[i] == ev) + return; + prev[off++] = ev; + GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_URGENT, + &restart_fs_task, NULL); +} + + +static void * +progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *event) +{ + switch (event->status) + { + case GNUNET_FS_STATUS_PUBLISH_PROGRESS: +#if VERBOSE + printf ("Publish is progressing (%llu/%llu at level %u off %llu)...\n", + (unsigned long long) event->value.publish.completed, + (unsigned long long) event->value.publish.size, + event->value.publish.specifics.progress.depth, + (unsigned long long) event->value.publish.specifics. + progress.offset); +#endif + break; + case GNUNET_FS_STATUS_PUBLISH_COMPLETED: + printf ("Publishing complete, %llu kbps.\n", + (unsigned long long) (FILESIZE * 1000 / + (1 + + GNUNET_TIME_absolute_get_duration + (start).rel_value) / 1024)); + start = GNUNET_TIME_absolute_get (); + unindex = GNUNET_FS_unindex_start (fs, fn, "unindex"); + GNUNET_assert (unindex != NULL); + break; + case GNUNET_FS_STATUS_UNINDEX_COMPLETED: + printf ("Unindex complete, %llu kbps.\n", + (unsigned long long) (FILESIZE * 1000 / + (1 + + GNUNET_TIME_absolute_get_duration + (start).rel_value) / 1024)); + GNUNET_SCHEDULER_add_continuation (&abort_unindex_task, NULL, + GNUNET_SCHEDULER_REASON_PREREQ_DONE); + break; + case GNUNET_FS_STATUS_UNINDEX_PROGRESS: + consider_restart (event->status); + GNUNET_assert (unindex == event->value.unindex.uc); +#if VERBOSE + printf ("Unindex is progressing (%llu/%llu at level %u off %llu)...\n", + (unsigned long long) event->value.unindex.completed, + (unsigned long long) event->value.unindex.size, + event->value.unindex.specifics.progress.depth, + (unsigned long long) event->value.unindex.specifics. + progress.offset); +#endif + break; + case GNUNET_FS_STATUS_PUBLISH_SUSPEND: + if (event->value.publish.pc == publish) + publish = NULL; + break; + case GNUNET_FS_STATUS_PUBLISH_RESUME: + if (NULL == publish) + { + publish = event->value.publish.pc; + return "publish-context"; + } + break; + case GNUNET_FS_STATUS_UNINDEX_SUSPEND: + GNUNET_assert (event->value.unindex.uc == unindex); + unindex = NULL; + break; + case GNUNET_FS_STATUS_UNINDEX_RESUME: + GNUNET_assert (NULL == unindex); + unindex = event->value.unindex.uc; + return "unindex"; + case GNUNET_FS_STATUS_PUBLISH_ERROR: + FPRINTF (stderr, "Error publishing file: %s\n", + event->value.publish.specifics.error.message); + GNUNET_break (0); + GNUNET_SCHEDULER_add_continuation (&abort_publish_task, NULL, + GNUNET_SCHEDULER_REASON_PREREQ_DONE); + break; + case GNUNET_FS_STATUS_UNINDEX_ERROR: + FPRINTF (stderr, "Error unindexing file: %s\n", + event->value.unindex.specifics.error.message); + GNUNET_SCHEDULER_add_continuation (&abort_unindex_task, NULL, + GNUNET_SCHEDULER_REASON_PREREQ_DONE); + break; + case GNUNET_FS_STATUS_PUBLISH_START: + GNUNET_assert (0 == strcmp ("publish-context", event->value.publish.cctx)); + GNUNET_assert (NULL == event->value.publish.pctx); + GNUNET_assert (FILESIZE == event->value.publish.size); + GNUNET_assert (0 == event->value.publish.completed); + GNUNET_assert (1 == event->value.publish.anonymity); + break; + case GNUNET_FS_STATUS_PUBLISH_STOPPED: + GNUNET_assert (publish == event->value.publish.pc); + GNUNET_assert (FILESIZE == event->value.publish.size); + GNUNET_assert (1 == event->value.publish.anonymity); + GNUNET_FS_stop (fs); + fs = NULL; + break; + case GNUNET_FS_STATUS_UNINDEX_START: + consider_restart (event->status); + GNUNET_assert (unindex == NULL); + GNUNET_assert (0 == strcmp ("unindex", event->value.unindex.cctx)); + GNUNET_assert (0 == strcmp (fn, event->value.unindex.filename)); + GNUNET_assert (FILESIZE == event->value.unindex.size); + GNUNET_assert (0 == event->value.unindex.completed); + break; + case GNUNET_FS_STATUS_UNINDEX_STOPPED: + GNUNET_assert (unindex == event->value.unindex.uc); + GNUNET_SCHEDULER_add_continuation (&abort_publish_task, NULL, + GNUNET_SCHEDULER_REASON_PREREQ_DONE); + break; + default: + printf ("Unexpected event: %d\n", event->status); + break; + } + return NULL; +} + + +static void +setup_peer (struct PeerContext *p, const char *cfgname) +{ + p->cfg = GNUNET_CONFIGURATION_create (); +#if START_ARM + p->arm_proc = + GNUNET_OS_start_process (GNUNET_YES, NULL, NULL, "gnunet-service-arm", + "gnunet-service-arm", +#if VERBOSE + "-L", "DEBUG", +#endif + "-c", cfgname, NULL); +#endif + GNUNET_assert (GNUNET_OK == GNUNET_CONFIGURATION_load (p->cfg, cfgname)); +} + + +static void +stop_arm (struct PeerContext *p) +{ +#if START_ARM + if (NULL != p->arm_proc) + { + if (0 != GNUNET_OS_process_kill (p->arm_proc, SIGTERM)) + GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING, "kill"); + if (GNUNET_OS_process_wait (p->arm_proc) != GNUNET_OK) + GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING, "waitpid"); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "ARM process %u stopped\n", + GNUNET_OS_process_get_pid (p->arm_proc)); + GNUNET_OS_process_close (p->arm_proc); + p->arm_proc = NULL; + } +#endif + GNUNET_CONFIGURATION_destroy (p->cfg); +} + + +static void +run (void *cls, char *const *args, const char *cfgfile, + const struct GNUNET_CONFIGURATION_Handle *c) +{ + const char *keywords[] = { + "down_foo", + "down_bar", + }; + char *buf; + struct GNUNET_CONTAINER_MetaData *meta; + struct GNUNET_FS_Uri *kuri; + struct GNUNET_FS_FileInformation *fi; + size_t i; + struct GNUNET_FS_BlockOptions bo; + + cfg = c; + setup_peer (&p1, "test_fs_unindex_data.conf"); + fn = GNUNET_DISK_mktemp ("gnunet-unindex-test-dst"); + fs = GNUNET_FS_start (cfg, "test-fs-unindex-persistence", &progress_cb, NULL, + GNUNET_FS_FLAGS_PERSISTENCE, GNUNET_FS_OPTIONS_END); + GNUNET_assert (NULL != fs); + buf = GNUNET_malloc (FILESIZE); + for (i = 0; i < FILESIZE; i++) + buf[i] = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 256); + GNUNET_assert (FILESIZE == + GNUNET_DISK_fn_write (fn, buf, FILESIZE, + GNUNET_DISK_PERM_USER_READ | + GNUNET_DISK_PERM_USER_WRITE)); + GNUNET_free (buf); + meta = GNUNET_CONTAINER_meta_data_create (); + kuri = GNUNET_FS_uri_ksk_create_from_args (2, keywords); + bo.content_priority = 42; + bo.anonymity_level = 1; + bo.replication_level = 0; + bo.expiration_time = GNUNET_TIME_relative_to_absolute (LIFETIME); + fi = GNUNET_FS_file_information_create_from_file (fs, "publish-context", fn, + kuri, meta, GNUNET_YES, + &bo); + GNUNET_FS_uri_destroy (kuri); + GNUNET_CONTAINER_meta_data_destroy (meta); + GNUNET_assert (NULL != fi); + start = GNUNET_TIME_absolute_get (); + publish = + GNUNET_FS_publish_start (fs, fi, NULL, NULL, NULL, + GNUNET_FS_PUBLISH_OPTION_NONE); + GNUNET_assert (publish != NULL); +} + + +int +main (int argc, char *argv[]) +{ + char *const argvx[] = { + "test-fs-unindex", + "-c", + "test_fs_unindex_data.conf", +#if VERBOSE + "-L", "DEBUG", +#endif + NULL + }; + struct GNUNET_GETOPT_CommandLineOption options[] = { + GNUNET_GETOPT_OPTION_END + }; + + GNUNET_log_setup ("test_fs_unindex_persistence", +#if VERBOSE + "DEBUG", +#else + "WARNING", +#endif + NULL); + GNUNET_PROGRAM_run ((sizeof (argvx) / sizeof (char *)) - 1, argvx, + "test-fs-unindex", "nohelp", options, &run, NULL); + stop_arm (&p1); + GNUNET_DISK_directory_remove ("/tmp/gnunet-test-fs-unindex/"); + if (NULL != fn) + { + GNUNET_DISK_directory_remove (fn); + GNUNET_free (fn); + } + return 0; +} + +/* end of test_fs_unindex_persistence.c */ diff --git a/src/fs/test_fs_uri.c b/src/fs/test_fs_uri.c new file mode 100644 index 0000000..b7a58ec --- /dev/null +++ b/src/fs/test_fs_uri.c @@ -0,0 +1,330 @@ +/* + This file is part of GNUnet. + (C) 2003, 2004, 2006, 2007, 2009 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/test_fs_uri.c + * @brief Test for fs_uri.c + * @author Christian Grothoff + */ + +#include "platform.h" +#include "gnunet_util_lib.h" +#include "gnunet_fs_service.h" +#include "fs_api.h" + +#define ABORT() { fprintf(stderr, "Error at %s:%d\n", __FILE__, __LINE__); return 1; } + +static int +testKeyword () +{ + char *uri; + struct GNUNET_FS_Uri *ret; + char *emsg; + + if (NULL != (ret = GNUNET_FS_uri_parse ("gnunet://fs/ksk/++", &emsg))) + { + GNUNET_FS_uri_destroy (ret); + ABORT (); + } + GNUNET_free (emsg); + ret = GNUNET_FS_uri_parse ("gnunet://fs/ksk/foo+bar", &emsg); + if (ret == NULL) + { + GNUNET_free (emsg); + ABORT (); + } + if (!GNUNET_FS_uri_test_ksk (ret)) + { + GNUNET_FS_uri_destroy (ret); + ABORT (); + } + if ((2 != ret->data.ksk.keywordCount) || + (0 != strcmp (" foo", ret->data.ksk.keywords[0])) || + (0 != strcmp (" bar", ret->data.ksk.keywords[1]))) + { + GNUNET_FS_uri_destroy (ret); + ABORT (); + } + + uri = GNUNET_FS_uri_to_string (ret); + if (0 != strcmp (uri, "gnunet://fs/ksk/foo+bar")) + { + GNUNET_free (uri); + GNUNET_FS_uri_destroy (ret); + ABORT (); + } + GNUNET_free (uri); + GNUNET_FS_uri_destroy (ret); + return 0; +} + +static int +testLocation () +{ + struct GNUNET_FS_Uri *uri; + char *uric; + struct GNUNET_FS_Uri *uri2; + struct GNUNET_FS_Uri *baseURI; + char *emsg; + struct GNUNET_CONFIGURATION_Handle *cfg; + + baseURI = + GNUNET_FS_uri_parse + ("gnunet://fs/chk/C282GG70GKK41O4551011DO413KFBVTVMQG1OG30I0K4045N0G41HAPB82G680A02JRVVFO8URVRU2F159011DO41000000022RG820.RNVVVVOOLCLK065B5D04HTNVNSIB2AI022RG8200HSLK1CO1000ATQ98824DMA2032LIMG50CG0K057NVUVG200000H000004400000.42", + &emsg); + GNUNET_assert (baseURI != NULL); + GNUNET_assert (emsg == NULL); + cfg = GNUNET_CONFIGURATION_create (); + if (GNUNET_OK != GNUNET_CONFIGURATION_load (cfg, "test_fs_uri_data.conf")) + { + FPRINTF (stderr, "%s", "Failed to parse configuration file\n"); + GNUNET_FS_uri_destroy (baseURI); + GNUNET_CONFIGURATION_destroy (cfg); + return 1; + } + uri = GNUNET_FS_uri_loc_create (baseURI, cfg, GNUNET_TIME_absolute_get ()); + if (uri == NULL) + { + GNUNET_break (0); + GNUNET_FS_uri_destroy (baseURI); + GNUNET_CONFIGURATION_destroy (cfg); + return 1; + } + if (!GNUNET_FS_uri_test_loc (uri)) + { + GNUNET_break (0); + GNUNET_FS_uri_destroy (uri); + GNUNET_FS_uri_destroy (baseURI); + GNUNET_CONFIGURATION_destroy (cfg); + return 1; + } + uri2 = GNUNET_FS_uri_loc_get_uri (uri); + if (!GNUNET_FS_uri_test_equal (baseURI, uri2)) + { + GNUNET_break (0); + GNUNET_FS_uri_destroy (uri); + GNUNET_FS_uri_destroy (uri2); + GNUNET_FS_uri_destroy (baseURI); + GNUNET_CONFIGURATION_destroy (cfg); + return 1; + } + GNUNET_FS_uri_destroy (uri2); + GNUNET_FS_uri_destroy (baseURI); + uric = GNUNET_FS_uri_to_string (uri); +#if 0 + /* not for the faint of heart: */ + printf ("URI: `%s'\n", uric); +#endif + uri2 = GNUNET_FS_uri_parse (uric, &emsg); + GNUNET_free (uric); + if (uri2 == NULL) + { + GNUNET_break (0); + GNUNET_FS_uri_destroy (uri); + GNUNET_CONFIGURATION_destroy (cfg); + GNUNET_free (emsg); + return 1; + } + GNUNET_assert (NULL == emsg); + if (GNUNET_YES != GNUNET_FS_uri_test_equal (uri, uri2)) + { + GNUNET_break (0); + GNUNET_FS_uri_destroy (uri); + GNUNET_FS_uri_destroy (uri2); + GNUNET_CONFIGURATION_destroy (cfg); + return 1; + } + GNUNET_FS_uri_destroy (uri2); + GNUNET_FS_uri_destroy (uri); + GNUNET_CONFIGURATION_destroy (cfg); + return 0; +} + +static int +testNamespace (int i) +{ + char *uri; + struct GNUNET_FS_Uri *ret; + char *emsg; + + if (NULL != + (ret = + GNUNET_FS_uri_parse ("gnunet://fs/sks/D1KJS9H2A82Q65VKQ0ML3RFU6U1D3VUK", + &emsg))) + { + GNUNET_FS_uri_destroy (ret); + ABORT (); + } + GNUNET_free (emsg); + if (NULL != + (ret = + GNUNET_FS_uri_parse + ("gnunet://fs/sks/D1KJS9H2A82Q65VKQ0ML3RFU6U1D3V/test", &emsg))) + { + GNUNET_FS_uri_destroy (ret); + ABORT (); + } + GNUNET_free (emsg); + if (NULL != (ret = GNUNET_FS_uri_parse ("gnunet://fs/sks/test", &emsg))) + { + GNUNET_FS_uri_destroy (ret); + ABORT (); + } + GNUNET_free (emsg); + ret = + GNUNET_FS_uri_parse + ("gnunet://fs/sks/C282GG70GKK41O4551011DO413KFBVTVMQG1OG30I0K4045N0G41HAPB82G680A02JRVVFO8URVRU2F159011DO41000000022RG820/test", + &emsg); + if (ret == NULL) + { + GNUNET_free (emsg); + ABORT (); + } + if (GNUNET_FS_uri_test_ksk (ret)) + { + GNUNET_FS_uri_destroy (ret); + ABORT (); + } + if (!GNUNET_FS_uri_test_sks (ret)) + { + GNUNET_FS_uri_destroy (ret); + ABORT (); + } + + uri = GNUNET_FS_uri_to_string (ret); + if (0 != + strcmp (uri, + "gnunet://fs/sks/C282GG70GKK41O4551011DO413KFBVTVMQG1OG30I0K4045N0G41HAPB82G680A02JRVVFO8URVRU2F159011DO41000000022RG820/test")) + { + GNUNET_FS_uri_destroy (ret); + GNUNET_free (uri); + ABORT (); + } + GNUNET_free (uri); + GNUNET_FS_uri_destroy (ret); + return 0; +} + +static int +testFile (int i) +{ + char *uri; + struct GNUNET_FS_Uri *ret; + char *emsg; + + if (NULL != + (ret = + GNUNET_FS_uri_parse + ("gnunet://fs/chk/C282GG70GKK41O4551011DO413KFBVTVMQG1OG30I0K4045N0G41HAPB82G680A02JRVVFO8URVRU2F159011DO41000000022RG820.RNVVVVOOLCLK065B5D04HTNVNSIB2AI022RG8200HSLK1CO1000ATQ98824DMA2032LIMG50CG0K057NVUVG200000H00000440000.42", + &emsg))) + { + GNUNET_FS_uri_destroy (ret); + ABORT (); + } + GNUNET_free (emsg); + if (NULL != + (ret = + GNUNET_FS_uri_parse + ("gnunet://fs/chk/C282GG70GKK41O4551011DO413KFBVTVMQG1OG30I0K4045N0G41HAPB82G680A02JRVVFO8URVRU2F159011DO41000000022RG820.RNVVVVOOLCLK065B5D04HTNVNSIB2AI022RG8200HSLK1CO1000ATQ98824DMA2032LIMG50CG0K057NVUVG200000H000004400000", + &emsg))) + { + GNUNET_FS_uri_destroy (ret); + ABORT (); + } + GNUNET_free (emsg); + if (NULL != + (ret = + GNUNET_FS_uri_parse + ("gnunet://fs/chk/C282GG70GKK41O4551011DO413KFBVTVMQG1OG30I0K4045N0G41HAPB82G680A02JRVVFO8URVRU2F159011DO41000000022RG820.RNVVVVOOLCLK065B5D04HTNVNSIB2AI022RG8200HSLK1CO1000ATQ98824DMA2032LIMG50CG0K057NVUVG200000H000004400000.FGH", + &emsg))) + { + GNUNET_FS_uri_destroy (ret); + ABORT (); + } + GNUNET_free (emsg); + ret = + GNUNET_FS_uri_parse + ("gnunet://fs/chk/C282GG70GKK41O4551011DO413KFBVTVMQG1OG30I0K4045N0G41HAPB82G680A02JRVVFO8URVRU2F159011DO41000000022RG820.RNVVVVOOLCLK065B5D04HTNVNSIB2AI022RG8200HSLK1CO1000ATQ98824DMA2032LIMG50CG0K057NVUVG200000H000004400000.42", + &emsg); + if (ret == NULL) + { + GNUNET_free (emsg); + ABORT (); + } + if (GNUNET_FS_uri_test_ksk (ret)) + { + GNUNET_FS_uri_destroy (ret); + ABORT (); + } + if (GNUNET_FS_uri_test_sks (ret)) + { + GNUNET_FS_uri_destroy (ret); + ABORT (); + } + if (GNUNET_ntohll (ret->data.chk.file_length) != 42) + { + GNUNET_FS_uri_destroy (ret); + ABORT (); + } + + uri = GNUNET_FS_uri_to_string (ret); + if (0 != + strcmp (uri, + "gnunet://fs/chk/C282GG70GKK41O4551011DO413KFBVTVMQG1OG30I0K4045N0G41HAPB82G680A02JRVVFO8URVRU2F159011DO41000000022RG820.RNVVVVOOLCLK065B5D04HTNVNSIB2AI022RG8200HSLK1CO1000ATQ98824DMA2032LIMG50CG0K057NVUVG200000H000004400000.42")) + { + GNUNET_free (uri); + GNUNET_FS_uri_destroy (ret); + ABORT (); + } + GNUNET_free (uri); + GNUNET_FS_uri_destroy (ret); + return 0; +} + +int +main (int argc, char *argv[]) +{ + int failureCount = 0; + int i; + + GNUNET_log_setup ("test_fs_uri", +#if VERBOSE + "DEBUG", +#else + "WARNING", +#endif + NULL); + GNUNET_CRYPTO_random_disable_entropy_gathering (); + failureCount += testKeyword (); + failureCount += testLocation (); + for (i = 0; i < 255; i++) + { + /* FPRINTF (stderr, "%s", "."); */ + failureCount += testNamespace (i); + failureCount += testFile (i); + } + /* FPRINTF (stderr, "%s", "\n"); */ + GNUNET_DISK_directory_remove ("/tmp/gnunet-test-fs-uri"); + if (failureCount != 0) + return 1; + return 0; +} + +/* end of test_fs_uri.c */ diff --git a/src/fs/test_fs_uri_data.conf b/src/fs/test_fs_uri_data.conf new file mode 100644 index 0000000..abc5a73 --- /dev/null +++ b/src/fs/test_fs_uri_data.conf @@ -0,0 +1,7 @@ +@INLINE@ test_fs_defaults.conf +[PATHS] +SERVICEHOME = /tmp/gnunet-test-fs-uri/ + +[arm] +DEFAULTSERVICES = topology hostlist + diff --git a/src/fs/test_gnunet_fs_idx.py.in b/src/fs/test_gnunet_fs_idx.py.in new file mode 100755 index 0000000..6bb7d0d --- /dev/null +++ b/src/fs/test_gnunet_fs_idx.py.in @@ -0,0 +1,73 @@ +#!@PYTHON@ +# This file is part of GNUnet. +# (C) 2010 Christian Grothoff (and other contributing authors) +# +# GNUnet is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published +# by the Free Software Foundation; either version 2, or (at your +# option) any later version. +# +# GNUnet is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with GNUnet; see the file COPYING. If not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. +# +# Testcase for file-sharing command-line tools (indexing and unindexing) +import sys +import os +import subprocess +import re +import shutil + +srcdir = "../.." +gnunet_pyexpect_dir = os.path.join (srcdir, "contrib") +if gnunet_pyexpect_dir not in sys.path: + sys.path.append (gnunet_pyexpect_dir) + +from gnunet_pyexpect import pexpect + +if os.name == 'posix': + download = 'gnunet-download' + gnunetarm = 'gnunet-arm' + publish = 'gnunet-publish' + unindex = 'gnunet-unindex' +elif os.name == 'nt': + download = 'gnunet-download.exe' + gnunetarm = 'gnunet-arm.exe' + publish = 'gnunet-publish.exe' + unindex = 'gnunet-unindex.exe' + +if os.name == "nt": + shutil.rmtree (os.path.join (os.getenv ("TEMP"), "gnunet-test-fs-py-idx"), True) +else: + shutil.rmtree ("/tmp/gnunet-test-fs-py-idx", True) + +arm = subprocess.Popen ([gnunetarm, '-sq', '-c', 'test_gnunet_fs_idx_data.conf']) +arm.communicate () + +try: + pub = pexpect () + pub.spawn (None, [publish, '-c', 'test_gnunet_fs_idx_data.conf', '-m', "description:The GNU Public License", '-k', 'gpl', '../../COPYING'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pub.expect ("stdout", re.compile (r"URI is `gnunet://fs/chk/PC0M19QMQC0BPSHR6BGA228PP6INER1D610MGEMOMEM87222FN8HVUO7PQGO0O9HD2GVLHF2N5IDHEQUNK6LKE428FPO96SKQEA486O\.PG7K85JGQ6N599MD5HEP3CHEVFPKQD9JB6NPSLVA3T1SKDS66CFI499VS6MGQ88B0QUAVT1282TCRD4GGFVUKDLGI8F0SPIANA3J2LG\.35147'\.\r?\n")) + + down = pexpect () + down.spawn (None, [download, '-c', 'test_gnunet_fs_idx_data.conf', '-o', 'COPYING', 'gnunet://fs/chk/PC0M19QMQC0BPSHR6BGA228PP6INER1D610MGEMOMEM87222FN8HVUO7PQGO0O9HD2GVLHF2N5IDHEQUNK6LKE428FPO96SKQEA486O.PG7K85JGQ6N599MD5HEP3CHEVFPKQD9JB6NPSLVA3T1SKDS66CFI499VS6MGQ88B0QUAVT1282TCRD4GGFVUKDLGI8F0SPIANA3J2LG.35147'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + down.expect ("stdout", re.compile (r"Downloading `COPYING' done (.*).\r?\n")) + os.remove ("COPYING") + + un = pexpect () + un.spawn (None, [unindex, '-c', 'test_gnunet_fs_idx_data.conf', '../../COPYING'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + un.expect ("stdout", re.compile (r'Unindexing done\.\r?\n')) + +finally: + arm = subprocess.Popen ([gnunetarm, '-eq', '-c', 'test_gnunet_fs_idx_data.conf']) + arm.communicate () + if os.name == "nt": + shutil.rmtree (os.path.join (os.getenv ("TEMP"), "gnunet-test-fs-py-idx"), True) + else: + shutil.rmtree ("/tmp/gnunet-test-fs-py-idx", True) diff --git a/src/fs/test_gnunet_fs_idx_data.conf b/src/fs/test_gnunet_fs_idx_data.conf new file mode 100644 index 0000000..f852011 --- /dev/null +++ b/src/fs/test_gnunet_fs_idx_data.conf @@ -0,0 +1,8 @@ +@INLINE@ test_fs_defaults.conf +[PATHS] +SERVICEHOME = /tmp/gnunet-test-fs-py-idx/ +DEFAULTCONFIG = test_gnunet_fs_idx_data.conf + +[transport] +PLUGINS = + diff --git a/src/fs/test_gnunet_fs_ns.py.in b/src/fs/test_gnunet_fs_ns.py.in new file mode 100755 index 0000000..ff892b4 --- /dev/null +++ b/src/fs/test_gnunet_fs_ns.py.in @@ -0,0 +1,80 @@ +#!@PYTHON@ +# This file is part of GNUnet. +# (C) 2010 Christian Grothoff (and other contributing authors) +# +# GNUnet is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published +# by the Free Software Foundation; either version 2, or (at your +# option) any later version. +# +# GNUnet is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with GNUnet; see the file COPYING. If not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. +# +# Testcase for file-sharing command-line tools (namespaces) +import sys +import os +import subprocess +import re +import shutil + +srcdir = "../.." +gnunet_pyexpect_dir = os.path.join (srcdir, "contrib") +if gnunet_pyexpect_dir not in sys.path: + sys.path.append (gnunet_pyexpect_dir) + +from gnunet_pyexpect import pexpect + +if os.name == 'posix': + pseudonym = 'gnunet-pseudonym' + gnunetarm = 'gnunet-arm' + publish = 'gnunet-publish' + unindex = 'gnunet-unindex' + search = 'gnunet-search' +elif os.name == 'nt': + pseudonym = 'gnunet-pseudonym.exe' + gnunetarm = 'gnunet-arm.exe' + publish = 'gnunet-publish.exe' + unindex = 'gnunet-unindex.exe' + search = 'gnunet-search.exe' + +if os.name == "nt": + shutil.rmtree (os.path.join (os.getenv ("TEMP"), "gnunet-test-fs-py-ns"), True) +else: + shutil.rmtree ("/tmp/gnunet-test-fs-py-ns", True) + +arm = subprocess.Popen ([gnunetarm, '-sq', '-c', 'test_gnunet_fs_ns_data.conf']) +arm.communicate () + +try: + pseu = pexpect () + pseu.spawn (None, [pseudonym, '-c', 'test_gnunet_fs_ns_data.conf', '-C', 'licenses', '-k', 'gplad', '-m', 'description:Free Software Licenses', '-R', 'myroot'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pseu.spawn (None, [pseudonym, '-c', 'test_gnunet_fs_ns_data.conf', '-o'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pseu.expect ("stdout", re.compile (r"licenses (.*)\r?\n")) + + pub = pexpect () + pub.spawn (None, [publish, '-c', 'test_gnunet_fs_ns_data.conf', '-k', 'licenses', '-P', 'licenses', '-u', 'gnunet://fs/chk/PC0M19QMQC0BPSHR6BGA228PP6INER1D610MGEMOMEM87222FN8HVUO7PQGO0O9HD2GVLHF2N5IDHEQUNK6LKE428FPO96SKQEA486O.PG7K85JGQ6N599MD5HEP3CHEVFPKQD9JB6NPSLVA3T1SKDS66CFI499VS6MGQ88B0QUAVT1282TCRD4GGFVUKDLGI8F0SPIANA3J2LG.35147', '-t', 'gpl', '-N', 'gpl3'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + + s = pexpect () + s.spawn (None, [search, '-V', '-t', '1000', '-N', '1', '-c', 'test_gnunet_fs_ns_data.conf', 'gplad'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + s.expect ("stdout", re.compile (r'#0:\r?\n')) + s.expect ("stdout", re.compile (r'gnunet-download gnunet://fs/sks/.*/myroot\r?\n')) + s.expect ("stdout", re.compile (r'\s*description: Free Software Licenses\r?\n')) + + pseu = pexpect () + pseu.spawn (None, [pseudonym, '-c', 'test_gnunet_fs_ns_data.conf'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pseu.expect ("stdout", re.compile (r'Free Software Licenses.*:\r?\n')) + +finally: + arm = subprocess.Popen ([gnunetarm, '-eq', '-c', 'test_gnunet_fs_ns_data.conf']) + arm.communicate () + if os.name == "nt": + shutil.rmtree (os.path.join (os.getenv ("TEMP"), "gnunet-test-fs-py-ns"), True) + else: + shutil.rmtree ("/tmp/gnunet-test-fs-py-ns", True) diff --git a/src/fs/test_gnunet_fs_ns_data.conf b/src/fs/test_gnunet_fs_ns_data.conf new file mode 100644 index 0000000..5f297ab --- /dev/null +++ b/src/fs/test_gnunet_fs_ns_data.conf @@ -0,0 +1,8 @@ +@INLINE@ test_fs_defaults.conf +[PATHS] +SERVICEHOME = /tmp/gnunet-test-fs-py-ns/ +DEFAULTCONFIG = test_gnunet_fs_ns_data.conf + +[transport] +PLUGINS = + diff --git a/src/fs/test_gnunet_fs_psd.py.in b/src/fs/test_gnunet_fs_psd.py.in new file mode 100755 index 0000000..9790e13 --- /dev/null +++ b/src/fs/test_gnunet_fs_psd.py.in @@ -0,0 +1,79 @@ +#!@PYTHON@ +# This file is part of GNUnet. +# (C) 2010 Christian Grothoff (and other contributing authors) +# +# GNUnet is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published +# by the Free Software Foundation; either version 2, or (at your +# option) any later version. +# +# GNUnet is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with GNUnet; see the file COPYING. If not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. +# +# Testcase for file-sharing command-line tools (publish, search, download) +import sys +import os +import subprocess +import re +import shutil + +srcdir = "../.." +gnunet_pyexpect_dir = os.path.join (srcdir, "contrib") +if gnunet_pyexpect_dir not in sys.path: + sys.path.append (gnunet_pyexpect_dir) + +from gnunet_pyexpect import pexpect + +if os.name == 'posix': + download = 'gnunet-download' + gnunetarm = 'gnunet-arm' + publish = 'gnunet-publish' + unindex = 'gnunet-unindex' + search = 'gnunet-search' +elif os.name == 'nt': + download = 'gnunet-download.exe' + gnunetarm = 'gnunet-arm.exe' + publish = 'gnunet-publish.exe' + unindex = 'gnunet-unindex.exe' + search = 'gnunet-search.exe' + +if os.name == "nt": + shutil.rmtree (os.path.join (os.getenv ("TEMP"), "gnunet-test-fs-py-psd"), True) +else: + shutil.rmtree ("/tmp/gnunet-test-fs-py-psd", True) + +arm = subprocess.Popen ([gnunetarm, '-sq', '-c', 'test_gnunet_fs_psd_data.conf']) +arm.communicate () + +# first, basic publish-search-download run +try: + pub = pexpect () + pub.spawn (None, [publish, '-c', 'test_gnunet_fs_psd_data.conf', '-n', '-m', "description:The GNU Public License", '-k', 'gpl', '../../COPYING'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pub.expect ("stdout", re.compile (r"Publishing `.+[\\/]..[\\/]..[\\/]COPYING' done\.\r?\n")) + pub.expect ("stdout", re.compile (r"URI is `gnunet://fs/chk/PC0M19QMQC0BPSHR6BGA228PP6INER1D610MGEMOMEM87222FN8HVUO7PQGO0O9HD2GVLHF2N5IDHEQUNK6LKE428FPO96SKQEA486O\.PG7K85JGQ6N599MD5HEP3CHEVFPKQD9JB6NPSLVA3T1SKDS66CFI499VS6MGQ88B0QUAVT1282TCRD4GGFVUKDLGI8F0SPIANA3J2LG\.35147'\.\r?\n")) + + s = pexpect () + s.spawn (None, [search, '-V', '-t', '1000', '-N', '1', '-c', 'test_gnunet_fs_psd_data.conf', 'gpl'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + s.expect ("stdout", re.compile (r'#0:\r?\n')) + s.expect ("stdout", re.compile (r'gnunet-download -o "COPYING" gnunet://fs/chk/PC0M19QMQC0BPSHR6BGA228PP6INER1D610MGEMOMEM87222FN8HVUO7PQGO0O9HD2GVLHF2N5IDHEQUNK6LKE428FPO96SKQEA486O\.PG7K85JGQ6N599MD5HEP3CHEVFPKQD9JB6NPSLVA3T1SKDS66CFI499VS6MGQ88B0QUAVT1282TCRD4GGFVUKDLGI8F0SPIANA3J2LG\.35147\r?\n')) + s.expect ("stdout", re.compile (r"\s*description: The GNU Public License\r?\n")) + + down = pexpect () + down.spawn (None, [download, '-c', 'test_gnunet_fs_psd_data.conf', '-o', 'COPYING', 'gnunet://fs/chk/PC0M19QMQC0BPSHR6BGA228PP6INER1D610MGEMOMEM87222FN8HVUO7PQGO0O9HD2GVLHF2N5IDHEQUNK6LKE428FPO96SKQEA486O.PG7K85JGQ6N599MD5HEP3CHEVFPKQD9JB6NPSLVA3T1SKDS66CFI499VS6MGQ88B0QUAVT1282TCRD4GGFVUKDLGI8F0SPIANA3J2LG.35147'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + down.expect ("stdout", re.compile (r"Downloading `COPYING' done (.*).\r?\n")) + os.remove ("COPYING") + +finally: + arm = subprocess.Popen ([gnunetarm, '-eq', '-c', 'test_gnunet_fs_psd_data.conf']) + arm.communicate () + if os.name == "nt": + shutil.rmtree (os.path.join (os.getenv ("TEMP"), "gnunet-test-fs-py-psd"), True) + else: + shutil.rmtree ("/tmp/gnunet-test-fs-py-psd", True) diff --git a/src/fs/test_gnunet_fs_psd_data.conf b/src/fs/test_gnunet_fs_psd_data.conf new file mode 100644 index 0000000..b68c6b5 --- /dev/null +++ b/src/fs/test_gnunet_fs_psd_data.conf @@ -0,0 +1,8 @@ +@INLINE@ test_fs_defaults.conf +[PATHS] +SERVICEHOME = /tmp/gnunet-test-fs-py-psd/ +DEFAULTCONFIG = test_gnunet_fs_psd_data.conf + +[transport] +PLUGINS = + diff --git a/src/fs/test_gnunet_fs_rec.py.in b/src/fs/test_gnunet_fs_rec.py.in new file mode 100755 index 0000000..e86bb0a --- /dev/null +++ b/src/fs/test_gnunet_fs_rec.py.in @@ -0,0 +1,109 @@ +#!@PYTHON@ +# This file is part of GNUnet. +# (C) 2010 Christian Grothoff (and other contributing authors) +# +# GNUnet is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published +# by the Free Software Foundation; either version 2, or (at your +# option) any later version. +# +# GNUnet is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with GNUnet; see the file COPYING. If not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. +# +# Testcase for file-sharing command-line tools (recursive publishing & download) +import sys +import os +import subprocess +import re +import shutil + +srcdir = "../.." +gnunet_pyexpect_dir = os.path.join (srcdir, "contrib") +if gnunet_pyexpect_dir not in sys.path: + sys.path.append (gnunet_pyexpect_dir) + +from gnunet_pyexpect import pexpect + +if os.name == 'posix': + download = 'gnunet-download' + gnunetarm = 'gnunet-arm' + publish = 'gnunet-publish' + unindex = 'gnunet-unindex' + search = 'gnunet-search' + directory = 'gnunet-directory' +elif os.name == 'nt': + download = 'gnunet-download.exe' + gnunetarm = 'gnunet-arm.exe' + publish = 'gnunet-publish.exe' + unindex = 'gnunet-unindex.exe' + search = 'gnunet-search.exe' + directory = 'gnunet-directory.exe' + +if os.name == "nt": + shutil.rmtree (os.path.join (os.getenv ("TEMP"), "gnunet-test-fs-py-rec"), True) +else: + shutil.rmtree ("/tmp/gnunet-test-fs-py-rec", True) + +arm = subprocess.Popen ([gnunetarm, '-sq', '-c', 'test_gnunet_fs_rec_data.conf']) +arm.communicate () + +# pray that `tar' is in PATH +os.system ('tar xfz test_gnunet_fs_rec_data.tgz') +# first, basic publish-search-download run +try: + pub = pexpect () + pub.spawn (None, [publish, '-c', 'test_gnunet_fs_rec_data.conf', '-d', '-k', 'testdir', 'dir/'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + # Can't say much for publishing, except that the last one is the toplevel directory + pub.expect ("stdout", re.compile (r"Publishing `.+' done\.\r?\n")) + pub.expect ("stdout", re.compile (r"URI is `gnunet://fs/chk/[A-Z0-9]{103}\.[A-Z0-9]{103}\.\d+'\.\r?\n")) + pub.expect ("stdout", re.compile (r"Publishing `.+' done\.\r?\n")) + pub.expect ("stdout", re.compile (r"URI is `gnunet://fs/chk/[A-Z0-9]{103}\.[A-Z0-9]{103}\.\d+'\.\r?\n")) + pub.expect ("stdout", re.compile (r"Publishing `.+' done\.\r?\n")) + pub.expect ("stdout", re.compile (r"URI is `gnunet://fs/chk/[A-Z0-9]{103}\.[A-Z0-9]{103}\.\d+'\.\r?\n")) + pub.expect ("stdout", re.compile (r"Publishing `.+' done\.\r?\n")) + pub.expect ("stdout", re.compile (r"URI is `gnunet://fs/chk/[A-Z0-9]{103}\.[A-Z0-9]{103}\.\d+'\.\r?\n")) + pub.expect ("stdout", re.compile (r"Publishing `.+' done\.\r?\n")) + pub.expect ("stdout", re.compile (r"URI is `gnunet://fs/chk/[A-Z0-9]{103}\.[A-Z0-9]{103}\.\d+'\.\r?\n")) + pub.expect ("stdout", re.compile (r"Publishing `.+' done\.\r?\n")) + pub.expect ("stdout", re.compile (r"URI is `gnunet://fs/chk/[A-Z0-9]{103}\.[A-Z0-9]{103}\.\d+'\.\r?\n")) + pub.expect ("stdout", re.compile (r"Publishing `.+[\\/]dir[\\/]' done\.\r?\n")) + m = pub.expect ("stdout", re.compile (r".+\r?\n")) + if not m: + sys.exit (3) + output = m.string + url = output[output.find ("`")+1:output.find("'")] + + down = pexpect () + down.spawn (None, [download, '-c', 'test_gnunet_fs_rec_data.conf', '-R', '-o', 'rdir.gnd', url], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + down.expect ("stdout", re.compile (r"Downloading `rdir.gnd' done (.*).\r?\n")) + + d = pexpect () + d.spawn (None, [directory, '-c', 'test_gnunet_fs_rec_data.conf', 'rdir/a.gnd'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + d.expect ("stdout", re.compile (r"Directory `a/' meta data:\r?\n")) + d.expect ("stdout", re.compile (r"Directory `a/' contents:\r?\n")) + d.expect ("stdout", re.compile (r"COPYING (.*)\r?\n")) + d.expect ("stdout", re.compile (r"INSTALL (.*)\r?\n")) + + os.remove ("rdir/b.gnd") + os.remove ("rdir/a.gnd") + if 0 != os.system ("diff -r dir rdir"): + raise Exception ("Unexpected difference between source directory and downloaded result") + + +finally: + arm = subprocess.Popen ([gnunetarm, '-eq', '-c', 'test_gnunet_fs_rec_data.conf']) + arm.communicate () + if os.name == "nt": + shutil.rmtree (os.path.join (os.getenv ("TEMP"), "gnunet-test-fs-py-rec"), True) + else: + shutil.rmtree ("/tmp/gnunet-test-fs-py-rec", True) + shutil.rmtree ("dir", True) + shutil.rmtree ("rdir", True) + shutil.rmtree ("rdir.gnd", True) diff --git a/src/fs/test_gnunet_fs_rec_data.conf b/src/fs/test_gnunet_fs_rec_data.conf new file mode 100644 index 0000000..dae8b19 --- /dev/null +++ b/src/fs/test_gnunet_fs_rec_data.conf @@ -0,0 +1,8 @@ +@INLINE@ test_fs_defaults.conf +[PATHS] +SERVICEHOME = /tmp/gnunet-test-fs-py-rec/ +DEFAULTCONFIG = test_gnunet_fs_rec_data.conf + +[transport] +PLUGINS = + diff --git a/src/fs/test_gnunet_fs_rec_data.tgz b/src/fs/test_gnunet_fs_rec_data.tgz Binary files differnew file mode 100644 index 0000000..6977943 --- /dev/null +++ b/src/fs/test_gnunet_fs_rec_data.tgz diff --git a/src/fs/test_gnunet_service_fs_migration.c b/src/fs/test_gnunet_service_fs_migration.c new file mode 100644 index 0000000..8b85e5e --- /dev/null +++ b/src/fs/test_gnunet_service_fs_migration.c @@ -0,0 +1,221 @@ +/* + This file is part of GNUnet. + (C) 2010 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/test_gnunet_service_fs_migration.c + * @brief test content migration between two peers + * @author Christian Grothoff + */ +#include "platform.h" +#include "fs_test_lib.h" +#include "gnunet_testing_lib.h" + +#define VERBOSE GNUNET_NO + +/** + * File-size we use for testing. + */ +#define FILESIZE (2 * 32 * 1024) + +/** + * How long until we give up on transmitting the message? + */ +#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 120) + +/** + * How long do we give the peers for content migration? + */ +#define MIGRATION_DELAY GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 90) + +#define SEED 42 + +static struct GNUNET_FS_TestDaemon *daemons[2]; + +static int ok; + +static struct GNUNET_TIME_Absolute start_time; + +static struct GNUNET_FS_TEST_ConnectContext *cc; + +static void +do_stop (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct GNUNET_TIME_Relative del; + char *fancy; + + if (NULL != cc) + { + GNUNET_FS_TEST_daemons_connect_cancel (cc); + cc = NULL; + } + GNUNET_FS_TEST_daemons_stop (2, daemons); + if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_PREREQ_DONE)) + { + del = GNUNET_TIME_absolute_get_duration (start_time); + if (del.rel_value == 0) + del.rel_value = 1; + fancy = + GNUNET_STRINGS_byte_size_fancy (((unsigned long long) FILESIZE) * + 1000LL / del.rel_value); + FPRINTF (stdout, "Download speed was %s/s\n", fancy); + GNUNET_free (fancy); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Finished download, shutting down\n", + (unsigned long long) FILESIZE); + } + else + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Timeout during download, shutting down with error\n"); + ok = 1; + } +} + + +static void +do_download (void *cls, const char *emsg) +{ + struct GNUNET_FS_Uri *uri = cls; + + if (emsg != NULL) + { + GNUNET_FS_TEST_daemons_stop (2, daemons); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Failed to stop source daemon: %s\n", + emsg); + GNUNET_FS_uri_destroy (uri); + ok = 1; + return; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Downloading %llu bytes\n", + (unsigned long long) FILESIZE); + start_time = GNUNET_TIME_absolute_get (); + GNUNET_FS_TEST_download (daemons[0], TIMEOUT, 1, SEED, uri, VERBOSE, &do_stop, + NULL); + GNUNET_FS_uri_destroy (uri); +} + + +static void +stop_source_peer (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct GNUNET_FS_Uri *uri = cls; + struct GNUNET_TESTING_PeerGroup *pg; + + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Stopping source peer\n"); + pg = GNUNET_FS_TEST_get_group (daemons); + GNUNET_TESTING_daemons_vary (pg, 1, GNUNET_NO, TIMEOUT, &do_download, uri); +} + + +static void +do_wait (void *cls, const struct GNUNET_FS_Uri *uri) +{ + struct GNUNET_FS_Uri *d; + + if (NULL == uri) + { + GNUNET_FS_TEST_daemons_stop (2, daemons); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Timeout during upload attempt, shutting down with error\n"); + ok = 1; + return; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Waiting to allow content to migrate\n"); + d = GNUNET_FS_uri_dup (uri); + (void) GNUNET_SCHEDULER_add_delayed (MIGRATION_DELAY, &stop_source_peer, d); +} + + +static void +do_publish (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + cc = NULL; + if (0 == (tc->reason & GNUNET_SCHEDULER_REASON_PREREQ_DONE)) + { + GNUNET_FS_TEST_daemons_stop (2, daemons); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Timeout during connect attempt, shutting down with error\n"); + ok = 1; + return; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Publishing %llu bytes\n", + (unsigned long long) FILESIZE); + GNUNET_FS_TEST_publish (daemons[1], TIMEOUT, 1, GNUNET_NO, FILESIZE, SEED, + VERBOSE, &do_wait, NULL); +} + + +static void +do_connect (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + if (0 == (tc->reason & GNUNET_SCHEDULER_REASON_PREREQ_DONE)) + { + FPRINTF (stderr, "%s", "Daemons failed to start!\n"); + GNUNET_break (0); + ok = 1; + return; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Daemons started, will now try to connect them\n"); + cc = GNUNET_FS_TEST_daemons_connect (daemons[0], daemons[1], TIMEOUT, + &do_publish, NULL); +} + + +static void +run (void *cls, char *const *args, const char *cfgfile, + const struct GNUNET_CONFIGURATION_Handle *cfg) +{ + GNUNET_FS_TEST_daemons_start ("test_gnunet_service_fs_migration_data.conf", + TIMEOUT, 2, daemons, &do_connect, NULL); +} + + +int +main (int argc, char *argv[]) +{ + char *const argvx[] = { + "test-gnunet-service-fs-migration", + "-c", + "fs_test_lib_data.conf", +#if VERBOSE + "-L", "DEBUG", +#endif + NULL + }; + struct GNUNET_GETOPT_CommandLineOption options[] = { + GNUNET_GETOPT_OPTION_END + }; + + GNUNET_DISK_directory_remove ("/tmp/test-gnunet-service-fs-migration/"); + GNUNET_log_setup ("test_gnunet_service_fs_migration", +#if VERBOSE + "DEBUG", +#else + "WARNING", +#endif + NULL); + GNUNET_PROGRAM_run ((sizeof (argvx) / sizeof (char *)) - 1, argvx, + "test-gnunet-service-fs-migration", "nohelp", options, + &run, NULL); + GNUNET_DISK_directory_remove ("/tmp/test-gnunet-service-fs-migration/"); + return ok; +} + +/* end of test_gnunet_service_fs_migration.c */ diff --git a/src/fs/test_gnunet_service_fs_migration_data.conf b/src/fs/test_gnunet_service_fs_migration_data.conf new file mode 100644 index 0000000..9c05a88 --- /dev/null +++ b/src/fs/test_gnunet_service_fs_migration_data.conf @@ -0,0 +1,9 @@ +@INLINE@ test_fs_defaults.conf +[PATHS] +SERVICEHOME = /tmp/test-gnunet-service-fs-migration/ +DEFAULTCONFIG = test_gnunet_service_fs_migration_data.conf + + +[ats] +WAN_QUOTA_IN = 3932160 +WAN_QUOTA_OUT = 3932160 diff --git a/src/fs/test_gnunet_service_fs_p2p.c b/src/fs/test_gnunet_service_fs_p2p.c new file mode 100644 index 0000000..a853897 --- /dev/null +++ b/src/fs/test_gnunet_service_fs_p2p.c @@ -0,0 +1,176 @@ +/* + This file is part of GNUnet. + (C) 2010 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ + +/** + * @file fs/test_gnunet_service_fs_p2p.c + * @brief test P2P routing using simple publish + download operation + * @author Christian Grothoff + */ +#include "platform.h" +#include "fs_test_lib.h" + +#define VERBOSE GNUNET_NO + +/** + * File-size we use for testing. + */ +#define FILESIZE (1024 * 1024 * 1) + +/** + * How long until we give up on the download? + */ +#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 300) + +#define NUM_DAEMONS 2 + +#define SEED 42 + +static struct GNUNET_FS_TestDaemon *daemons[NUM_DAEMONS]; + +static int ok; + +static struct GNUNET_TIME_Absolute start_time; + +static struct GNUNET_FS_TEST_ConnectContext *cc; + +static void +do_stop (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct GNUNET_TIME_Relative del; + char *fancy; + + if (NULL != cc) + { + GNUNET_FS_TEST_daemons_connect_cancel (cc); + cc = NULL; + } + GNUNET_FS_TEST_daemons_stop (NUM_DAEMONS, daemons); + if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_PREREQ_DONE)) + { + del = GNUNET_TIME_absolute_get_duration (start_time); + if (del.rel_value == 0) + del.rel_value = 1; + fancy = + GNUNET_STRINGS_byte_size_fancy (((unsigned long long) FILESIZE) * + 1000LL / del.rel_value); + FPRINTF (stdout, "Download speed was %s/s\n", fancy); + GNUNET_free (fancy); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Finished download, shutting down\n", + (unsigned long long) FILESIZE); + } + else + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Timeout during download, shutting down with error\n"); + ok = 1; + } +} + + +static void +do_download (void *cls, const struct GNUNET_FS_Uri *uri) +{ + if (NULL == uri) + { + GNUNET_FS_TEST_daemons_stop (NUM_DAEMONS, daemons); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Timeout during upload attempt, shutting down with error\n"); + ok = 1; + return; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Downloading %llu bytes\n", + (unsigned long long) FILESIZE); + start_time = GNUNET_TIME_absolute_get (); + GNUNET_FS_TEST_download (daemons[0], TIMEOUT, 1, SEED, uri, VERBOSE, &do_stop, + NULL); +} + + +static void +do_publish (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + cc = NULL; + if (0 == (tc->reason & GNUNET_SCHEDULER_REASON_PREREQ_DONE)) + { + GNUNET_FS_TEST_daemons_stop (NUM_DAEMONS, daemons); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Timeout during connect attempt, shutting down with error\n"); + ok = 1; + return; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Publishing %llu bytes\n", + (unsigned long long) FILESIZE); + GNUNET_FS_TEST_publish (daemons[1], TIMEOUT, 1, GNUNET_NO, FILESIZE, SEED, + VERBOSE, &do_download, NULL); +} + + +static void +do_connect (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + GNUNET_assert (0 != (tc->reason & GNUNET_SCHEDULER_REASON_PREREQ_DONE)); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Daemons started, will now try to connect them\n"); + cc = GNUNET_FS_TEST_daemons_connect (daemons[0], daemons[1], TIMEOUT, + &do_publish, NULL); +} + + +static void +run (void *cls, char *const *args, const char *cfgfile, + const struct GNUNET_CONFIGURATION_Handle *cfg) +{ + GNUNET_FS_TEST_daemons_start ("fs_test_lib_data.conf", TIMEOUT, NUM_DAEMONS, + daemons, &do_connect, NULL); +} + + +int +main (int argc, char *argv[]) +{ + char *const argvx[] = { + "test-gnunet-service-fs-p2p", + "-c", + "fs_test_lib_data.conf", +#if VERBOSE + "-L", "DEBUG", +#endif + NULL + }; + struct GNUNET_GETOPT_CommandLineOption options[] = { + GNUNET_GETOPT_OPTION_END + }; + + GNUNET_DISK_directory_remove ("/tmp/gnunet-test-fs-lib/"); + GNUNET_log_setup ("test_gnunet_service_fs_p2p", +#if VERBOSE + "DEBUG", +#else + "WARNING", +#endif + NULL); + GNUNET_PROGRAM_run ((sizeof (argvx) / sizeof (char *)) - 1, argvx, + "test-gnunet-service-fs-p2p", "nohelp", options, &run, + NULL); + GNUNET_DISK_directory_remove ("/tmp/gnunet-test-fs-lib/"); + return ok; +} + +/* end of test_gnunet_service_fs_p2p.c */ |