diff options
159 files changed, 19284 insertions, 14735 deletions
@@ -32,7 +32,6 @@ Werner Koch <wk@gnupg.org> Contributions also came from: Adam Warrington [ UPnP ] Adriano Peluso [ Documentation export to Texinfo ] -ng0 <ng0@infotropique.org> [ Documentation export to Texinfo ] Alex Harper [ OS X CPU load ] Andrew McDonald <andrew@mcdonald.org.uk> [ SHA-512] Andy Green <andy@warmcat.com> @@ -69,6 +68,7 @@ Marko Räihä Michael John Wensley <michael@wensley.org.uk> Milan Bouchet-Valat <nalimilan@club.fr> Nathan Evans <evans@net.in.tum.de> +ng0 <ng0@infotropique.org> [ Documentation export to Texinfo ] Paul Ruth <ruth@cs.purdue.edu> Philipp Tölke <toelke@in.tum.de>, <pt@philipptoelke.de> Renaldo Ferreira <rf@cs.purdue.edu> @@ -51,7 +51,7 @@ These are the direct dependencies for running GNUnet: - Texinfo >= 5.2 - libglpk >= 4.45 (optional for experimental code) -Recommended autotools for compiling the SVN version are: +Recommended autotools for compiling the git version are: - autoconf >= 2.59 - automake >= 1.11.1 - libtool >= 2.2 @@ -135,7 +135,7 @@ install' process with SUDO rights, the libraries will be installed to "$GNUNET_PREFIX" and you will have to move them to "/lib/" manually. -Finally, if you are compiling the code from subversion, you have to +Finally, if you are compiling the code from git, you have to run ". bootstrap" before ./configure. If you receive an error during the running of ". bootstrap" that looks like "macro `AM_PATH_GTK' not found in library", you may need to run aclocal by hand with the -I @@ -1,4 +1,9 @@ #!/bin/sh rm -rf libltdl +echo -n "checking for libtoolize / libtool... " +which glibtoolize || which libtoolize || which libtool || { + echo "*** No libtoolize (libtool) or libtool found, please install it ***" + exit 1 +} autoreconf -if contrib/pogen.sh diff --git a/configure.ac b/configure.ac index 206a407fa1..9ab0a23163 100644 --- a/configure.ac +++ b/configure.ac @@ -248,6 +248,11 @@ then fi AC_DEFINE_UNQUOTED([NEED_LIBGCRYPT_VERSION], "$NEED_LIBGCRYPT_VERSION", [required libgcrypt version]) +# TODO: add check for VERSION +# TODO: add check for alternatives +# TODO: add switch to skip documentation building +AM_MISSING_PROG([MAKEINFO], [makeinfo]) + # Adam shostack suggests the following for Windows: # -D_FORTIFY_SOURCE=2 -fstack-protector-all AC_ARG_ENABLE(gcc-hardening, @@ -1547,6 +1552,7 @@ contrib/Makefile doc/Makefile doc/man/Makefile doc/doxygen/Makefile +doc/documentation/Makefile m4/Makefile po/Makefile.in src/Makefile @@ -1654,6 +1660,7 @@ src/vpn/vpn.conf src/zonemaster/Makefile src/zonemaster/zonemaster.conf src/rest/Makefile +src/identity-attribute/Makefile src/identity-provider/Makefile pkgconfig/Makefile pkgconfig/gnunetarm.pc diff --git a/contrib/Makefile.am b/contrib/Makefile.am index 07cff424c0..ac8b15188b 100644 --- a/contrib/Makefile.am +++ b/contrib/Makefile.am @@ -70,7 +70,20 @@ CLEANFILES = \ do_subst = $(SED) -e 's,[@]PYTHON[@],$(PYTHON),g' -%.py: %.py.in Makefile +# Use SUFFIX Extension rules, they are more portable for every +# implementation of 'make'. +# You'll also run into the "'%' is a GNU make extension warning" +# if you use this: +# +#%.py: %.py.in Makefile +# $(do_subst) < $< > $@ +# chmod +x $@ +# +# instead of this: + +SUFFIXES = .py.in .py + +.py.in.py: $(do_subst) < $< > $@ chmod +x $@ diff --git a/contrib/packages/guix/gnunet-doc.scm b/contrib/packages/guix/gnunet-doc.scm index 9974c1b516..8e775e8018 100644 --- a/contrib/packages/guix/gnunet-doc.scm +++ b/contrib/packages/guix/gnunet-doc.scm @@ -94,7 +94,7 @@ ("gnurl" ,gnurl) ("gstreamer" ,gstreamer) ("gst-plugins-base" ,gst-plugins-base) - ("gnutls" ,gnutls) ;Change to gnutls/dane once it is merged. + ("gnutls/dane" ,gnutls/dane) ("libextractor" ,libextractor) ("libgcrypt" ,libgcrypt) ("libidn" ,libidn) @@ -109,7 +109,7 @@ ("mysql" ,mysql) ("zlib" ,zlib) ("perl" ,perl) - ("python" ,python) ; tests and gnunet-qr + ("python-2" ,python-2) ; tests and gnunet-qr ("jansson" ,jansson) ("nss" ,nss) ("glib" ,glib "bin") @@ -126,6 +126,7 @@ ("gnu-gettext" ,gnu-gettext) ("graphviz" ,graphviz) ; dot ("texinfo-5" ,texinfo-5) ; Debian stable + ("which" ,which) ("libtool" ,libtool))) (arguments `(#:configure-flags @@ -142,11 +143,16 @@ (zero? (system* "sh" "bootstrap")))) (replace 'build (lambda _ - (chdir "doc") - (zero? (system* "make" "doc-all-give-me-the-noise")))) + (chdir "doc/documentation") + ;;(zero? (system* "make" "dev-build")))) + (zero? (system* "sh" "run-gendocs.sh")))) + ;; (zero? (system* "make" "pdf")) + ;; (zero? (system* "make" "html")) + ;; (zero? (system* "make" "info")))) + ;;(zero? (system* "make" "doc-all-give-me-the-noise")))) (replace 'install (lambda _ - (zero? (system* "make" "doc-all-install"))))))) + (zero? (system* "make" "doc-gendoc-install"))))))) ;;(lambda* (#:key outputs #:allow-other-keys) ;; (let* ((out (assoc-ref outputs "out")) ;; (doc (string-append out "/share/doc/gnunet"))) diff --git a/contrib/packages/guix/gnunet.scm b/contrib/packages/guix/gnunet.scm index a358fcecc1..d8eee18050 100644 --- a/contrib/packages/guix/gnunet.scm +++ b/contrib/packages/guix/gnunet.scm @@ -60,6 +60,7 @@ (gnu packages texinfo) (gnu packages tex) (gnu packages tls) + (gnu packages upnp) (gnu packages video) (gnu packages web) (gnu packages xiph) @@ -91,7 +92,7 @@ ("gnurl" ,gnurl) ("gstreamer" ,gstreamer) ("gst-plugins-base" ,gst-plugins-base) - ("gnutls" ,gnutls) ;Change to gnutls/dane once it is merged. + ("gnutls/dane" ,gnutls/dane) ;Change to gnutls/dane once it is merged. ("libextractor" ,libextractor) ("libgcrypt" ,libgcrypt) ("libidn" ,libidn) @@ -113,24 +114,25 @@ ("gmp" ,gmp) ("bluez" ,bluez) ; for optional bluetooth feature ("glib" ,glib) - ;; There are currently no binary substitutes for texlive on - ;; hydra.gnu.org or its mirrors due to its size. Uncomment if you need it. - ;;("texlive-minimal" ,texlive-minimal) ; optional. + ;; TODO: figure out the right texlive parts. + ;;("texlive-minimal" ,texlive-minimal) ("texlive" ,texlive) + ("miniupnpc" ,miniupnpc) ("libogg" ,libogg))) (native-inputs `(("pkg-config" ,pkg-config) ("autoconf" ,autoconf) ("automake" ,automake) ("gnu-gettext" ,gnu-gettext) + ("which" ,which) ("texinfo" ,texinfo-5) ; Debian stable: 5.2 ("libtool" ,libtool))) (outputs '("out" "debug")) (arguments `(#:configure-flags (list (string-append "--with-nssdir=" %output "/lib") - "--enable-gcc-hardening" - "--enable-linker-hardening" + ;;"--enable-gcc-hardening" + ;;"--enable-linker-hardening" "--enable-logging=verbose" "CFLAGS=-ggdb -O0") #:phases diff --git a/contrib/packages/guix/guix-env.scm b/contrib/packages/guix/guix-env.scm index c62a713a2b..da4a60b735 100644 --- a/contrib/packages/guix/guix-env.scm +++ b/contrib/packages/guix/guix-env.scm @@ -146,6 +146,7 @@ ("autoconf" ,autoconf) ("automake" ,automake) ("gnu-gettext" ,gnu-gettext) + ("which" ,which) ("texinfo" ,texinfo-5) ; Debian stable: 5.2 ("libtool" ,libtool))) ;; TODO: To make use of out:debug, which carries the symbols, diff --git a/contrib/packages/guix/packages/gnunet/packages/gnunet.scm b/contrib/packages/guix/packages/gnunet/packages/gnunet.scm index fbc132d78f..be529ec1de 100644 --- a/contrib/packages/guix/packages/gnunet/packages/gnunet.scm +++ b/contrib/packages/guix/packages/gnunet/packages/gnunet.scm @@ -26,6 +26,7 @@ #:use-module (gnu packages admin) #:use-module (gnu packages aidc) #:use-module (gnu packages autotools) + #:use-module (gnu packages base) #:use-module (gnu packages bison) #:use-module (gnu packages compression) #:use-module (gnu packages databases) @@ -60,6 +61,7 @@ ;; TODO: Use HEAD without checking sum of it. ;; Explanation for name scheme: UNIXPATH is capped at 108 characters, ;; this causes lots of tests to fail. +;; FIXME: make this file MUCH shorter. (define-public gnunetg (let* ((commit "3c3090717610ea787fdd3562901329254a6af0d6") (revision "32")) @@ -112,6 +114,7 @@ ("autoconf" ,autoconf) ("automake" ,automake) ("gnu-gettext" ,gnu-gettext) + ("which" ,which) ("texinfo" ,texinfo) ("libtool" ,libtool))) (outputs '("out" "debug")) @@ -199,6 +202,7 @@ (native-inputs `(("pkg-config" ,pkg-config) ("autoconf" ,autoconf) + ("which" ,which) ("automake" ,automake) ("gnu-gettext" ,gnu-gettext) ("texinfo" ,texinfo) diff --git a/doc/.gitignore b/doc/.gitignore index 25617d1b07..c56a90359f 100644 --- a/doc/.gitignore +++ b/doc/.gitignore @@ -5,14 +5,18 @@ *.toc *.cp *.cps -*.html *~ *.info +*.info-1 +*.info-2 +*.info-3 \#*\# version.texi -gnunet.info-1 -gnunet.info-2 -gnunet.info-3 mdate-sh stamp-vti texinfo.tex +gnunet.t2p/ +gnunet-c-tutorial.t2p/ +*.t2p/ +documentation/manuals +.\#*
\ No newline at end of file diff --git a/doc/Makefile.am b/doc/Makefile.am index 1a8bb64b9f..f84c667537 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -1,232 +1,7 @@ # This Makefile.am is in the public domain -SUBDIRS = man doxygen - -docdir = $(datadir)/doc/gnunet/ - -infoimagedir = $(infodir)/images - -#DOT_FILES = images/$(wildcard *.dot) - -#DOT_VECTOR_GRAPHICS = \ -# $(DOT_FILES:%.dot=%.eps) \ -# $(DOT_FILES:%.dot=%.pdf) - -dist_infoimage_DATA = \ - images/gnunet-gtk-0-10-gns-a-done.png \ - images/gnunet-gtk-0-10-gns-a.png \ - images/daemon_lego_block.png \ - images/gnunet-gtk-0-10-gns.png \ - images/gnunet-0-10-peerinfo.png \ - images/gnunet-gtk-0-10-identity.png \ - images/gnunet-fs-gtk-0-10-star-tab.png \ - images/gnunet-gtk-0-10.png \ - images/gnunet-gtk-0-10-download-area.png \ - images/gnunet-gtk-0-10-search-selected.png \ - images/gnunet-gtk-0-10-fs-menu.png \ - images/gnunet-gtk-0-10-traffic.png \ - images/gnunet-gtk-0-10-fs.png \ - images/gnunet-namestore-gtk-phone.png \ - images/gnunet-gtk-0-10-fs-publish-editing.png \ - images/gnunet-namestore-gtk-vpn.png \ - images/gnunet-gtk-0-10-fs-published.png \ - images/gnunet-setup-exit.png \ - images/gnunet-gtk-0-10-fs-publish.png \ - images/iceweasel-preferences.png \ - images/gnunet-gtk-0-10-fs-publish-select.png \ - images/iceweasel-proxy.png \ - images/gnunet-gtk-0-10-fs-publish-with-file_0.png \ - images/service_lego_block.png \ - images/gnunet-gtk-0-10-fs-publish-with-file.png \ - images/service_stack.png \ - images/gnunet-gtk-0-10-fs-search.png \ - images/gnunet-tutorial-service.png \ - images/gnunet-tutorial-system.png \ - images/daemon_lego_block.svg \ - images/lego_stack.svg \ - images/service_lego_block.svg \ - images/structure.dot - -# images/$(wildcard *.png) \ -# images/$(wildcard *.svg) -# $(DOT_FILES:%.dot=%.png) - -#DOT_OPTIONS = \ -# -Gratio=.9 -Gnodesep=.005 -Granksep=.00005 \ -# -Nfontsite=9 -Nheight=.1 -Nwidth=.1 - -# .dot.png: -# $(AM_V_DOT)$(DOT) -Tpng $(DOT_OPTIONS) < "$<" > "$(srcdir)/$@.tmp"; \ -# mv "$(srcdir)/$@.tmp" "$(srcdir)/$@" - -# .dot.pdf: -# $(AM_V_DOT)$(DOT) -Tpdf $(DOT_OPTIONS) < "$<" > "$(srcdir)/$@.tmp"; \ -# mv "$(srcdir)/$@.tmp" "$(srcdir)/$@" - -# .dot.eps: -# $(AM_V_DOT)$(DOT) -Teps $(DOT_OPTIONS) < "$<" > "$(srcdir)/$@.tmp"; \ -# mv "$(srcdir)/$@.tmp" "$(srcdir)/$@" - -# .png.eps: -# $(AM_V_GEN)convert "$<" "$@-tmp.eps"; \ -# mv "$@-tmp.eps" "$@" - -# pdf-local: $(DOT_FILES=%.dot=$(top_srcdir)/%.pdf) -# info-local: $(DOT_FILES=%.dot=$(top_srcdir)/%.png) -# ps-local: $(DOT_FILES=%.dot=$(top_srcdir)/%.eps) \ -# $(top_srcdir)/%D%/images/coreutils-size-map.eps -# dvi-local: ps-local - -gnunet_tutorial_examples = \ - 001.c \ - 002.c \ - 003.c \ - 004.c \ - 005.c \ - 006.c \ - 007.c \ - 008.c \ - 009.c \ - 010.c \ - 011.c \ - 012.c \ - 013.c \ - 014.c \ - 015.c \ - 016.c \ - 017.c \ - 018.c \ - 019.c \ - 020.c \ - 021.c \ - 022.c \ - 023.c \ - 024.c \ - 025.c \ - 026.c - -info_TEXINFOS = \ - gnunet.texi \ - gnunet-c-tutorial.texi - -gnunet_TEXINFOS = \ - chapters/developer.texi \ - chapters/installation.texi \ - chapters/philosophy.texi \ - chapters/user.texi \ - fdl-1.3.texi \ - gpl-3.0.texi +SUBDIRS = man doxygen documentation EXTRA_DIST = \ - $(gnunet_TEXINFOS) \ outdated-and-old-installation-instructions.txt \ gnunet-c-tutorial-v1.pdf \ - $(gnunet_tutorial_examples) \ README.txt - - -# $(DOT_FILES) \ -# $(DOT_VECTOR_GRAPHICS) - -DISTCLEANFILES = \ - gnunet.cps \ - gnunet-c-tutorial.cps \ - chapters/developer.cps \ - chapters/installation.cps \ - chapter/philosophy.cps \ - chapters/user.cps \ - fdl-1.3.cps \ - gpl-3.0.cps - -# if HAVE_EXTENDED_DOCUMENTATION_BUILDING -daemon_lego_block.png: images/daemon_lego_block.svg - convert images/daemon_lego_block.svg images/daemon_lego_block.png && - pngcrush images/daemon_lego_block.png images/daemon_lego_block.png - -service_lego_block.png: images/service_lego_block.svg - convert images/service_lego_block.svg images/service_lego_block.png && - pngcrush images/service_lego_block.png images/serivce_lego_block.png - -lego_stack.png: images/lego_stack.svg - convert images/lego_stack.svg images/lego_stack.png && - pngcrush images/lego_stack.png images/lego_stack.png - -version.texi: - echo "@set UPDATED $(date +'%d %B %Y')" > $@ - echo "@set UPDATED-MONTH $(date +'%B %Y')" >> $@ - echo "@set EDITION $(PACKAGE_VERSION)" >> $@ - echo "@set VERSION $(PACKAGE_VERSION)" >> $@ - -doc-pdf: version.texi - @makeinfo --pdf --quiet gnunet.texi -doc-pdf-tutorial: version.texi - @makeinfo --pdf --quiet gnunet-c-tutorial.texi - -doc-html: version.texi - @makeinfo --html gnunet.texi -doc-html-tutorial: version.texi - @makeinfo --html gnunet-c-tutorial.texi - -doc-info: version.texi - @makeinfo --no-split gnunet.texi -doc-info-tutorial: version.texi - @makeinfo --no-split gnunet-c-tutorial.texi - -# FIXME: rm *.html and *.pdf -doc-clean: - @rm *.aux *.log *.toc *.cp *.cps - -doc-all: doc-pdf doc-html doc-info doc-pdf-tutorial doc-html-tutorial doc-info-tutorial - -doc-pdf-noise: version.texi - @makeinfo --pdf gnunet.texi -doc-pdf-tutorial-noise: version.texi - @makeinfo --pdf gnunet-c-tutorial.texi - -doc-html-noise: version.texi - @makeinfo --html gnunet.texi -doc-html-tutorial-noise: version.texi - @makeinfo --html gnunet-c-tutorial.texi - -doc-info-noise: version.texi - @makeinfo --no-split gnunet.texi -doc-info-tutorial-noise: version.texi - @makeinfo --no-split gnunet-c-tutorial.texi - -doc-all-give-me-the-noise: doc-pdf-noise doc-html-noise doc-info-noise doc-pdf-tutorial-noise doc-html-tutorial-noise doc-info-tutorial-noise - -doc-all-install: doc-all-give-me-the-noise - @mkdir -p $(DESTDIR)/$(docdir) - @mkdir -p $(DESTDIR)/$(infoimagedir) - @mkdir -p $(DESTDIR)/$(infodir) - @install -m 0755 gnunet.pdf $(DESTDIR)/$(docdir) - @install -m 0755 gnunet-c-tutorial.pdf $(DESTDIR)/$(docdir) - @install -m 0755 gnunet-c-tutorial.info $(DESTDIR)/$(infodir) - @install -m 0755 gnunet.info $(DESTDIR)/$(infodir) - @cp -r gnunet $(DESTDIR)/$(docdir) - @cp -r gnunet-c-tutorial $(DESTDIR)/$(docdir) - -# @cp -r images $(DESTDIR)/$(infoimagedir) - -# TODO: Add more to clean. -# clean: -# @rm gnunet.pdf -# @rm gnunet-c-tutorial.pdf -# @rm gnunet.info -# @rm gnunet-c-tutorial.info - -# CLEANFILES = \ -# gnunet.log \ -# gnunet-c-tutorial.log \ -# $(wildcard *.aux) \ -# $(wildcard *.toc) \ -# $(wildcard *.cp) \ -# $(wildcard *.cps) - -.PHONY: version.texi -# if HAVE_EXTENDED_DOCUMENTATION_BUILDING_PDF - -# if HAVE_EXTENDED_DOCUMENTATION_BUILDING_HTML - -# endif -# endif -# endif diff --git a/doc/README.txt b/doc/README.txt deleted file mode 100644 index fbbc244244..0000000000 --- a/doc/README.txt +++ /dev/null @@ -1,42 +0,0 @@ -* What's left to do - -- Which Texlive modules are needed? Decrease the size. -- Update the content of gnunet documentation. - -* How to use (hack) on this - -** with guix - -export the environment variable GUIX_PACKAGE_PATH as $GUIX_PACKAGE_PATH:gnunet/contrib/packages/guix/packages -guix environment gnunet-doc - -** without guix - -You need to have Texinfo and Texlive in your path. -sh bootstrap -./configure --enable-documentation -cd doc -make doc-all-give-me-the-noise - -* structure (relations) - -** gnunet.texi - -> chapters/developer.texi - -> chapters/installation.texi - -> chapters/philosophy.texi - -> chapters/user.texi - -> images/* - -> gpl-3.0.texi - -> fdl-1.3.texi - -** gnunet-c-tutorial.texi - -> figs/Service.pdf - -> figs/System.pdf - -> tutorial-examples/*.c - -> gpl-3.0.texi - -> fdl-1.3.texi - -- gnunet-c-tutorial-v1.pdf: original LaTeX "gnunet-c-tutorial.pdf". -- man folder: the man pages. -- doxygen folder -- outdated-and-old-installation-instructions.txt: self described within the file. diff --git a/doc/chapters/developer.texi b/doc/chapters/developer.texi deleted file mode 100644 index f5493fd636..0000000000 --- a/doc/chapters/developer.texi +++ /dev/null @@ -1,7487 +0,0 @@ -@c *************************************************************************** -@node GNUnet Developer Handbook -@chapter GNUnet Developer Handbook - -This book is intended to be an introduction for programmers that want to -extend the GNUnet framework. GNUnet is more than a simple peer-to-peer -application. For developers, GNUnet is: - -@itemize @bullet -@item Free software under the GNU General Public License, with a community -that believes in the GNU philosophy -@item -A set of standards, including coding conventions and architectural rules -@item -A set of layered protocols, both specifying the communication between peers as -well as the communication between components of a single peer. -@item -A set of libraries with well-defined APIs suitable for writing extensions -@end itemize - -In particular, the architecture specifies that a peer consists of many -processes communicating via protocols. Processes can be written in almost -any language. C and Java APIs exist for accessing existing services and for -writing extensions. It is possible to write extensions in other languages by -implementing the necessary IPC protocols. - -GNUnet can be extended and improved along many possible dimensions, and anyone -interested in free software and freedom-enhancing networking is welcome to -join the effort. This developer handbook attempts to provide an initial -introduction to some of the key design choices and central components of the -system. This manual is far from complete, and we welcome informed -contributions, be it in the form of new chapters or insightful comments. - -However, the website is experiencing a constant onslaught of sophisticated -link-spam entered manually by exploited workers solving puzzles and -customizing text. To limit this commercial defacement, we are strictly -moderating comments and have disallowed "normal" users from posting new -content. However, this is really only intended to keep the spam at bay. If -you are a real user or aspiring developer, please drop us a note (IRC, e-mail, -contact form) with your user profile ID number included. We will then relax -these restrictions on your account. We're sorry for this inconvenience; -however, few people would want to read this site if 99% of it was -advertisements for bogus websites. - - - -@c *************************************************************************** - - - - - - - - -@menu -* Developer Introduction:: -* Code overview:: -* System Architecture:: -* Subsystem stability:: -* Naming conventions and coding style guide:: -* Build-system:: -* Developing extensions for GNUnet using the gnunet-ext template:: -* Writing testcases:: -* GNUnet's TESTING library:: -* Performance regression analysis with Gauger:: -* GNUnet's TESTBED Subsystem:: -* libgnunetutil:: -* The Automatic Restart Manager (ARM):: -* GNUnet's TRANSPORT Subsystem:: -* NAT library:: -* Distance-Vector plugin:: -* SMTP plugin:: -* Bluetooth plugin:: -* WLAN plugin:: -* The ATS Subsystem:: -* GNUnet's CORE Subsystem:: -* GNUnet's CADET subsystem:: -* GNUnet's NSE subsystem:: -* GNUnet's HOSTLIST subsystem:: -* GNUnet's IDENTITY subsystem:: -* GNUnet's NAMESTORE Subsystem:: -* GNUnet's PEERINFO subsystem:: -* GNUnet's PEERSTORE subsystem:: -* GNUnet's SET Subsystem:: -* GNUnet's STATISTICS subsystem:: -* GNUnet's Distributed Hash Table (DHT):: -* The GNU Name System (GNS):: -* The GNS Namecache:: -* The REVOCATION Subsystem:: -* GNUnet's File-sharing (FS) Subsystem:: -* GNUnet's REGEX Subsystem:: -@end menu - -@node Developer Introduction -@section Developer Introduction - -This developer handbook is intended as first introduction to GNUnet for new -developers that want to extend the GNUnet framework. After the introduction, -each of the GNUnet subsystems (directories in the @file{src/} tree) is (supposed to -be) covered in its own chapter. In addition to this documentation, GNUnet -developers should be aware of the services available on the GNUnet server to -them. - -New developers can have a look a the GNUnet tutorials for C and java available -in the @file{src/} directory of the repository or under the following links: - -@c ** FIXME: Link to files in source, not online. -@c ** FIXME: Where is the Java tutorial? -@itemize @bullet -@item @uref{https://gnunet.org/git/gnunet.git/plain/doc/gnunet-c-tutorial.pdf, GNUnet C tutorial} -@item GNUnet Java tutorial -@end itemize - -In addition to this book, the GNUnet server contains various resources for -GNUnet developers. They are all conveniently reachable via the "Developer" -entry in the navigation menu. Some additional tools (such as static analysis -reports) require a special developer access to perform certain operations. If -you feel you need access, you should contact -@uref{http://grothoff.org/christian/, Christian Grothoff}, GNUnet's maintainer. - -The public subsystems on the GNUnet server that help developers are: - -@itemize @bullet -@item The Version control system keeps our code and enables distributed -development. Only developers with write access can commit code, everyone else -is encouraged to submit patches to the -@uref{https://lists.gnu.org/mailman/listinfo/gnunet-developers, GNUnet-developers mailinglist}. -@item The GNUnet bugtracking system is used to track feature requests, open bug -reports and their resolutions. Anyone can report bugs, only developers can -claim to have fixed them. -@item A buildbot is used to check GNUnet builds automatically on a range of -platforms. Builds are triggered automatically after 30 minutes of no changes to -Git. -@item The current quality of our automated test suite is assessed using Code -coverage analysis. This analysis is run daily; however the webpage is only -updated if all automated tests pass at that time. Testcases that improve our -code coverage are always welcome. -@item We try to automatically find bugs using a static analysis scan. This scan -is run daily; however the webpage is only updated if all automated tests pass -at the time. Note that not everything that is flagged by the analysis is a bug, -sometimes even good code can be marked as possibly problematic. Nevertheless, -developers are encouraged to at least be aware of all issues in their code that -are listed. -@item We use Gauger for automatic performance regression visualization. Details -on how to use Gauger are here. -@item We use @uref{http://junit.org/, junit} to automatically test gnunet-java. -Automatically generated, current reports on the test suite are here. -@item We use Cobertura to generate test coverage reports for gnunet-java. -Current reports on test coverage are here. -@end itemize - - - -@c *************************************************************************** -@menu -* Project overview:: -@end menu - -@node Project overview -@subsection Project overview - -The GNUnet project consists at this point of several sub-projects. This section -is supposed to give an initial overview about the various sub-projects. Note -that this description also lists projects that are far from complete, including -even those that have literally not a single line of code in them yet. - -GNUnet sub-projects in order of likely relevance are currently: - -@table @asis - -@item gnunet Core of the P2P framework, including file-sharing, VPN and -chat applications; this is what the developer handbook covers mostly -@item gnunet-gtk Gtk+-based user interfaces, including gnunet-fs-gtk -(file-sharing), gnunet-statistics-gtk (statistics over time), -gnunet-peerinfo-gtk (information about current connections and known peers), -gnunet-chat-gtk (chat GUI) and gnunet-setup (setup tool for "everything") -@item gnunet-fuse Mounting directories shared via GNUnet's file-sharing on Linux -@item gnunet-update Installation and update tool -@item gnunet-ext Template for starting 'external' GNUnet projects -@item gnunet-java Java APIs for writing GNUnet services and applications -@c ** FIXME: Point to new website repository once we have it: -@c ** @item svn/gnunet-www/ Code and media helping drive the GNUnet website -@item eclectic Code to run -GNUnet nodes on testbeds for research, development, testing and evaluation -@c ** FIXME: Solve the status and location of gnunet-qt -@item gnunet-qt qt-based GNUnet GUI (dead?) -@item gnunet-cocoa cocoa-based GNUnet GUI (dead?) - -@end table - -We are also working on various supporting libraries and tools: -@c ** FIXME: What about gauger, and what about libmwmodem? - -@table @asis -@item libextractor GNU libextractor (meta data extraction) -@item libmicrohttpd GNU libmicrohttpd (embedded HTTP(S) server library) -@item gauger Tool for performance regression analysis -@item monkey Tool for automated debugging of distributed systems -@item libmwmodem Library for accessing satellite connection quality reports -@end table - -Finally, there are various external projects (see links for a list of those -that have a public website) which build on top of the GNUnet framework. - -@c *************************************************************************** -@node Code overview -@section Code overview - -This section gives a brief overview of the GNUnet source code. Specifically, we -sketch the function of each of the subdirectories in the @file{gnunet/src/} -directory. The order given is roughly bottom-up (in terms of the layers of the -system). -@table @asis - -@item util/ --- libgnunetutil Library with general utility functions, all -GNUnet binaries link against this library. Anything from memory allocation and -data structures to cryptography and inter-process communication. The goal is to -provide an OS-independent interface and more 'secure' or convenient -implementations of commonly used primitives. The API is spread over more than a -dozen headers, developers should study those closely to avoid duplicating -existing functions. -@item hello/ --- libgnunethello HELLO messages are used to -describe under which addresses a peer can be reached (for example, protocol, -IP, port). This library manages parsing and generating of HELLO messages. -@item block/ --- libgnunetblock The DHT and other components of GNUnet store -information in units called 'blocks'. Each block has a type and the type -defines a particular format and how that binary format is to be linked to a -hash code (the key for the DHT and for databases). The block library is a -wapper around block plugins which provide the necessary functions for each -block type. -@item statistics/ The statistics service enables associating -values (of type uint64_t) with a componenet name and a string. The main uses is -debugging (counting events), performance tracking and user entertainment (what -did my peer do today?). -@item arm/ The automatic-restart-manager (ARM) service -is the GNUnet master service. Its role is to start gnunet-services, to re-start -them when they crashed and finally to shut down the system when requested. -@item peerinfo/ The peerinfo service keeps track of which peers are known to -the local peer and also tracks the validated addresses for each peer (in the -form of a HELLO message) for each of those peers. The peer is not necessarily -connected to all peers known to the peerinfo service. Peerinfo provides -persistent storage for peer identities --- peers are not forgotten just because -of a system restart. -@item datacache/ --- libgnunetdatacache The datacache -library provides (temporary) block storage for the DHT. Existing plugins can -store blocks in Sqlite, Postgres or MySQL databases. All data stored in the -cache is lost when the peer is stopped or restarted (datacache uses temporary -tables). -@item datastore/ The datastore service stores file-sharing blocks in -databases for extended periods of time. In contrast to the datacache, data is -not lost when peers restart. However, quota restrictions may still cause old, -expired or low-priority data to be eventually discarded. Existing plugins can -store blocks in Sqlite, Postgres or MySQL databases. -@item template/ Template -for writing a new service. Does nothing. -@item ats/ The automatic transport -selection (ATS) service is responsible for deciding which address (i.e. which -transport plugin) should be used for communication with other peers, and at -what bandwidth. -@item nat/ --- libgnunetnat Library that provides basic -functions for NAT traversal. The library supports NAT traversal with manual -hole-punching by the user, UPnP and ICMP-based autonomous NAT traversal. The -library also includes an API for testing if the current configuration works and -the @code{gnunet-nat-server} which provides an external service to test the -local configuration. -@item fragmentation/ --- libgnunetfragmentation Some -transports (UDP and WLAN, mostly) have restrictions on the maximum transfer -unit (MTU) for packets. The fragmentation library can be used to break larger -packets into chunks of at most 1k and transmit the resulting fragments -reliabily (with acknowledgement, retransmission, timeouts, etc.). -@item transport/ The transport service is responsible for managing the basic P2P -communication. It uses plugins to support P2P communication over TCP, UDP, -HTTP, HTTPS and other protocols.The transport service validates peer addresses, -enforces bandwidth restrictions, limits the total number of connections and -enforces connectivity restrictions (i.e. friends-only). -@item peerinfo-tool/ -This directory contains the gnunet-peerinfo binary which can be used to inspect -the peers and HELLOs known to the peerinfo service. -@item core/ The core -service is responsible for establishing encrypted, authenticated connections -with other peers, encrypting and decrypting messages and forwarding messages to -higher-level services that are interested in them. -@item testing/ --- -libgnunettesting The testing library allows starting (and stopping) peers for -writing testcases.@ -It also supports automatic generation of configurations for -peers ensuring that the ports and paths are disjoint. libgnunettesting is also -the foundation for the testbed service -@item testbed/ The testbed service is -used for creating small or large scale deployments of GNUnet peers for -evaluation of protocols. It facilitates peer depolyments on multiple hosts (for -example, in a cluster) and establishing varous network topologies (both -underlay and overlay). -@item nse/ The network size estimation (NSE) service -implements a protocol for (securely) estimating the current size of the P2P -network. -@item dht/ The distributed hash table (DHT) service provides a -distributed implementation of a hash table to store blocks under hash keys in -the P2P network. -@item hostlist/ The hostlist service allows learning about -other peers in the network by downloading HELLO messages from an HTTP server, -can be configured to run such an HTTP server and also implements a P2P protocol -to advertise and automatically learn about other peers that offer a public -hostlist server. -@item topology/ The topology service is responsible for -maintaining the mesh topology. It tries to maintain connections to friends -(depending on the configuration) and also tries to ensure that the peer has a -decent number of active connections at all times. If necessary, new connections -are added. All peers should run the topology service, otherwise they may end up -not being connected to any other peer (unless some other service ensures that -core establishes the required connections). The topology service also tells the -transport service which connections are permitted (for friend-to-friend -networking) -@item fs/ The file-sharing (FS) service implements GNUnet's -file-sharing application. Both anonymous file-sharing (using gap) and -non-anonymous file-sharing (using dht) are supported. -@item cadet/ The CADET -service provides a general-purpose routing abstraction to create end-to-end -encrypted tunnels in mesh networks. We wrote a paper documenting key aspects of -the design. -@item tun/ --- libgnunettun Library for building IPv4, IPv6 -packets and creating checksums for UDP, TCP and ICMP packets. The header -defines C structs for common Internet packet formats and in particular structs -for interacting with TUN (virtual network) interfaces. -@item mysql/ --- -libgnunetmysql Library for creating and executing prepared MySQL statements and -to manage the connection to the MySQL database. Essentially a lightweight -wrapper for the interaction between GNUnet components and libmysqlclient. -@item dns/ Service that allows intercepting and modifying DNS requests of the -local machine. Currently used for IPv4-IPv6 protocol translation (DNS-ALG) as -implemented by "pt/" and for the GNUnet naming system. The service can also be -configured to offer an exit service for DNS traffic. -@item vpn/ The virtual -public network (VPN) service provides a virtual tunnel interface (VTUN) for IP -routing over GNUnet. Needs some other peers to run an "exit" service to work. -Can be activated using the "gnunet-vpn" tool or integrated with DNS using the -"pt" daemon. -@item exit/ Daemon to allow traffic from the VPN to exit this -peer to the Internet or to specific IP-based services of the local peer. -Currently, an exit service can only be restricted to IPv4 or IPv6, not to -specific ports and or IP address ranges. If this is not acceptable, additional -firewall rules must be added manually. exit currently only works for normal -UDP, TCP and ICMP traffic; DNS queries need to leave the system via a DNS -service. -@item pt/ protocol translation daemon. This daemon enables 4-to-6, -6-to-4, 4-over-6 or 6-over-4 transitions for the local system. It essentially -uses "DNS" to intercept DNS replies and then maps results to those offered by -the VPN, which then sends them using mesh to some daemon offering an -appropriate exit service. -@item identity/ Management of egos (alter egos) of a -user; identities are essentially named ECC private keys and used for zones in -the GNU name system and for namespaces in file-sharing, but might find other -uses later -@item revocation/ Key revocation service, can be used to revoke the -private key of an identity if it has been compromised -@item namecache/ Cache -for resolution results for the GNU name system; data is encrypted and can be -shared among users, loss of the data should ideally only result in a -performance degradation (persistence not required) -@item namestore/ Database -for the GNU name system with per-user private information, persistence required -@item gns/ GNU name system, a GNU approach to DNS and PKI. -@item dv/ A plugin -for distance-vector (DV)-based routing. DV consists of a service and a -transport plugin to provide peers with the illusion of a direct P2P connection -for connections that use multiple (typically up to 3) hops in the actual -underlay network. -@item regex/ Service for the (distributed) evaluation of -regular expressions. -@item scalarproduct/ The scalar product service offers an -API to perform a secure multiparty computation which calculates a scalar -product between two peers without exposing the private input vectors of the -peers to each other. -@item consensus/ The consensus service will allow a set -of peers to agree on a set of values via a distributed set union computation. -@item rest/ The rest API allows access to GNUnet services using RESTful -interaction. The services provide plugins that can exposed by the rest server. -@item experimentation/ The experimentation daemon coordinates distributed -experimentation to evaluate transport and ats properties -@end table - -@c *************************************************************************** -@node System Architecture -@section System Architecture - -GNUnet developers like legos. The blocks are indestructible, can be stacked -together to construct complex buildings and it is generally easy to swap one -block for a different one that has the same shape. GNUnet's architecture is -based on legos: - - - -This chapter documents the GNUnet lego system, also known as GNUnet's system -architecture. - -The most common GNUnet component is a service. Services offer an API (or -several, depending on what you count as "an API") which is implemented as a -library. The library communicates with the main process of the service using a -service-specific network protocol. The main process of the service typically -doesn't fully provide everything that is needed --- it has holes to be filled -by APIs to other services. - -A special kind of component in GNUnet are user interfaces and daemons. Like -services, they have holes to be filled by APIs of other services. Unlike -services, daemons do not implement their own network protocol and they have no -API: - -The GNUnet system provides a range of services, daemons and user interfaces, -which are then combined into a layered GNUnet instance (also known as a peer). - -Note that while it is generally possible to swap one service for another -compatible service, there is often only one implementation. However, during -development we often have a "new" version of a service in parallel with an -"old" version. While the "new" version is not working, developers working on -other parts of the service can continue their development by simply using the -"old" service. Alternative design ideas can also be easily investigated by -swapping out individual components. This is typically achieved by simply -changing the name of the "BINARY" in the respective configuration section. - -Key properties of GNUnet services are that they must be separate processes and -that they must protect themselves by applying tight error checking against the -network protocol they implement (thereby achieving a certain degree of -robustness). - -On the other hand, the APIs are implemented to tolerate failures of the -service, isolating their host process from errors by the service. If the -service process crashes, other services and daemons around it should not also -fail, but instead wait for the service process to be restarted by ARM. - - -@c *************************************************************************** -@node Subsystem stability -@section Subsystem stability - -This page documents the current stability of the various GNUnet subsystems. -Stability here describes the expected degree of compatibility with future -versions of GNUnet. For each subsystem we distinguish between compatibility on -the P2P network level (communication protocol between peers), the IPC level -(communication between the service and the service library) and the API level -(stability of the API). P2P compatibility is relevant in terms of which -applications are likely going to be able to communicate with future versions of -the network. IPC communication is relevant for the implementation of language -bindings that re-implement the IPC messages. Finally, API compatibility is -relevant to developers that hope to be able to avoid changes to applications -build on top of the APIs of the framework. - -The following table summarizes our current view of the stability of the -respective protocols or APIs: - -@multitable @columnfractions .20 .20 .20 .20 -@headitem Subsystem @tab P2P @tab IPC @tab C API -@item util @tab n/a @tab n/a @tab stable -@item arm @tab n/a @tab stable @tab stable -@item ats @tab n/a @tab unstable @tab testing -@item block @tab n/a @tab n/a @tab stable -@item cadet @tab testing @tab testing @tab testing -@item consensus @tab experimental @tab experimental @tab experimental -@item core @tab stable @tab stable @tab stable -@item datacache @tab n/a @tab n/a @tab stable -@item datastore @tab n/a @tab stable @tab stable -@item dht @tab stable @tab stable @tab stable -@item dns @tab stable @tab stable @tab stable -@item dv @tab testing @tab testing @tab n/a -@item exit @tab testing @tab n/a @tab n/a -@item fragmentation @tab stable @tab n/a @tab stable -@item fs @tab stable @tab stable @tab stable -@item gns @tab stable @tab stable @tab stable -@item hello @tab n/a @tab n/a @tab testing -@item hostlist @tab stable @tab stable @tab n/a -@item identity @tab stable @tab stable @tab n/a -@item multicast @tab experimental @tab experimental @tab experimental -@item mysql @tab stable @tab n/a @tab stable -@item namestore @tab n/a @tab stable @tab stable -@item nat @tab n/a @tab n/a @tab stable -@item nse @tab stable @tab stable @tab stable -@item peerinfo @tab n/a @tab stable @tab stable -@item psyc @tab experimental @tab experimental @tab experimental -@item pt @tab n/a @tab n/a @tab n/a -@item regex @tab stable @tab stable @tab stable -@item revocation @tab stable @tab stable @tab stable -@item social @tab experimental @tab experimental @tab experimental -@item statistics @tab n/a @tab stable @tab stable -@item testbed @tab n/a @tab testing @tab testing -@item testing @tab n/a @tab n/a @tab testing -@item topology @tab n/a @tab n/a @tab n/a -@item transport @tab stable @tab stable @tab stable -@item tun @tab n/a @tab n/a @tab stable -@item vpn @tab testing @tab n/a @tab n/a -@end multitable - -Here is a rough explanation of the values: - -@table @samp -@item stable -No incompatible changes are planned at this time; for IPC/APIs, if -there are incompatible changes, they will be minor and might only require -minimal changes to existing code; for P2P, changes will be avoided if at all -possible for the 0.10.x-series - -@item testing -No incompatible changes are -planned at this time, but the code is still known to be in flux; so while we -have no concrete plans, our expectation is that there will still be minor -modifications; for P2P, changes will likely be extensions that should not break -existing code - -@item unstable -Changes are planned and will happen; however, they -will not be totally radical and the result should still resemble what is there -now; nevertheless, anticipated changes will break protocol/API compatibility - -@item experimental -Changes are planned and the result may look nothing like -what the API/protocol looks like today - -@item unknown -Someone should think about where this subsystem headed - -@item n/a -This subsystem does not have an API/IPC-protocol/P2P-protocol -@end table - -@c *************************************************************************** -@node Naming conventions and coding style guide -@section Naming conventions and coding style guide - -Here you can find some rules to help you write code for GNUnet. - - - -@c *************************************************************************** -@menu -* Naming conventions:: -* Coding style:: -@end menu - -@node Naming conventions -@subsection Naming conventions - - -@c *************************************************************************** -@menu -* include files:: -* binaries:: -* logging:: -* configuration:: -* exported symbols:: -* private (library-internal) symbols (including structs and macros):: -* testcases:: -* performance tests:: -* src/ directories:: -@end menu - -@node include files -@subsubsection include files - -@itemize @bullet -@item _lib: library without need for a process -@item _service: library that needs a service process -@item _plugin: plugin definition -@item _protocol: structs used in network protocol -@item exceptions: -@itemize @bullet -@item gnunet_config.h --- generated -@item platform.h --- first included -@item plibc.h --- external library -@item gnunet_common.h --- fundamental routines -@item gnunet_directories.h --- generated -@item gettext.h --- external library -@end itemize -@end itemize - -@c *************************************************************************** -@node binaries -@subsubsection binaries - -@itemize @bullet -@item gnunet-service-xxx: service process (has listen socket) -@item gnunet-daemon-xxx: daemon process (no listen socket) -@item gnunet-helper-xxx[-yyy]: SUID helper for module xxx -@item gnunet-yyy: command-line tool for end-users -@item libgnunet_plugin_xxx_yyy.so: plugin for API xxx -@item libgnunetxxx.so: library for API xxx -@end itemize - -@c *************************************************************************** -@node logging -@subsubsection logging - -@itemize @bullet -@item services and daemons use their directory name in GNUNET_log_setup (i.e. -'core') and log using plain 'GNUNET_log'. -@item command-line tools use their full name in GNUNET_log_setup (i.e. -'gnunet-publish') and log using plain 'GNUNET_log'. -@item service access libraries log using 'GNUNET_log_from' and use -'DIRNAME-api' for the component (i.e. 'core-api') -@item pure libraries (without associated service) use 'GNUNET_log_from' with -the component set to their library name (without lib or '.so'), which should -also be their directory name (i.e. 'nat') -@item plugins should use 'GNUNET_log_from' with the directory name and the -plugin name combined to produce the component name (i.e. 'transport-tcp'). -@item logging should be unified per-file by defining a LOG macro with the -appropriate arguments, along these lines:@ #define LOG(kind,...) -GNUNET_log_from (kind, "example-api",__VA_ARGS__) -@end itemize - -@c *************************************************************************** -@node configuration -@subsubsection configuration - -@itemize @bullet -@item paths (that are substituted in all filenames) are in PATHS (have as few -as possible) -@item all options for a particular module (src/MODULE) are under [MODULE] -@item options for a plugin of a module are under [MODULE-PLUGINNAME] -@end itemize - -@c *************************************************************************** -@node exported symbols -@subsubsection exported symbols - -@itemize @bullet -@item must start with "GNUNET_modulename_" and be defined in "modulename.c" -@item exceptions: those defined in gnunet_common.h -@end itemize - -@c *************************************************************************** -@node private (library-internal) symbols (including structs and macros) -@subsubsection private (library-internal) symbols (including structs and macros) - -@itemize @bullet -@item must NOT start with any prefix -@item must not be exported in a way that linkers could use them or@ other -libraries might see them via headers; they must be either@ declared/defined in -C source files or in headers that are in@ the respective directory under -src/modulename/ and NEVER be@ declared in src/include/. -@end itemize - -@node testcases -@subsubsection testcases - -@itemize @bullet -@item must be called "test_module-under-test_case-description.c" -@item "case-description" maybe omitted if there is only one test -@end itemize - -@c *************************************************************************** -@node performance tests -@subsubsection performance tests - -@itemize @bullet -@item must be called "perf_module-under-test_case-description.c" -@item "case-description" maybe omitted if there is only one performance test -@item Must only be run if HAVE_BENCHMARKS is satisfied -@end itemize - -@c *************************************************************************** -@node src/ directories -@subsubsection src/ directories - -@itemize @bullet -@item gnunet-NAME: end-user applications (i.e., gnunet-search, gnunet-arm) -@item gnunet-service-NAME: service processes with accessor library (i.e., -gnunet-service-arm) -@item libgnunetNAME: accessor library (_service.h-header) or standalone library -(_lib.h-header) -@item gnunet-daemon-NAME: daemon process without accessor library (i.e., -gnunet-daemon-hostlist) and no GNUnet management port -@item libgnunet_plugin_DIR_NAME: loadable plugins (i.e., -libgnunet_plugin_transport_tcp) -@end itemize - -@c *************************************************************************** -@node Coding style -@subsection Coding style - -@itemize @bullet -@item GNU guidelines generally apply -@item Indentation is done with spaces, two per level, no tabs -@item C99 struct initialization is fine -@item declare only one variable per line, so@ - -@example -int i; int j; -@end example - -instead of - -@example -int i,j; -@end example - -This helps keep diffs small and forces developers to think precisely about the -type of every variable. Note that @code{char *} is different from @code{const -char*} and @code{int} is different from @code{unsigned int} or @code{uint32_t}. -Each variable type should be chosen with care. - -@item While @code{goto} should generally be avoided, having a @code{goto} to -the end of a function to a block of clean up statements (free, close, etc.) can -be acceptable. - -@item Conditions should be written with constants on the left (to avoid -accidental assignment) and with the 'true' target being either the 'error' case -or the significantly simpler continuation. For example:@ - -@example -if (0 != stat ("filename," &sbuf)) @{ error(); @} else @{ - /* handle normal case here */ -@} -@end example - - -instead of -@example -if (stat ("filename," &sbuf) == 0) @{ - /* handle normal case here */ -@} else @{ error(); @} -@end example - - -If possible, the error clause should be terminated with a 'return' (or 'goto' -to some cleanup routine) and in this case, the 'else' clause should be omitted: -@example -if (0 != stat ("filename," &sbuf)) @{ error(); return; @} -/* handle normal case here */ -@end example - - -This serves to avoid deep nesting. The 'constants on the left' rule applies to -all constants (including. @code{GNUNET_SCHEDULER_NO_TASK}), NULL, and enums). -With the two above rules (constants on left, errors in 'true' branch), there is -only one way to write most branches correctly. - -@item Combined assignments and tests are allowed if they do not hinder code -clarity. For example, one can write:@ - -@example -if (NULL == (value = lookup_function())) @{ error(); return; @} -@end example - - -@item Use @code{break} and @code{continue} wherever possible to avoid deep(er) -nesting. Thus, we would write:@ - -@example -next = head; while (NULL != (pos = next)) @{ next = pos->next; if (! -should_free (pos)) continue; GNUNET_CONTAINER_DLL_remove (head, tail, pos); -GNUNET_free (pos); @} -@end example - - -instead of -@example -next = head; while (NULL != (pos = next)) @{ next = -pos->next; if (should_free (pos)) @{ - /* unnecessary nesting! */ - GNUNET_CONTAINER_DLL_remove (head, tail, pos); GNUNET_free (pos); @} @} -@end example - - -@item We primarily use @code{for} and @code{while} loops. A @code{while} loop -is used if the method for advancing in the loop is not a straightforward -increment operation. In particular, we use:@ - -@example -next = head; -while (NULL != (pos = next)) -@{ - next = pos->next; - if (! should_free (pos)) - continue; - GNUNET_CONTAINER_DLL_remove (head, tail, pos); - GNUNET_free (pos); -@} -@end example - - -to free entries in a list (as the iteration changes the structure of the list -due to the free; the equivalent @code{for} loop does no longer follow the -simple @code{for} paradigm of @code{for(INIT;TEST;INC)}). However, for loops -that do follow the simple @code{for} paradigm we do use @code{for}, even if it -involves linked lists: -@example -/* simple iteration over a linked list */ -for (pos = head; NULL != pos; pos = pos->next) -@{ - use (pos); -@} -@end example - - -@item The first argument to all higher-order functions in GNUnet must be -declared to be of type @code{void *} and is reserved for a closure. We do not -use inner functions, as trampolines would conflict with setups that use -non-executable stacks.@ The first statement in a higher-order function, which -unusually should be part of the variable declarations, should assign the -@code{cls} argument to the precise expected type. For example: -@example -int callback (void *cls, char *args) @{ - struct Foo *foo = cls; int other_variables; - - /* rest of function */ -@} -@end example - - -@item It is good practice to write complex @code{if} expressions instead of -using deeply nested @code{if} statements. However, except for addition and -multiplication, all operators should use parens. This is fine:@ - -@example -if ( (1 == foo) || ((0 == bar) && (x != y)) ) - return x; -@end example - - -However, this is not: -@example -if (1 == foo) - return x; -if (0 == bar && x != y) - return x; -@end example - - -Note that splitting the @code{if} statement above is debateable as the -@code{return x} is a very trivial statement. However, once the logic after the -branch becomes more complicated (and is still identical), the "or" formulation -should be used for sure. - -@item There should be two empty lines between the end of the function and the -comments describing the following function. There should be a single empty line -after the initial variable declarations of a function. If a function has no -local variables, there should be no initial empty line. If a long function -consists of several complex steps, those steps might be separated by an empty -line (possibly followed by a comment describing the following step). The code -should not contain empty lines in arbitrary places; if in doubt, it is likely -better to NOT have an empty line (this way, more code will fit on the screen). -@end itemize - -@c *************************************************************************** -@node Build-system -@section Build-system - -If you have code that is likely not to compile or build rules you might want to -not trigger for most developers, use "if HAVE_EXPERIMENTAL" in your -Makefile.am. Then it is OK to (temporarily) add non-compiling (or -known-to-not-port) code. - -If you want to compile all testcases but NOT run them, run configure with the@ -@code{--enable-test-suppression} option. - -If you want to run all testcases, including those that take a while, run -configure with the@ @code{--enable-expensive-testcases} option. - -If you want to compile and run benchmarks, run configure with the@ -@code{--enable-benchmarks} option. - -If you want to obtain code coverage results, run configure with the@ -@code{--enable-coverage} option and run the coverage.sh script in contrib/. - -@c *************************************************************************** -@node Developing extensions for GNUnet using the gnunet-ext template -@section Developing extensions for GNUnet using the gnunet-ext template - - -For developers who want to write extensions for GNUnet we provide the -gnunet-ext template to provide an easy to use skeleton. - -gnunet-ext contains the build environment and template files for the -development of GNUnet services, command line tools, APIs and tests. - -First of all you have to obtain gnunet-ext from git: - -@code{git clone https://gnunet.org/git/gnunet-ext.git} - -The next step is to bootstrap and configure it. For configure you have to -provide the path containing GNUnet with @code{--with-gnunet=/path/to/gnunet} -and the prefix where you want the install the extension using -@code{--prefix=/path/to/install}@ @code{@ ./bootstrap@ ./configure ---prefix=/path/to/install --with-gnunet=/path/to/gnunet@ } - -When your GNUnet installation is not included in the default linker search -path, you have to add @code{/path/to/gnunet} to the file @code{/etc/ld.so.conf} -and run @code{ldconfig} or your add it to the environmental variable -@code{LD_LIBRARY_PATH} by using - -@code{export LD_LIBRARY_PATH=/path/to/gnunet/lib} - -@c *************************************************************************** -@node Writing testcases -@section Writing testcases - -Ideally, any non-trivial GNUnet code should be covered by automated testcases. -Testcases should reside in the same place as the code that is being tested. The -name of source files implementing tests should begin with "test_" followed by -the name of the file that contains the code that is being tested. - -Testcases in GNUnet should be integrated with the autotools build system. This -way, developers and anyone building binary packages will be able to run all -testcases simply by running @code{make check}. The final testcases shipped with -the distribution should output at most some brief progress information and not -display debug messages by default. The success or failure of a testcase must be -indicated by returning zero (success) or non-zero (failure) from the main -method of the testcase. The integration with the autotools is relatively -straightforward and only requires modifications to the @code{Makefile.am} in -the directory containing the testcase. For a testcase testing the code in -@code{foo.c} the @code{Makefile.am} would contain the following lines: -@example -check_PROGRAMS = test_foo TESTS = $(check_PROGRAMS) test_foo_SOURCES = -test_foo.c test_foo_LDADD = $(top_builddir)/src/util/libgnunetutil.la -@end example - -Naturally, other libraries used by the testcase may be specified in the -@code{LDADD} directive as necessary. - -Often testcases depend on additional input files, such as a configuration file. -These support files have to be listed using the EXTRA_DIST directive in order -to ensure that they are included in the distribution. Example: -@example -EXTRA_DIST = test_foo_data.conf -@end example - - -Executing @code{make check} will run all testcases in the current directory and -all subdirectories. Testcases can be compiled individually by running -@code{make test_foo} and then invoked directly using @code{./test_foo}. Note -that due to the use of plugins in GNUnet, it is typically necessary to run -@code{make install} before running any testcases. Thus the canonical command -@code{make check install} has to be changed to @code{make install check} for -GNUnet. - -@c *************************************************************************** -@node GNUnet's TESTING library -@section GNUnet's TESTING library - -The TESTING library is used for writing testcases which involve starting a -single or multiple peers. While peers can also be started by testcases using -the ARM subsystem, using TESTING library provides an elegant way to do this. -The configurations of the peers are auto-generated from a given template to -have non-conflicting port numbers ensuring that peers' services do not run into -bind errors. This is achieved by testing ports' availability by binding a -listening socket to them before allocating them to services in the generated -configurations. - -An another advantage while using TESTING is that it shortens the testcase -startup time as the hostkeys for peers are copied from a pre-computed set of -hostkeys instead of generating them at peer startup which may take a -considerable amount of time when starting multiple peers or on an embedded -processor. - -TESTING also allows for certain services to be shared among peers. This feature -is invaluable when testing with multiple peers as it helps to reduce the number -of services run per each peer and hence the total number of processes run per -testcase. - -TESTING library only handles creating, starting and stopping peers. Features -useful for testcases such as connecting peers in a topology are not available -in TESTING but are available in the TESTBED subsystem. Furthermore, TESTING -only creates peers on the localhost, however by using TESTBED testcases can -benefit from creating peers across multiple hosts. - -@menu -* API:: -* Finer control over peer stop:: -* Helper functions:: -* Testing with multiple processes:: -@end menu - -@c *************************************************************************** -@node API -@subsection API - -TESTING abstracts a group of peers as a TESTING system. All peers in a system -have common hostname and no two services of these peers have a same port or a -UNIX domain socket path. - -TESTING system can be created with the function -@code{GNUNET_TESTING_system_create()} which returns a handle to the system. -This function takes a directory path which is used for generating the -configurations of peers, an IP address from which connections to the peers' -services should be allowed, the hostname to be used in peers' configuration, -and an array of shared service specifications of type @code{struct -GNUNET_TESTING_SharedService}. - -The shared service specification must specify the name of the service to share, -the configuration pertaining to that shared service and the maximum number of -peers that are allowed to share a single instance of the shared service. - -TESTING system created with @code{GNUNET_TESTING_system_create()} chooses ports -from the default range 12000 - 56000 while auto-generating configurations for -peers. This range can be customised with the function -@code{GNUNET_TESTING_system_create_with_portrange()}. This function is similar -to @code{GNUNET_TESTING_system_create()} except that it take 2 additional -parameters --- the start and end of the port range to use. - -A TESTING system is destroyed with the funciton -@code{GNUNET_TESTING_system_destory()}. This function takes the handle of the -system and a flag to remove the files created in the directory used to generate -configurations. - -A peer is created with the function @code{GNUNET_TESTING_peer_configure()}. -This functions takes the system handle, a configuration template from which the -configuration for the peer is auto-generated and the index from where the -hostkey for the peer has to be copied from. When successfull, this function -returs a handle to the peer which can be used to start and stop it and to -obtain the identity of the peer. If unsuccessful, a NULL pointer is returned -with an error message. This function handles the generated configuration to -have non-conflicting ports and paths. - -Peers can be started and stopped by calling the functions -@code{GNUNET_TESTING_peer_start()} and @code{GNUNET_TESTING_peer_stop()} -respectively. A peer can be destroyed by calling the function -@code{GNUNET_TESTING_peer_destroy}. When a peer is destroyed, the ports and -paths in allocated in its configuration are reclaimed for usage in new -peers. - -@c *************************************************************************** -@node Finer control over peer stop -@subsection Finer control over peer stop - -Using @code{GNUNET_TESTING_peer_stop()} is normally fine for testcases. -However, calling this function for each peer is inefficient when trying to -shutdown multiple peers as this function sends the termination signal to the -given peer process and waits for it to terminate. It would be faster in this -case to send the termination signals to the peers first and then wait on them. -This is accomplished by the functions @code{GNUNET_TESTING_peer_kill()} which -sends a termination signal to the peer, and the function -@code{GNUNET_TESTING_peer_wait()} which waits on the peer. - -Further finer control can be achieved by choosing to stop a peer asynchronously -with the function @code{GNUNET_TESTING_peer_stop_async()}. This function takes -a callback parameter and a closure for it in addition to the handle to the peer -to stop. The callback function is called with the given closure when the peer -is stopped. Using this function eliminates blocking while waiting for the peer -to terminate. - -An asynchronous peer stop can be cancelled by calling the function -@code{GNUNET_TESTING_peer_stop_async_cancel()}. Note that calling this function -does not prevent the peer from terminating if the termination signal has -already been sent to it. It does, however, cancels the callback to be called -when the peer is stopped. - -@c *************************************************************************** -@node Helper functions -@subsection Helper functions - -Most of the testcases can benefit from an abstraction which configures a peer -and starts it. This is provided by the function -@code{GNUNET_TESTING_peer_run()}. This function takes the testing directory -pathname, a configuration template, a callback and its closure. This function -creates a peer in the given testing directory by using the configuration -template, starts the peer and calls the given callback with the given closure. - -The function @code{GNUNET_TESTING_peer_run()} starts the ARM service of the -peer which starts the rest of the configured services. A similar function -@code{GNUNET_TESTING_service_run} can be used to just start a single service of -a peer. In this case, the peer's ARM service is not started; instead, only the -given service is run. - -@c *************************************************************************** -@node Testing with multiple processes -@subsection Testing with multiple processes - -When testing GNUnet, the splitting of the code into a services and clients -often complicates testing. The solution to this is to have the testcase fork -@code{gnunet-service-arm}, ask it to start the required server and daemon -processes and then execute appropriate client actions (to test the client APIs -or the core module or both). If necessary, multiple ARM services can be forked -using different ports (!) to simulate a network. However, most of the time only -one ARM process is needed. Note that on exit, the testcase should shutdown ARM -with a @code{TERM} signal (to give it the chance to cleanly stop its child -processes). - -The following code illustrates spawning and killing an ARM process from a -testcase: -@example -static void run (void *cls, char *const *args, const char -*cfgfile, const struct GNUNET_CONFIGURATION_Handle *cfg) @{ struct -GNUNET_OS_Process *arm_pid; arm_pid = GNUNET_OS_start_process (NULL, NULL, -"gnunet-service-arm", "gnunet-service-arm", "-c", cfgname, NULL); - /* do real test work here */ - if (0 != GNUNET_OS_process_kill (arm_pid, SIGTERM)) GNUNET_log_strerror - (GNUNET_ERROR_TYPE_WARNING, "kill"); GNUNET_assert (GNUNET_OK == - GNUNET_OS_process_wait (arm_pid)); GNUNET_OS_process_close (arm_pid); @} - -GNUNET_PROGRAM_run (argc, argv, "NAME-OF-TEST", "nohelp", options, &run, cls); -@end example - - -An alternative way that works well to test plugins is to implement a -mock-version of the environment that the plugin expects and then to simply load -the plugin directly. - -@c *************************************************************************** -@node Performance regression analysis with Gauger -@section Performance regression analysis with Gauger - -To help avoid performance regressions, GNUnet uses Gauger. Gauger is a simple -logging tool that allows remote hosts to send performance data to a central -server, where this data can be analyzed and visualized. Gauger shows graphs of -the repository revisions and the performace data recorded for each revision, so -sudden performance peaks or drops can be identified and linked to a specific -revision number. - -In the case of GNUnet, the buildbots log the performance data obtained during -the tests after each build. The data can be accesed on GNUnet's Gauger page. - -The menu on the left allows to select either the results of just one build bot -(under "Hosts") or review the data from all hosts for a given test result -(under "Metrics"). In case of very different absolute value of the results, for -instance arm vs. amd64 machines, the option "Normalize" on a metric view can -help to get an idea about the performance evolution across all hosts. - -Using Gauger in GNUnet and having the performance of a module tracked over time -is very easy. First of course, the testcase must generate some consistent -metric, which makes sense to have logged. Highly volatile or random dependant -metrics probably are not ideal candidates for meaningful regression detection. - -To start logging any value, just include @code{gauger.h} in your testcase code. -Then, use the macro @code{GAUGER()} to make the buildbots log whatever value is -of interest for you to @code{gnunet.org}'s Gauger server. No setup is necessary -as most buildbots have already everything in place and new metrics are created -on demand. To delete a metric, you need to contact a member of the GNUnet -development team (a file will need to be removed manually from the respective -directory). - -The code in the test should look like this: -@example -[other includes] -#include <gauger.h> - -int main (int argc, char *argv[]) @{ - - [run test, generate data] GAUGER("YOUR_MODULE", "METRIC_NAME", (float)value, - "UNIT"); @} -@end example - - -Where: -@table @asis - -@item @strong{YOUR_MODULE} is a category in the gauger page and should be the -name of the module or subsystem like "Core" or "DHT" -@item @strong{METRIC} is -the name of the metric being collected and should be concise and descriptive, -like "PUT operations in sqlite-datastore". -@item @strong{value} is the value -of the metric that is logged for this run. -@item @strong{UNIT} is the unit in -which the value is measured, for instance "kb/s" or "kb of RAM/node". -@end table - -If you wish to use Gauger for your own project, you can grab a copy of the -latest stable release or check out Gauger's Subversion repository. - -@c *************************************************************************** -@node GNUnet's TESTBED Subsystem -@section GNUnet's TESTBED Subsystem - -The TESTBED subsystem facilitates testing and measuring of multi-peer -deployments on a single host or over multiple hosts. - -The architecture of the testbed module is divided into the following: -@itemize @bullet - -@item Testbed API: An API which is used by the testing driver programs. It -provides with functions for creating, destroying, starting, stopping peers, -etc. - -@item Testbed service (controller): A service which is started through the -Testbed API. This service handles operations to create, destroy, start, stop -peers, connect them, modify their configurations. - -@item Testbed helper: When a controller has to be started on a host, the -testbed API starts the testbed helper on that host which in turn starts the -controller. The testbed helper receives a configuration for the controller -through its stdin and changes it to ensure the controller doesn't run into any -port conflict on that host. -@end itemize - - -The testbed service (controller) is different from the other GNUnet services in -that it is not started by ARM and is not supposed to be run as a daemon. It is -started by the testbed API through a testbed helper. In a typical scenario -involving multiple hosts, a controller is started on each host. Controllers -take up the actual task of creating peers, starting and stopping them on the -hosts they run. - -While running deployments on a single localhost the testbed API starts the -testbed helper directly as a child process. When running deployments on remote -hosts the testbed API starts Testbed Helpers on each remote host through remote -shell. By default testbed API uses SSH as a remote shell. This can be changed -by setting the environmental variable GNUNET_TESTBED_RSH_CMD to the required -remote shell program. This variable can also contain parameters which are to be -passed to the remote shell program. For e.g:@ @code{@ export -GNUNET_TESTBED_RSH_CMD="ssh -o BatchMode=yes -o -NoHostAuthenticationForLocalhost=yes %h"@ }@ Substitutions are allowed int the -above command string also allows for substitions. through placemarks which -begin with a `%'. At present the following substitutions are supported -@itemize @bullet -@item -%h: hostname -@item -%u: username -@item -%p: port -@end itemize - -Note that the substitution placemark is replaced only when the corresponding -field is available and only once. Specifying @code{%u@@%h} doesn't work either. -If you want to user username substitutions for SSH use the argument @code{-l} -before the username substitution. Ex: @code{ssh -l %u -p %p %h} - -The testbed API and the helper communicate through the helpers stdin and -stdout. As the helper is started through a remote shell on remote hosts any -output messages from the remote shell interfere with the communication and -results in a failure while starting the helper. For this reason, it is -suggested to use flags to make the remote shells produce no output messages and -to have password-less logins. The default remote shell, SSH, the default -options are "-o BatchMode=yes -o NoHostBasedAuthenticationForLocalhost=yes". -Password-less logins should be ensured by using SSH keys. - -Since the testbed API executes the remote shell as a non-interactive shell, -certain scripts like .bashrc, .profiler may not be executed. If this is the -case testbed API can be forced to execute an interactive shell by setting up -the environmental variable `GNUNET_TESTBED_RSH_CMD_SUFFIX' to a shell program. -An example could be:@ @code{@ export GNUNET_TESTBED_RSH_CMD_SUFFIX="sh -lc"@ }@ -The testbed API will then execute the remote shell program as: @code{ -$GNUNET_TESTBED_RSH_CMD -p $port $dest $GNUNET_TESTBED_RSH_CMD_SUFFIX -gnunet-helper-testbed } - -On some systems, problems may arise while starting testbed helpers if GNUnet is -installed into a custom location since the helper may not be found in the -standard path. This can be addressed by setting the variable -`HELPER_BINARY_PATH' to the path of the testbed helper. Testbed API will then -use this path to start helper binaries both locally and remotely. - -Testbed API can accessed by including "gnunet_testbed_service.h" file and -linking with -lgnunettestbed. - - - -@c *************************************************************************** -@menu -* Supported Topologies:: -* Hosts file format:: -* Topology file format:: -* Testbed Barriers:: -* Automatic large-scale deployment of GNUnet in the PlanetLab testbed:: -* TESTBED Caveats:: -@end menu - -@node Supported Topologies -@subsection Supported Topologies - -While testing multi-peer deployments, it is often needed that the peers are -connected in some topology. This requirement is addressed by the function -@code{GNUNET_TESTBED_overlay_connect()} which connects any given two peers in -the testbed. - -The API also provides a helper function -@code{GNUNET_TESTBED_overlay_configure_topology()} to connect a given set of -peers in any of the following supported topologies: -@itemize @bullet - -@item @code{GNUNET_TESTBED_TOPOLOGY_CLIQUE}: All peers are connected with each -other - -@item @code{GNUNET_TESTBED_TOPOLOGY_LINE}: Peers are connected to form a line - -@item @code{GNUNET_TESTBED_TOPOLOGY_RING}: Peers are connected to form a ring -topology - -@item @code{GNUNET_TESTBED_TOPOLOGY_2D_TORUS}: Peers are connected to form a 2 -dimensional torus topology. The number of peers may not be a perfect square, in -that case the resulting torus may not have the uniform poloidal and toroidal -lengths - -@item @code{GNUNET_TESTBED_TOPOLOGY_ERDOS_RENYI}: Topology is generated to form -a random graph. The number of links to be present should be given - -@item @code{GNUNET_TESTBED_TOPOLOGY_SMALL_WORLD}: Peers are connected to form a -2D Torus with some random links among them. The number of random links are to -be given - -@item @code{GNUNET_TESTBED_TOPOLOGY_SMALL_WORLD_RING}: Peers are connected to -form a ring with some random links among them. The number of random links are -to be given - -@item @code{GNUNET_TESTBED_TOPOLOGY_SCALE_FREE}: Connects peers in a topology -where peer connectivity follows power law - new peers are connected with high -probabililty to well connected peers. See Emergence of Scaling in Random -Networks. Science 286, 509-512, 1999. - -@item @code{GNUNET_TESTBED_TOPOLOGY_FROM_FILE}: The topology information is -loaded from a file. The path to the file has to be given. See Topology file -format for the format of this file. - -@item @code{GNUNET_TESTBED_TOPOLOGY_NONE}: No topology -@end itemize - - -The above supported topologies can be specified respectively by setting the -variable @code{OVERLAY_TOPOLOGY} to the following values in the configuration -passed to Testbed API functions @code{GNUNET_TESTBED_test_run()} and -@code{GNUNET_TESTBED_run()}: -@itemize @bullet -@item @code{CLIQUE} -@item @code{RING} -@item @code{LINE} -@item @code{2D_TORUS} -@item @code{RANDOM} -@item @code{SMALL_WORLD} -@item @code{SMALL_WORLD_RING} -@item @code{SCALE_FREE} -@item @code{FROM_FILE} -@item @code{NONE} -@end itemize - - -Topologies @code{RANDOM}, @code{SMALL_WORLD} and @code{SMALL_WORLD_RING} -require the option @code{OVERLAY_RANDOM_LINKS} to be set to the number of -random links to be generated in the configuration. The option will be ignored -for the rest of the topologies. - -Topology @code{SCALE_FREE} requires the options @code{SCALE_FREE_TOPOLOGY_CAP} -to be set to the maximum number of peers which can connect to a peer and -@code{SCALE_FREE_TOPOLOGY_M} to be set to how many peers a peer should be -atleast connected to. - -Similarly, the topology @code{FROM_FILE} requires the option -@code{OVERLAY_TOPOLOGY_FILE} to contain the path of the file containing the -topology information. This option is ignored for the rest of the topologies. -See Topology file format for the format of this file. - -@c *************************************************************************** -@node Hosts file format -@subsection Hosts file format - -The testbed API offers the function GNUNET_TESTBED_hosts_load_from_file() to -load from a given file details about the hosts which testbed can use for -deploying peers. This function is useful to keep the data about hosts separate -instead of hard coding them in code. - -Another helper function from testbed API, GNUNET_TESTBED_run() also takes a -hosts file name as its parameter. It uses the above function to populate the -hosts data structures and start controllers to deploy peers. - -These functions require the hosts file to be of the following format: -@itemize @bullet -@item Each line is interpreted to have details about a host -@item Host details should include the username to use for logging into the -host, the hostname of the host and the port number to use for the remote shell -program. All thee values should be given. -@item These details should be given in the following format: -@code{<username>@@<hostname>:<port>} -@end itemize - -Note that having canonical hostnames may cause problems while resolving the IP -addresses (See this bug). Hence it is advised to provide the hosts' IP -numerical addresses as hostnames whenever possible. - -@c *************************************************************************** -@node Topology file format -@subsection Topology file format - -A topology file describes how peers are to be connected. It should adhere to -the following format for testbed to parse it correctly. - -Each line should begin with the target peer id. This should be followed by a -colon(`:') and origin peer ids seperated by `|'. All spaces except for newline -characters are ignored. The API will then try to connect each origin peer to -the target peer. - -For example, the following file will result in 5 overlay connections: [2->1], -[3->1],[4->3], [0->3], [2->0]@ @code{@ 1:2|3@ 3:4| 0@ 0: 2@ } - -@c *************************************************************************** -@node Testbed Barriers -@subsection Testbed Barriers - -The testbed subsystem's barriers API facilitates coordination among the peers -run by the testbed and the experiment driver. The concept is similar to the -barrier synchronisation mechanism found in parallel programming or -multi-threading paradigms - a peer waits at a barrier upon reaching it until -the barrier is reached by a predefined number of peers. This predefined number -of peers required to cross a barrier is also called quorum. We say a peer has -reached a barrier if the peer is waiting for the barrier to be crossed. -Similarly a barrier is said to be reached if the required quorum of peers reach -the barrier. A barrier which is reached is deemed as crossed after all the -peers waiting on it are notified. - -The barriers API provides the following functions: -@itemize @bullet -@item @strong{@code{GNUNET_TESTBED_barrier_init()}:} function to initialse a -barrier in the experiment -@item @strong{@code{GNUNET_TESTBED_barrier_cancel()}:} function to cancel a -barrier which has been initialised before -@item @strong{@code{GNUNET_TESTBED_barrier_wait()}:} function to signal barrier -service that the caller has reached a barrier and is waiting for it to be -crossed -@item @strong{@code{GNUNET_TESTBED_barrier_wait_cancel()}:} function to stop -waiting for a barrier to be crossed -@end itemize - - -Among the above functions, the first two, namely -@code{GNUNET_TESTBED_barrier_init()} and @code{GNUNET_TESTBED_barrier_cancel()} -are used by experiment drivers. All barriers should be initialised by the -experiment driver by calling @code{GNUNET_TESTBED_barrier_init()}. This -function takes a name to identify the barrier, the quorum required for the -barrier to be crossed and a notification callback for notifying the experiment -driver when the barrier is crossed. @code{GNUNET_TESTBED_barrier_cancel()} -cancels an initialised barrier and frees the resources allocated for it. This -function can be called upon a initialised barrier before it is crossed. - -The remaining two functions @code{GNUNET_TESTBED_barrier_wait()} and -@code{GNUNET_TESTBED_barrier_wait_cancel()} are used in the peer's processes. -@code{GNUNET_TESTBED_barrier_wait()} connects to the local barrier service -running on the same host the peer is running on and registers that the caller -has reached the barrier and is waiting for the barrier to be crossed. Note that -this function can only be used by peers which are started by testbed as this -function tries to access the local barrier service which is part of the testbed -controller service. Calling @code{GNUNET_TESTBED_barrier_wait()} on an -uninitialised barrier results in failure. -@code{GNUNET_TESTBED_barrier_wait_cancel()} cancels the notification registered -by @code{GNUNET_TESTBED_barrier_wait()}. - - -@c *************************************************************************** -@menu -* Implementation:: -@end menu - -@node Implementation -@subsubsection Implementation - -Since barriers involve coordination between experiment driver and peers, the -barrier service in the testbed controller is split into two components. The -first component responds to the message generated by the barrier API used by -the experiment driver (functions @code{GNUNET_TESTBED_barrier_init()} and -@code{GNUNET_TESTBED_barrier_cancel()}) and the second component to the -messages generated by barrier API used by peers (functions -@code{GNUNET_TESTBED_barrier_wait()} and -@code{GNUNET_TESTBED_barrier_wait_cancel()}). - -Calling @code{GNUNET_TESTBED_barrier_init()} sends a -@code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_INIT} message to the master -controller. The master controller then registers a barrier and calls -@code{GNUNET_TESTBED_barrier_init()} for each its subcontrollers. In this way -barrier initialisation is propagated to the controller hierarchy. While -propagating initialisation, any errors at a subcontroller such as timeout -during further propagation are reported up the hierarchy back to the experiment -driver. - -Similar to @code{GNUNET_TESTBED_barrier_init()}, -@code{GNUNET_TESTBED_barrier_cancel()} propagates -@code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_CANCEL} message which causes -controllers to remove an initialised barrier. - -The second component is implemented as a separate service in the binary -`gnunet-service-testbed' which already has the testbed controller service. -Although this deviates from the gnunet process architecture of having one -service per binary, it is needed in this case as this component needs access to -barrier data created by the first component. This component responds to -@code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_WAIT} messages from local peers when -they call @code{GNUNET_TESTBED_barrier_wait()}. Upon receiving -@code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_WAIT} message, the service checks if -the requested barrier has been initialised before and if it was not -initialised, an error status is sent through -@code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_STATUS} message to the local peer and -the connection from the peer is terminated. If the barrier is initialised -before, the barrier's counter for reached peers is incremented and a -notification is registered to notify the peer when the barrier is reached. The -connection from the peer is left open. - -When enough peers required to attain the quorum send -@code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_WAIT} messages, the controller sends -a @code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_STATUS} message to its parent -informing that the barrier is crossed. If the controller has started further -subcontrollers, it delays this message until it receives a similar notification -from each of those subcontrollers. Finally, the barriers API at the experiment -driver receives the @code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_STATUS} when the -barrier is reached at all the controllers. - -The barriers API at the experiment driver responds to the -@code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_STATUS} message by echoing it back to -the master controller and notifying the experiment controller through the -notification callback that a barrier has been crossed. The echoed -@code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_STATUS} message is propagated by the -master controller to the controller hierarchy. This propagation triggers the -notifications registered by peers at each of the controllers in the hierarchy. -Note the difference between this downward propagation of the -@code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_STATUS} message from its upward -propagation --- the upward propagation is needed for ensuring that the barrier -is reached by all the controllers and the downward propagation is for -triggering that the barrier is crossed. - -@c *************************************************************************** -@node Automatic large-scale deployment of GNUnet in the PlanetLab testbed -@subsection Automatic large-scale deployment of GNUnet in the PlanetLab testbed - -PlanetLab is as a testbed for computer networking and distributed systems -research. It was established in 2002 and as of June 2010 was composed of 1090 -nodes at 507 sites worldwide. - -To automate the GNUnet we created a set of automation tools to simplify the -large-scale deployment. We provide you a set of scripts you can use to deploy -GNUnet on a set of nodes and manage your installation. - -Please also check @uref{https://gnunet.org/installation-fedora8-svn} and@ -@uref{https://gnunet.org/installation-fedora12-svn} to find detailled -instructions how to install GNUnet on a PlanetLab node. - - -@c *************************************************************************** -@menu -* PlanetLab Automation for Fedora8 nodes:: -* Install buildslave on PlanetLab nodes running fedora core 8:: -* Setup a new PlanetLab testbed using GPLMT:: -* Why do i get an ssh error when using the regex profiler?:: -@end menu - -@node PlanetLab Automation for Fedora8 nodes -@subsubsection PlanetLab Automation for Fedora8 nodes - -@c *************************************************************************** -@node Install buildslave on PlanetLab nodes running fedora core 8 -@subsubsection Install buildslave on PlanetLab nodes running fedora core 8 -@c ** Actually this is a subsubsubsection, but must be fixed differently -@c ** as subsubsection is the lowest. - -Since most of the PlanetLab nodes are running the very old fedora core 8 image, -installing the buildslave software is quite some pain. For our PlanetLab -testbed we figured out how to install the buildslave software best. - -Install Distribute for python:@ @code{@ curl -http://python-distribute.org/distribute_setup.py | sudo python@ } - -Install Distribute for zope.interface <= 3.8.0 (4.0 and 4.0.1 will not work):@ -@code{@ wget -http://pypi.python.org/packages/source/z/zope.interface/zope.interface-3.8.0.tar.gz@ -tar zvfz zope.interface-3.8.0.tar.gz@ cd zope.interface-3.8.0@ sudo python -setup.py install@ } - -Install the buildslave software (0.8.6 was the latest version):@ @code{@ wget -http://buildbot.googlecode.com/files/buildbot-slave-0.8.6p1.tar.gz@ tar xvfz -buildbot-slave-0.8.6p1.tar.gz@ cd buildslave-0.8.6p1@ sudo python setup.py -install@ } - -The setup will download the matching twisted package and install it.@ It will -also try to install the latest version of zope.interface which will fail to -install. Buildslave will work anyway since version 3.8.0 was installed before! - -@c *************************************************************************** -@node Setup a new PlanetLab testbed using GPLMT -@subsubsection Setup a new PlanetLab testbed using GPLMT - -@itemize @bullet -@item Get a new slice and assign nodes -Ask your PlanetLab PI to give you a new slice and assign the nodes you need -@item Install a buildmaster -You can stick to the buildbot documentation:@ -@uref{http://buildbot.net/buildbot/docs/current/manual/installation.html} -@item Install the buildslave software on all nodes -To install the buildslave on all nodes assigned to your slice you can use the -tasklist @code{install_buildslave_fc8.xml} provided with GPLMT: - -@code{@ ./gplmt.py -c contrib/tumple_gnunet.conf -t -contrib/tasklists/install_buildslave_fc8.xml -a -p <planetlab password>@ } - -@item Create the buildmaster configuration and the slave setup commands - -The master and the and the slaves have need to have credentials and the master -has to have all nodes configured. This can be done with the -@code{create_buildbot_configuration.py} script in the @code{scripts} directory - -This scripts takes a list of nodes retrieved directly from PlanetLab or read -from a file and a configuration template and creates:@ - - a tasklist which can be executed with gplmt to setup the slaves@ - - a master.cfg file containing a PlanetLab nodes - -A configuration template is included in the <contrib>, most important is that -the script replaces the following tags in the template: - -%GPLMT_BUILDER_DEFINITION :@ GPLMT_BUILDER_SUMMARY@ GPLMT_SLAVES@ -%GPLMT_SCHEDULER_BUILDERS - -Create configuration for all nodes assigned to a slice:@ @code{@ -./create_buildbot_configuration.py -u <planetlab username> -p <planetlab -password> -s <slice> -m <buildmaster+port> -t <template>@ }@ Create -configuration for some nodes in a file:@ @code{@ -./create_buildbot_configuration.p -f <node_file> -m <buildmaster+port> -t -<template>@ } - -@item Copy the @code{master.cfg} to the buildmaster and start it -Use @code{buildbot start <basedir>} to start the server -@item Setup the buildslaves -@end itemize - -@c *************************************************************************** -@node Why do i get an ssh error when using the regex profiler? -@subsubsection Why do i get an ssh error when using the regex profiler? - -Why do i get an ssh error "Permission denied (publickey,password)." when using -the regex profiler although passwordless ssh to localhost works using publickey -and ssh-agent? - -You have to generate a public/private-key pair with no password:@ -@code{ssh-keygen -t rsa -b 4096 -f ~/.ssh/id_localhost}@ -and then add the following to your ~/.ssh/config file: - -@code{Host 127.0.0.1@ IdentityFile ~/.ssh/id_localhost} - -now make sure your hostsfile looks like@ - -[USERNAME]@@127.0.0.1:22@ -[USERNAME]@@127.0.0.1:22 - -You can test your setup by running `ssh 127.0.0.1` in a terminal and then in -the opened session run it again. If you were not asked for a password on either -login, then you should be good to go. - -@c *************************************************************************** -@node TESTBED Caveats -@subsection TESTBED Caveats - -This section documents a few caveats when using the GNUnet testbed -subsystem. - - -@c *************************************************************************** -@menu -* CORE must be started:: -* ATS must want the connections:: -@end menu - -@node CORE must be started -@subsubsection CORE must be started - -A simple issue is #3993: Your configuration MUST somehow ensure that for each -peer the CORE service is started when the peer is setup, otherwise TESTBED may -fail to connect peers when the topology is initialized, as TESTBED will start -some CORE services but not necessarily all (but it relies on all of them -running). The easiest way is to set 'FORCESTART = YES' in the '[core]' section -of the configuration file. Alternatively, having any service that directly or -indirectly depends on CORE being started with FORCESTART will also do. This -issue largely arises if users try to over-optimize by not starting any services -with FORCESTART. - -@c *************************************************************************** -@node ATS must want the connections -@subsubsection ATS must want the connections - -When TESTBED sets up connections, it only offers the respective HELLO -information to the TRANSPORT service. It is then up to the ATS service to -@strong{decide} to use the connection. The ATS service will typically eagerly -establish any connection if the number of total connections is low (relative to -bandwidth). Details may further depend on the specific ATS backend that was -configured. If ATS decides to NOT establish a connection (even though TESTBED -provided the required information), then that connection will count as failed -for TESTBED. Note that you can configure TESTBED to tolerate a certain number -of connection failures (see '-e' option of gnunet-testbed-profiler). This issue -largely arises for dense overlay topologies, especially if you try to create -cliques with more than 20 peers. - -@c *************************************************************************** -@node libgnunetutil -@section libgnunetutil - -libgnunetutil is the fundamental library that all GNUnet code builds upon. -Ideally, this library should contain most of the platform dependent code -(except for user interfaces and really special needs that only few applications -have). It is also supposed to offer basic services that most if not all GNUnet -binaries require. The code of libgnunetutil is in the src/util/ directory. The -public interface to the library is in the gnunet_util.h header. The functions -provided by libgnunetutil fall roughly into the following categories (in -roughly the order of importance for new developers): -@itemize @bullet -@item logging (common_logging.c) -@item memory allocation (common_allocation.c) -@item endianess conversion (common_endian.c) -@item internationalization (common_gettext.c) -@item String manipulation (string.c) -@item file access (disk.c) -@item buffered disk IO (bio.c) -@item time manipulation (time.c) -@item configuration parsing (configuration.c) -@item command-line handling (getopt*.c) -@item cryptography (crypto_*.c) -@item data structures (container_*.c) -@item CPS-style scheduling (scheduler.c) -@item Program initialization (program.c) -@item Networking (network.c, client.c, server*.c, service.c) -@item message queueing (mq.c) -@item bandwidth calculations (bandwidth.c) -@item Other OS-related (os*.c, plugin.c, signal.c) -@item Pseudonym management (pseudonym.c) -@end itemize - -It should be noted that only developers that fully understand this entire API -will be able to write good GNUnet code. - -Ideally, porting GNUnet should only require porting the gnunetutil library. -More testcases for the gnunetutil APIs are therefore a great way to make -porting of GNUnet easier. - -@menu -* Logging:: -* Interprocess communication API (IPC):: -* Cryptography API:: -* Message Queue API:: -* Service API:: -* Optimizing Memory Consumption of GNUnet's (Multi-) Hash Maps:: -* The CONTAINER_MDLL API:: -@end menu - -@c *************************************************************************** -@node Logging -@subsection Logging - -GNUnet is able to log its activity, mostly for the purposes of debugging the -program at various levels. - -@file{gnunet_common.h} defines several @strong{log levels}: -@table @asis - -@item ERROR for errors (really problematic situations, often leading to -crashes) -@item WARNING for warnings (troubling situations that might have -negative consequences, although not fatal) -@item INFO for various information. -Used somewhat rarely, as GNUnet statistics is used to hold and display most of -the information that users might find interesting. -@item DEBUG for debugging. -Does not produce much output on normal builds, but when extra logging is -enabled at compile time, a staggering amount of data is outputted under this -log level. -@end table - - -Normal builds of GNUnet (configured with @code{--enable-logging[=yes]}) are -supposed to log nothing under DEBUG level. The @code{--enable-logging=verbose} -configure option can be used to create a build with all logging enabled. -However, such build will produce large amounts of log data, which is -inconvenient when one tries to hunt down a specific problem. - -To mitigate this problem, GNUnet provides facilities to apply a filter to -reduce the logs: -@table @asis - -@item Logging by default When no log levels are configured in any other way -(see below), GNUnet will default to the WARNING log level. This mostly applies -to GNUnet command line utilities, services and daemons; tests will always set -log level to WARNING or, if @code{--enable-logging=verbose} was passed to -configure, to DEBUG. The default level is suggested for normal operation. -@item The -L option Most GNUnet executables accept an "-L loglevel" or -"--log=loglevel" option. If used, it makes the process set a global log level -to "loglevel". Thus it is possible to run some processes with -L DEBUG, for -example, and others with -L ERROR to enable specific settings to diagnose -problems with a particular process. -@item Configuration files. Because GNUnet -service and deamon processes are usually launched by gnunet-arm, it is not -possible to pass different custom command line options directly to every one of -them. The options passed to @code{gnunet-arm} only affect gnunet-arm and not -the rest of GNUnet. However, one can specify a configuration key "OPTIONS" in -the section that corresponds to a service or a daemon, and put a value of "-L -loglevel" there. This will make the respective service or daemon set its log -level to "loglevel" (as the value of OPTIONS will be passed as a command-line -argument). - -To specify the same log level for all services without creating separate -"OPTIONS" entries in the configuration for each one, the user can specify a -config key "GLOBAL_POSTFIX" in the [arm] section of the configuration file. The -value of GLOBAL_POSTFIX will be appended to all command lines used by the ARM -service to run other services. It can contain any option valid for all GNUnet -commands, thus in particular the "-L loglevel" option. The ARM service itself -is, however, unaffected by GLOBAL_POSTFIX; to set log level for it, one has to -specify "OPTIONS" key in the [arm] section. -@item Environment variables. -Setting global per-process log levels with "-L loglevel" does not offer -sufficient log filtering granularity, as one service will call interface -libraries and supporting libraries of other GNUnet services, potentially -producing lots of debug log messages from these libraries. Also, changing the -config file is not always convenient (especially when running the GNUnet test -suite).@ To fix that, and to allow GNUnet to use different log filtering at -runtime without re-compiling the whole source tree, the log calls were changed -to be configurable at run time. To configure them one has to define environment -variables "GNUNET_FORCE_LOGFILE", "GNUNET_LOG" and/or "GNUNET_FORCE_LOG": -@itemize @bullet - -@item "GNUNET_LOG" only affects the logging when no global log level is -configured by any other means (that is, the process does not explicitly set its -own log level, there are no "-L loglevel" options on command line or in -configuration files), and can be used to override the default WARNING log -level. - -@item "GNUNET_FORCE_LOG" will completely override any other log configuration -options given. - -@item "GNUNET_FORCE_LOGFILE" will completely override the location of the file -to log messages to. It should contain a relative or absolute file name. Setting -GNUNET_FORCE_LOGFILE is equivalent to passing "--log-file=logfile" or "-l -logfile" option (see below). It supports "[]" format in file names, but not -"@{@}" (see below). -@end itemize - - -Because environment variables are inherited by child processes when they are -launched, starting or re-starting the ARM service with these variables will -propagate them to all other services. - -"GNUNET_LOG" and "GNUNET_FORCE_LOG" variables must contain a specially -formatted @strong{logging definition} string, which looks like this:@ @code{@ -[component];[file];[function];[from_line[-to_line]];loglevel@emph{[/component...]}@ -}@ That is, a logging definition consists of definition entries, separated by -slashes ('/'). If only one entry is present, there is no need to add a slash -to its end (although it is not forbidden either).@ All definition fields -(component, file, function, lines and loglevel) are mandatory, but (except for -the loglevel) they can be empty. An empty field means "match anything". Note -that even if fields are empty, the semicolon (';') separators must be -present.@ The loglevel field is mandatory, and must contain one of the log -level names (ERROR, WARNING, INFO or DEBUG).@ The lines field might contain -one non-negative number, in which case it matches only one line, or a range -"from_line-to_line", in which case it matches any line in the interval -[from_line;to_line] (that is, including both start and end line).@ GNUnet -mostly defaults component name to the name of the service that is implemented -in a process ('transport', 'core', 'peerinfo', etc), but logging calls can -specify custom component names using @code{GNUNET_log_from}.@ File name and -function name are provided by the compiler (__FILE__ and __FUNCTION__ -built-ins). - -Component, file and function fields are interpreted as non-extended regular -expressions (GNU libc regex functions are used). Matching is case-sensitive, ^ -and $ will match the beginning and the end of the text. If a field is empty, -its contents are automatically replaced with a ".*" regular expression, which -matches anything. Matching is done in the default way, which means that the -expression matches as long as it's contained anywhere in the string. Thus -"GNUNET_" will match both "GNUNET_foo" and "BAR_GNUNET_BAZ". Use '^' and/or '$' -to make sure that the expression matches at the start and/or at the end of the -string.@ The semicolon (';') can't be escaped, and GNUnet will not use it in -component names (it can't be used in function names and file names anyway).@ - -@end table - - -Every logging call in GNUnet code will be (at run time) matched against the -log definitions passed to the process. If a log definition fields are matching -the call arguments, then the call log level is compared the the log level of -that definition. If the call log level is less or equal to the definition log -level, the call is allowed to proceed. Otherwise the logging call is -forbidden, and nothing is logged. If no definitions matched at all, GNUnet -will use the global log level or (if a global log level is not specified) will -default to WARNING (that is, it will allow the call to proceed, if its level -is less or equal to the global log level or to WARNING). - -That is, definitions are evaluated from left to right, and the first matching -definition is used to allow or deny the logging call. Thus it is advised to -place narrow definitions at the beginning of the logdef string, and generic -definitions - at the end. - -Whether a call is allowed or not is only decided the first time this particular -call is made. The evaluation result is then cached, so that any attempts to -make the same call later will be allowed or disallowed right away. Because of -that runtime log level evaluation should not significantly affect the process -performance.@ Log definition parsing is only done once, at the first call to -GNUNET_log_setup () made by the process (which is usually done soon after it -starts). - -At the moment of writing there is no way to specify logging definitions from -configuration files, only via environment variables. - -At the moment GNUnet will stop processing a log definition when it encounters -an error in definition formatting or an error in regular expression syntax, and -will not report the failure in any way. - - -@c *************************************************************************** -@menu -* Examples:: -* Log files:: -* Updated behavior of GNUNET_log:: -@end menu - -@node Examples -@subsubsection Examples - -@table @asis - -@item @code{GNUNET_FORCE_LOG=";;;;DEBUG" gnunet-arm -s} Start GNUnet process -tree, running all processes with DEBUG level (one should be careful with it, as -log files will grow at alarming rate!) -@item @code{GNUNET_FORCE_LOG="core;;;;DEBUG" gnunet-arm -s} Start GNUnet process -tree, running the core service under DEBUG level (everything else will use -configured or default level). -@item @code{GNUNET_FORCE_LOG=";gnunet-service-transport_validation.c;;;DEBUG" gnunet-arm -s} -Start GNUnet process tree, allowing any logging calls from -gnunet-service-transport_validation.c (everything else will use configured or -default level). -@item @code{GNUNET_FORCE_LOG="fs;gnunet-service-fs_push.c;;;DEBUG" gnunet-arm -s} -Start GNUnet process tree, allowing any logging calls from -gnunet-gnunet-service-fs_push.c (everything else will use configured or default -level). -@item @code{GNUNET_FORCE_LOG=";;GNUNET_NETWORK_socket_select;;DEBUG" gnunet-arm -s} -Start GNUnet process tree, allowing any logging calls from the -GNUNET_NETWORK_socket_select function (everything else will use configured or -default level). -@item @code{GNUNET_FORCE_LOG="transport.*;;.*send.*;;DEBUG/;;;;WARNING" gnunet-arm -s} -Start GNUnet process tree, allowing any logging calls from the components -that have "transport" in their names, and are made from function that have -"send" in their names. Everything else will be allowed to be logged only if it -has WARNING level. -@end table - - -On Windows, one can use batch files to run GNUnet processes with special -environment variables, without affecting the whole system. Such batch file will -look like this:@ @code{@ set GNUNET_FORCE_LOG=;;do_transmit;;DEBUG@ gnunet-arm --s@ }@ (note the absence of double quotes in the environment variable -definition, as opposed to earlier examples, which use the shell).@ Another -limitation, on Windows, GNUNET_FORCE_LOGFILE @strong{MUST} be set in order to -GNUNET_FORCE_LOG to work. - - -@c *************************************************************************** -@node Log files -@subsubsection Log files - -GNUnet can be told to log everything into a file instead of stderr (which is -the default) using the "--log-file=logfile" or "-l logfile" option. This option -can also be passed via command line, or from the "OPTION" and "GLOBAL_POSTFIX" -configuration keys (see above). The file name passed with this option is -subject to GNUnet filename expansion. If specified in "GLOBAL_POSTFIX", it is -also subject to ARM service filename expansion, in particular, it may contain -"@{@}" (left and right curly brace) sequence, which will be replaced by ARM -with the name of the service. This is used to keep logs from more than one -service separate, while only specifying one template containing "@{@}" in -GLOBAL_POSTFIX. - -As part of a secondary file name expansion, the first occurrence of "[]" -sequence ("left square brace" followed by "right square brace") in the file -name will be replaced with a process identifier or the process when it -initializes its logging subsystem. As a result, all processes will log into -different files. This is convenient for isolating messages of a particular -process, and prevents I/O races when multiple processes try to write into the -file at the same time. This expansion is done independently of "@{@}" -expansion that ARM service does (see above). - -The log file name that is specified via "-l" can contain format characters -from the 'strftime' function family. For example, "%Y" will be replaced with -the current year. Using "basename-%Y-%m-%d.log" would include the current -year, month and day in the log file. If a GNUnet process runs for long enough -to need more than one log file, it will eventually clean up old log files. -Currently, only the last three log files (plus the current log file) are -preserved. So once the fifth log file goes into use (so after 4 days if you -use "%Y-%m-%d" as above), the first log file will be automatically deleted. -Note that if your log file name only contains "%Y", then log files would be -kept for 4 years and the logs from the first year would be deleted once year 5 -begins. If you do not use any date-related string format codes, logs would -never be automatically deleted by GNUnet. - - -@c *************************************************************************** - -@node Updated behavior of GNUNET_log -@subsubsection Updated behavior of GNUNET_log - -It's currently quite common to see constructions like this all over the code: -@example -#if MESH_DEBUG GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "MESH: client -disconnected\n"); #endif -@end example - -The reason for the #if is not to avoid displaying the message when disabled -(GNUNET_ERROR_TYPE takes care of that), but to avoid the compiler including it -in the binary at all, when compiling GNUnet for platforms with restricted -storage space / memory (MIPS routers, ARM plug computers / dev boards, etc). - -This presents several problems: the code gets ugly, hard to write and it is -very easy to forget to include the #if guards, creating non-consistent code. A -new change in GNUNET_log aims to solve these problems. - -@strong{This change requires to @code{./configure} with at least -@code{--enable-logging=verbose} to see debug messages.} - -Here is an example of code with dense debug statements: -@example -switch (restrict_topology) @{ -case GNUNET_TESTING_TOPOLOGY_CLIQUE: #if VERBOSE_TESTING -GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, _("Blacklisting all but clique -topology\n")); #endif unblacklisted_connections = create_clique (pg, -&remove_connections, BLACKLIST, GNUNET_NO); break; case -GNUNET_TESTING_TOPOLOGY_SMALL_WORLD_RING: #if VERBOSE_TESTING GNUNET_log -(GNUNET_ERROR_TYPE_DEBUG, _("Blacklisting all but small world (ring) -topology\n")); #endif unblacklisted_connections = create_small_world_ring (pg, -&remove_connections, BLACKLIST); break; -@end example - - -Pretty hard to follow, huh? - -From now on, it is not necessary to include the #if / #endif statements to -acheive the same behavior. The GNUNET_log and GNUNET_log_from macros take care -of it for you, depending on the configure option: -@itemize @bullet -@item If @code{--enable-logging} is set to @code{no}, the binary will contain -no log messages at all. -@item If @code{--enable-logging} is set to @code{yes}, the binary will contain -no DEBUG messages, and therefore running with -L DEBUG will have no effect. -Other messages (ERROR, WARNING, INFO, etc) will be included. -@item If @code{--enable-logging} is set to @code{verbose}, or -@code{veryverbose} the binary will contain DEBUG messages (still, it will be -neccessary to run with -L DEBUG or set the DEBUG config option to show them). -@end itemize - - -If you are a developer: -@itemize @bullet -@item please make sure that you @code{./configure ---enable-logging=@{verbose,veryverbose@}}, so you can see DEBUG messages. -@item please remove the @code{#if} statements around @code{GNUNET_log -(GNUNET_ERROR_TYPE_DEBUG, ...)} lines, to improve the readibility of your code. -@end itemize - -Since now activating DEBUG automatically makes it VERBOSE and activates -@strong{all} debug messages by default, you probably want to use the -https://gnunet.org/logging functionality to filter only relevant messages. A -suitable configuration could be:@ @code{$ export -GNUNET_FORCE_LOG="^YOUR_SUBSYSTEM$;;;;DEBUG/;;;;WARNING"}@ Which will behave -almost like enabling DEBUG in that subsytem before the change. Of course you -can adapt it to your particular needs, this is only a quick example. - -@c *************************************************************************** -@node Interprocess communication API (IPC) -@subsection Interprocess communication API (IPC) - -In GNUnet a variety of new message types might be defined and used in -interprocess communication, in this tutorial we use the @code{struct -AddressLookupMessage} as a example to introduce how to construct our own -message type in GNUnet and how to implement the message communication between -service and client.@ (Here, a client uses the @code{struct -AddressLookupMessage} as a request to ask the server to return the address of -any other peer connecting to the service.) - - -@c *************************************************************************** -@menu -* Define new message types:: -* Define message struct:: -* Client - Establish connection:: -* Client - Initialize request message:: -* Client - Send request and receive response:: -* Server - Startup service:: -* Server - Add new handles for specified messages:: -* Server - Process request message:: -* Server - Response to client:: -* Server - Notification of clients:: -* Conversion between Network Byte Order (Big Endian) and Host Byte Order:: -@end menu - -@node Define new message types -@subsubsection Define new message types - -First of all, you should define the new message type in -@code{gnunet_protocols.h}: -@example - // Request to look addresses of peers in server. -#define GNUNET_MESSAGE_TYPE_TRANSPORT_ADDRESS_LOOKUP 29 - // Response to the address lookup request. -#define GNUNET_MESSAGE_TYPE_TRANSPORT_ADDRESS_REPLY 30 -@end example - -@c *************************************************************************** -@node Define message struct -@subsubsection Define message struct - -After the type definition, the specified message structure should also be -described in the header file, e.g. transport.h in our case. -@example -GNUNET_NETWORK_STRUCT_BEGIN - -struct AddressLookupMessage @{ struct GNUNET_MessageHeader header; int32_t -numeric_only GNUNET_PACKED; struct GNUNET_TIME_AbsoluteNBO timeout; uint32_t -addrlen GNUNET_PACKED; - /* followed by 'addrlen' bytes of the actual address, then - followed by the 0-terminated name of the transport */ @}; - GNUNET_NETWORK_STRUCT_END -@end example - - -Please note @code{GNUNET_NETWORK_STRUCT_BEGIN} and @code{GNUNET_PACKED} which -both ensure correct alignment when sending structs over the network - -@menu -@end menu - -@c *************************************************************************** -@node Client - Establish connection -@subsubsection Client - Establish connection -@c %**end of header - - -At first, on the client side, the underlying API is employed to create a new -connection to a service, in our example the transport service would be -connected. -@example -struct GNUNET_CLIENT_Connection *client; client = -GNUNET_CLIENT_connect ("transport", cfg); -@end example - -@c *************************************************************************** -@node Client - Initialize request message -@subsubsection Client - Initialize request message -@c %**end of header - -When the connection is ready, we initialize the message. In this step, all the -fields of the message should be properly initialized, namely the size, type, -and some extra user-defined data, such as timeout, name of transport, address -and name of transport. -@example -struct AddressLookupMessage *msg; size_t len = -sizeof (struct AddressLookupMessage) + addressLen + strlen (nameTrans) + 1; -msg->header->size = htons (len); msg->header->type = htons -(GNUNET_MESSAGE_TYPE_TRANSPORT_ADDRESS_LOOKUP); msg->timeout = -GNUNET_TIME_absolute_hton (abs_timeout); msg->addrlen = htonl (addressLen); -char *addrbuf = (char *) &msg[1]; memcpy (addrbuf, address, addressLen); char -*tbuf = &addrbuf[addressLen]; memcpy (tbuf, nameTrans, strlen (nameTrans) + 1); -@end example - -Note that, here the functions @code{htonl}, @code{htons} and -@code{GNUNET_TIME_absolute_hton} are applied to convert little endian into big -endian, about the usage of the big/small edian order and the corresponding -conversion function please refer to Introduction of Big Endian and Little -Endian. - -@c *************************************************************************** -@node Client - Send request and receive response -@subsubsection Client - Send request and receive response -@c %**end of header - -FIXME: This is very outdated, see the tutorial for the -current API! - -Next, the client would send the constructed message as a request to the service -and wait for the response from the service. To accomplish this goal, there are -a number of API calls that can be used. In this example, -@code{GNUNET_CLIENT_transmit_and_get_response} is chosen as the most -appropriate function to use. -@example -GNUNET_CLIENT_transmit_and_get_response -(client, msg->header, timeout, GNUNET_YES, &address_response_processor, -arp_ctx); -@end example - -the argument @code{address_response_processor} is a function with -@code{GNUNET_CLIENT_MessageHandler} type, which is used to process the reply -message from the service. - -@node Server - Startup service -@subsubsection Server - Startup service - -After receiving the request message, we run a standard GNUnet service startup -sequence using @code{GNUNET_SERVICE_run}, as follows, -@example -int main(int -argc, char**argv) @{ GNUNET_SERVICE_run(argc, argv, "transport" -GNUNET_SERVICE_OPTION_NONE, &run, NULL)); @} -@end example - -@c *************************************************************************** -@node Server - Add new handles for specified messages -@subsubsection Server - Add new handles for specified messages -@c %**end of header - -in the function above the argument @code{run} is used to initiate transport -service,and defined like this: -@example -static void run (void *cls, struct -GNUNET_SERVER_Handle *serv, const struct GNUNET_CONFIGURATION_Handle *cfg) @{ -GNUNET_SERVER_add_handlers (serv, handlers); @} -@end example - - -Here, @code{GNUNET_SERVER_add_handlers} must be called in the run function to -add new handlers in the service. The parameter @code{handlers} is a list of -@code{struct GNUNET_SERVER_MessageHandler} to tell the service which function -should be called when a particular type of message is received, and should be -defined in this way: -@example -static struct GNUNET_SERVER_MessageHandler -handlers[] = @{ @{&handle_start, NULL, GNUNET_MESSAGE_TYPE_TRANSPORT_START, -0@}, @{&handle_send, NULL, GNUNET_MESSAGE_TYPE_TRANSPORT_SEND, 0@}, -@{&handle_try_connect, NULL, GNUNET_MESSAGE_TYPE_TRANSPORT_TRY_CONNECT, sizeof -(struct TryConnectMessage)@}, @{&handle_address_lookup, NULL, -GNUNET_MESSAGE_TYPE_TRANSPORT_ADDRESS_LOOKUP, 0@}, @{NULL, NULL, 0, 0@} @}; -@end example - - -As shown, the first member of the struct in the first area is a callback -function, which is called to process the specified message types, given as the -third member. The second parameter is the closure for the callback function, -which is set to @code{NULL} in most cases, and the last parameter is the -expected size of the message of this type, usually we set it to 0 to accept -variable size, for special cases the exact size of the specified message also -can be set. In addition, the terminator sign depicted as @code{@{NULL, NULL, 0, -0@}} is set in the last aera. - -@c *************************************************************************** -@node Server - Process request message -@subsubsection Server - Process request message -@c %**end of header - -After the initialization of transport service, the request message would be -processed. Before handling the main message data, the validity of this message -should be checked out, e.g., to check whether the size of message is correct. -@example -size = ntohs (message->size); if (size < sizeof (struct -AddressLookupMessage)) @{ GNUNET_break_op (0); GNUNET_SERVER_receive_done -(client, GNUNET_SYSERR); return; @} -@end example - - -Note that, opposite to the construction method of the request message in the -client, in the server the function @code{nothl} and @code{ntohs} should be -employed during the extraction of the data from the message, so that the data -in big endian order can be converted back into little endian order. See more in -detail please refer to Introduction of Big Endian and Little Endian. - -Moreover in this example, the name of the transport stored in the message is a -0-terminated string, so we should also check whether the name of the transport -in the received message is 0-terminated: -@example -nameTransport = (const char *) -&address[addressLen]; if (nameTransport[size - sizeof (struct -AddressLookupMessage) - - addressLen - 1] != '\0') @{ GNUNET_break_op - (0); GNUNET_SERVER_receive_done (client, - GNUNET_SYSERR); return; @} -@end example - -Here, @code{GNUNET_SERVER_receive_done} should be called to tell the service -that the request is done and can receive the next message. The argument -@code{GNUNET_SYSERR} here indicates that the service didn't understand the -request message, and the processing of this request would be terminated. - -In comparison to the aforementioned situation, when the argument is equal to -@code{GNUNET_OK}, the service would continue to process the requst message. - -@c *************************************************************************** -@node Server - Response to client -@subsubsection Server - Response to client -@c %**end of header - -Once the processing of current request is done, the server should give the -response to the client. A new @code{struct AddressLookupMessage} would be -produced by the server in a similar way as the client did and sent to the -client, but here the type should be -@code{GNUNET_MESSAGE_TYPE_TRANSPORT_ADDRESS_REPLY} rather than -@code{GNUNET_MESSAGE_TYPE_TRANSPORT_ADDRESS_LOOKUP} in client. -@example -struct -AddressLookupMessage *msg; size_t len = sizeof (struct AddressLookupMessage) + -addressLen + strlen (nameTrans) + 1; msg->header->size = htons (len); -msg->header->type = htons (GNUNET_MESSAGE_TYPE_TRANSPORT_ADDRESS_REPLY); - -// ... - -struct GNUNET_SERVER_TransmitContext *tc; tc = -GNUNET_SERVER_transmit_context_create (client); -GNUNET_SERVER_transmit_context_append_data (tc, NULL, 0, -GNUNET_MESSAGE_TYPE_TRANSPORT_ADDRESS_REPLY); -GNUNET_SERVER_transmit_context_run (tc, rtimeout); -@end example - - -Note that, there are also a number of other APIs provided to the service to -send the message. - -@c *************************************************************************** -@node Server - Notification of clients -@subsubsection Server - Notification of clients -@c %**end of header - -Often a service needs to (repeatedly) transmit notifications to a client or a -group of clients. In these cases, the client typically has once registered for -a set of events and then needs to receive a message whenever such an event -happens (until the client disconnects). The use of a notification context can -help manage message queues to clients and handle disconnects. Notification -contexts can be used to send individualized messages to a particular client or -to broadcast messages to a group of clients. An individualized notification -might look like this: -@example - GNUNET_SERVER_notification_context_unicast(nc, - client, msg, GNUNET_YES); -@end example - - -Note that after processing the original registration message for notifications, -the server code still typically needs to call@ -@code{GNUNET_SERVER_receive_done} so that the client can transmit further -messages to the server. - -@c *************************************************************************** -@node Conversion between Network Byte Order (Big Endian) and Host Byte Order -@subsubsection Conversion between Network Byte Order (Big Endian) and Host Byte Order -@c %** subsub? it's a referenced page on the ipc document. -@c %**end of header - -Here we can simply comprehend big endian and little endian as Network Byte -Order and Host Byte Order respectively. What is the difference between both -two? - -Usually in our host computer we store the data byte as Host Byte Order, for -example, we store a integer in the RAM which might occupies 4 Byte, as Host -Byte Order the higher Byte would be stored at the lower address of RAM, and -the lower Byte would be stored at the higher address of RAM. However, contrast -to this, Network Byte Order just take the totally opposite way to store the -data, says, it will store the lower Byte at the lower address, and the higher -Byte will stay at higher address. - -For the current communication of network, we normally exchange the information -by surveying the data package, every two host wants to communicate with each -other must send and receive data package through network. In order to maintain -the identity of data through the transmission in the network, the order of the -Byte storage must changed before sending and after receiving the data. - -There ten convenient functions to realize the conversion of Byte Order in -GNUnet, as following: -@table @asis - -@item uint16_t htons(uint16_t hostshort) Convert host byte order to net byte -order with short int -@item uint32_t htonl(uint32_t hostlong) Convert host byte -order to net byte order with long int -@item uint16_t ntohs(uint16_t netshort) -Convert net byte order to host byte order with short int -@item uint32_t -ntohl(uint32_t netlong) Convert net byte order to host byte order with long int -@item unsigned long long GNUNET_ntohll (unsigned long long netlonglong) Convert -net byte order to host byte order with long long int -@item unsigned long long -GNUNET_htonll (unsigned long long hostlonglong) Convert host byte order to net -byte order with long long int -@item struct GNUNET_TIME_RelativeNBO -GNUNET_TIME_relative_hton (struct GNUNET_TIME_Relative a) Convert relative time -to network byte order. -@item struct GNUNET_TIME_Relative -GNUNET_TIME_relative_ntoh (struct GNUNET_TIME_RelativeNBO a) Convert relative -time from network byte order. -@item struct GNUNET_TIME_AbsoluteNBO -GNUNET_TIME_absolute_hton (struct GNUNET_TIME_Absolute a) Convert relative time -to network byte order. -@item struct GNUNET_TIME_Absolute -GNUNET_TIME_absolute_ntoh (struct GNUNET_TIME_AbsoluteNBO a) Convert relative -time from network byte order. -@end table - -@c *************************************************************************** - -@node Cryptography API -@subsection Cryptography API -@c %**end of header - -The gnunetutil APIs provides the cryptographic primitives used in GNUnet. -GNUnet uses 2048 bit RSA keys for the session key exchange and for signing -messages by peers and most other public-key operations. Most researchers in -cryptography consider 2048 bit RSA keys as secure and practically unbreakable -for a long time. The API provides functions to create a fresh key pair, read a -private key from a file (or create a new file if the file does not exist), -encrypt, decrypt, sign, verify and extraction of the public key into a format -suitable for network transmission. - -For the encryption of files and the actual data exchanged between peers GNUnet -uses 256-bit AES encryption. Fresh, session keys are negotiated for every new -connection.@ Again, there is no published technique to break this cipher in any -realistic amount of time. The API provides functions for generation of keys, -validation of keys (important for checking that decryptions using RSA -succeeded), encryption and decryption. - -GNUnet uses SHA-512 for computing one-way hash codes. The API provides -functions to compute a hash over a block in memory or over a file on disk. - -The crypto API also provides functions for randomizing a block of memory, -obtaining a single random number and for generating a permuation of the numbers -0 to n-1. Random number generation distinguishes between WEAK and STRONG random -number quality; WEAK random numbers are pseudo-random whereas STRONG random -numbers use entropy gathered from the operating system. - -Finally, the crypto API provides a means to deterministically generate a -1024-bit RSA key from a hash code. These functions should most likely not be -used by most applications; most importantly,@ -GNUNET_CRYPTO_rsa_key_create_from_hash does not create an RSA-key that should -be considered secure for traditional applications of RSA. - -@c *************************************************************************** -@node Message Queue API -@subsection Message Queue API -@c %**end of header - -@strong{ Introduction }@ Often, applications need to queue messages that are to -be sent to other GNUnet peers, clients or services. As all of GNUnet's -message-based communication APIs, by design, do not allow messages to be -queued, it is common to implement custom message queues manually when they are -needed. However, writing very similar code in multiple places is tedious and -leads to code duplication. - -MQ (for Message Queue) is an API that provides the functionality to implement -and use message queues. We intend to eventually replace all of the custom -message queue implementations in GNUnet with MQ. - -@strong{ Basic Concepts }@ The two most important entities in MQ are queues and -envelopes. - -Every queue is backed by a specific implementation (e.g. for mesh, stream, -connection, server client, etc.) that will actually deliver the queued -messages. For convenience,@ some queues also allow to specify a list of message -handlers. The message queue will then also wait for incoming messages and -dispatch them appropriately. - -An envelope holds the the memory for a message, as well as metadata (Where is -the envelope queued? What should happen after it has been sent?). Any envelope -can only be queued in one message queue. - -@strong{ Creating Queues }@ The following is a list of currently available -message queues. Note that to avoid layering issues, message queues for higher -level APIs are not part of @code{libgnunetutil}, but@ the respective API itself -provides the queue implementation. -@table @asis - -@item @code{GNUNET_MQ_queue_for_connection_client} Transmits queued messages -over a @code{GNUNET_CLIENT_Connection}@ handle. Also supports receiving with -message handlers.@ - -@item @code{GNUNET_MQ_queue_for_server_client} Transmits queued messages over a -@code{GNUNET_SERVER_Client}@ handle. Does not support incoming message -handlers.@ - -@item @code{GNUNET_MESH_mq_create} Transmits queued messages over a -@code{GNUNET_MESH_Tunnel}@ handle. Does not support incoming message handlers.@ - -@item @code{GNUNET_MQ_queue_for_callbacks} This is the most general -implementation. Instead of delivering and receiving messages with one of -GNUnet's communication APIs, implementation callbacks are called. Refer to -"Implementing Queues" for a more detailed explanation. -@end table - - -@strong{ Allocating Envelopes }@ A GNUnet message (as defined by the -GNUNET_MessageHeader) has three parts: The size, the type, and the body. - -MQ provides macros to allocate an envelope containing a message conveniently,@ -automatically setting the size and type fields of the message. - -Consider the following simple message, with the body consisting of a single -number value.@ @code{} -@example -struct NumberMessage @{ - /** Type: GNUNET_MESSAGE_TYPE_EXAMPLE_1 */ - struct GNUNET_MessageHeader header; uint32_t number GNUNET_PACKED; @}; -@end example - -An envelope containing an instance of the NumberMessage can be constructed like -this: -@example -struct GNUNET_MQ_Envelope *ev; struct NumberMessage *msg; ev = -GNUNET_MQ_msg (msg, GNUNET_MESSAGE_TYPE_EXAMPLE_1); msg->number = htonl (42); -@end example - - -In the above code, @code{GNUNET_MQ_msg} is a macro. The return value is the -newly allocated envelope. The first argument must be a pointer to some -@code{struct} containing a @code{struct GNUNET_MessageHeader header} field, -while the second argument is the desired message type, in host byte order. - -The @code{msg} pointer now points to an allocated message, where the message -type and the message size are already set. The message's size is inferred from -the type of the @code{msg} pointer: It will be set to 'sizeof(*msg)', properly -converted to network byte order. - -If the message body's size is dynamic, the the macro @code{GNUNET_MQ_msg_extra} -can be used to allocate an envelope whose message has additional space -allocated after the @code{msg} structure. - -If no structure has been defined for the message, -@code{GNUNET_MQ_msg_header_extra} can be used to allocate additional space -after the message header. The first argument then must be a pointer to a -@code{GNUNET_MessageHeader}. - -@strong{Envelope Properties}@ A few functions in MQ allow to set additional -properties on envelopes: -@table @asis - -@item @code{GNUNET_MQ_notify_sent} Allows to specify a function that will be -called once the envelope's message@ has been sent irrevocably. An envelope can -be canceled precisely up to the@ point where the notify sent callback has been -called. -@item @code{GNUNET_MQ_disable_corking} No corking will be used when -sending the message. Not every@ queue supports this flag, per default, -envelopes are sent with corking.@ - -@end table - - -@strong{Sending Envelopes}@ Once an envelope has been constructed, it can be -queued for sending with @code{GNUNET_MQ_send}. - -Note that in order to avoid memory leaks, an envelope must either be sent (the -queue will free it) or destroyed explicitly with @code{GNUNET_MQ_discard}. - -@strong{Canceling Envelopes}@ An envelope queued with @code{GNUNET_MQ_send} can -be canceled with @code{GNUNET_MQ_cancel}. Note that after the notify sent -callback has been called, canceling a message results in undefined behavior. -Thus it is unsafe to cancel an envelope that does not have a notify sent -callback. When canceling an envelope, it is not necessary@ to call -@code{GNUNET_MQ_discard}, and the envelope can't be sent again. - -@strong{ Implementing Queues }@ @code{TODO} - -@c *************************************************************************** -@node Service API -@subsection Service API -@c %**end of header - -Most GNUnet code lives in the form of services. Services are processes that -offer an API for other components of the system to build on. Those other -components can be command-line tools for users, graphical user interfaces or -other services. Services provide their API using an IPC protocol. For this, -each service must listen on either a TCP port or a UNIX domain socket; for -this, the service implementation uses the server API. This use of server is -exposed directly to the users of the service API. Thus, when using the service -API, one is usually also often using large parts of the server API. The service -API provides various convenience functions, such as parsing command-line -arguments and the configuration file, which are not found in the server API. -The dual to the service/server API is the client API, which can be used to -access services. - -The most common way to start a service is to use the GNUNET_SERVICE_run -function from the program's main function. GNUNET_SERVICE_run will then parse -the command line and configuration files and, based on the options found there, -start the server. It will then give back control to the main program, passing -the server and the configuration to the GNUNET_SERVICE_Main callback. -GNUNET_SERVICE_run will also take care of starting the scheduler loop. If this -is inappropriate (for example, because the scheduler loop is already running), -GNUNET_SERVICE_start and related functions provide an alternative to -GNUNET_SERVICE_run. - -When starting a service, the service_name option is used to determine which -sections in the configuration file should be used to configure the service. A -typical value here is the name of the src/ sub-directory, for example -"statistics". The same string would also be given to GNUNET_CLIENT_connect to -access the service. - -Once a service has been initialized, the program should use the@ -GNUNET_SERVICE_Main callback to register message handlers using -GNUNET_SERVER_add_handlers. The service will already have registered a handler -for the "TEST" message. - -The option bitfield (enum GNUNET_SERVICE_Options) determines how a service -should behave during shutdown. There are three key strategies: -@table @asis - -@item instant (GNUNET_SERVICE_OPTION_NONE) Upon receiving the shutdown signal -from the scheduler, the service immediately terminates the server, closing all -existing connections with clients. -@item manual -(GNUNET_SERVICE_OPTION_MANUAL_SHUTDOWN) The service does nothing by itself -during shutdown. The main program will need to take the appropriate action by -calling GNUNET_SERVER_destroy or GNUNET_SERVICE_stop (depending on how the -service was initialized) to terminate the service. This method is used by -gnunet-service-arm and rather uncommon. -@item soft -(GNUNET_SERVICE_OPTION_SOFT_SHUTDOWN) Upon receiving the shutdown signal from -the scheduler, the service immediately tells the server to stop listening for -incoming clients. Requests from normal existing clients are still processed and -the server/service terminates once all normal clients have disconnected. -Clients that are not expected to ever disconnect (such as clients that monitor -performance values) can be marked as 'monitor' clients using -GNUNET_SERVER_client_mark_monitor. Those clients will continue to be processed -until all 'normal' clients have disconnected. Then, the server will terminate, -closing the monitor connections. This mode is for example used by 'statistics', -allowing existing 'normal' clients to set (possibly persistent) statistic -values before terminating. -@end table - -@c *************************************************************************** -@node Optimizing Memory Consumption of GNUnet's (Multi-) Hash Maps -@subsection Optimizing Memory Consumption of GNUnet's (Multi-) Hash Maps -@c %**end of header - -A commonly used data structure in GNUnet is a (multi-)hash map. It is most -often used to map a peer identity to some data structure, but also to map -arbitrary keys to values (for example to track requests in the distributed hash -table or in file-sharing). As it is commonly used, the DHT is actually -sometimes responsible for a large share of GNUnet's overall memory consumption -(for some processes, 30% is not uncommon). The following text documents some -API quirks (and their implications for applications) that were recently -introduced to minimize the footprint of the hash map. - - -@c *************************************************************************** -@menu -* Analysis:: -* Solution:: -* Migration:: -* Conclusion:: -* Availability:: -@end menu - -@node Analysis -@subsubsection Analysis -@c %**end of header - -The main reason for the "excessive" memory consumption by the hash map is that -GNUnet uses 512-bit cryptographic hash codes --- and the (multi-)hash map also -uses the same 512-bit 'struct GNUNET_HashCode'. As a result, storing just the -keys requires 64 bytes of memory for each key. As some applications like to -keep a large number of entries in the hash map (after all, that's what maps -are good for), 64 bytes per hash is significant: keeping a pointer to the -value and having a linked list for collisions consume between 8 and 16 bytes, -and 'malloc' may add about the same overhead per allocation, putting us in the -16 to 32 byte per entry ballpark. Adding a 64-byte key then triples the -overall memory requirement for the hash map. - -To make things "worse", most of the time storing the key in the hash map is -not required: it is typically already in memory elsewhere! In most cases, the -values stored in the hash map are some application-specific struct that _also_ -contains the hash. Here is a simplified example: -@example -struct MyValue @{ -struct GNUNET_HashCode key; unsigned int my_data; @}; - -// ... -val = GNUNET_malloc (sizeof (struct MyValue)); val->key = key; val->my_data = -42; GNUNET_CONTAINER_multihashmap_put (map, &key, val, ...); -@end example - - -This is a common pattern as later the entries might need to be removed, and at -that time it is convenient to have the key immediately at hand: -@example -GNUNET_CONTAINER_multihashmap_remove (map, &val->key, val); -@end example - - -Note that here we end up with two times 64 bytes for the key, plus maybe 64 -bytes total for the rest of the 'struct MyValue' and the map entry in the hash -map. The resulting redundant storage of the key increases overall memory -consumption per entry from the "optimal" 128 bytes to 192 bytes. This is not -just an extreme example: overheads in practice are actually sometimes close to -those highlighted in this example. This is especially true for maps with a -significant number of entries, as there we tend to really try to keep the -entries small. -@c *************************************************************************** -@node Solution -@subsubsection Solution -@c %**end of header - -The solution that has now been implemented is to @strong{optionally} allow the -hash map to not make a (deep) copy of the hash but instead have a pointer to -the hash/key in the entry. This reduces the memory consumption for the key -from 64 bytes to 4 to 8 bytes. However, it can also only work if the key is -actually stored in the entry (which is the case most of the time) and if the -entry does not modify the key (which in all of the code I'm aware of has been -always the case if there key is stored in the entry). Finally, when the client -stores an entry in the hash map, it @strong{must} provide a pointer to the key -within the entry, not just a pointer to a transient location of the key. If -the client code does not meet these requirements, the result is a dangling -pointer and undefined behavior of the (multi-)hash map API. -@c *************************************************************************** -@node Migration -@subsubsection Migration -@c %**end of header - -To use the new feature, first check that the values contain the respective key -(and never modify it). Then, all calls to -@code{GNUNET_CONTAINER_multihashmap_put} on the respective map must be audited -and most likely changed to pass a pointer into the value's struct. For the -initial example, the new code would look like this: -@example -struct MyValue @{ -struct GNUNET_HashCode key; unsigned int my_data; @}; - -// ... -val = GNUNET_malloc (sizeof (struct MyValue)); val->key = key; val->my_data = -42; GNUNET_CONTAINER_multihashmap_put (map, &val->key, val, ...); -@end example - - -Note that @code{&val} was changed to @code{&val->key} in the argument to the -@code{put} call. This is critical as often @code{key} is on the stack or in -some other transient data structure and thus having the hash map keep a pointer -to @code{key} would not work. Only the key inside of @code{val} has the same -lifetime as the entry in the map (this must of course be checked as well). -Naturally, @code{val->key} must be intiialized before the @code{put} call. Once -all @code{put} calls have been converted and double-checked, you can change the -call to create the hash map from -@example -map = -GNUNET_CONTAINER_multihashmap_create (SIZE, GNUNET_NO); -@end example - -to - -@example -map = GNUNET_CONTAINER_multihashmap_create (SIZE, GNUNET_YES); -@end example - -If everything was done correctly, you now use about 60 bytes less memory per -entry in @code{map}. However, if now (or in the future) any call to @code{put} -does not ensure that the given key is valid until the entry is removed from the -map, undefined behavior is likely to be observed. -@c *************************************************************************** -@node Conclusion -@subsubsection Conclusion -@c %**end of header - -The new optimization can is often applicable and can result in a reduction in -memory consumption of up to 30% in practice. However, it makes the code less -robust as additional invariants are imposed on the multi hash map client. Thus -applications should refrain from enabling the new mode unless the resulting -performance increase is deemed significant enough. In particular, it should -generally not be used in new code (wait at least until benchmarks exist). -@c *************************************************************************** -@node Availability -@subsubsection Availability -@c %**end of header - -The new multi hash map code was committed in SVN 24319 (will be in GNUnet -0.9.4). Various subsystems (transport, core, dht, file-sharing) were -previously audited and modified to take advantage of the new capability. In -particular, memory consumption of the file-sharing service is expected to drop -by 20-30% due to this change. - -@c *************************************************************************** -@node The CONTAINER_MDLL API -@subsection The CONTAINER_MDLL API -@c %**end of header - -This text documents the GNUNET_CONTAINER_MDLL API. The GNUNET_CONTAINER_MDLL -API is similar to the GNUNET_CONTAINER_DLL API in that it provides operations -for the construction and manipulation of doubly-linked lists. The key -difference to the (simpler) DLL-API is that the MDLL-version allows a single -element (instance of a "struct") to be in multiple linked lists at the same -time. - -Like the DLL API, the MDLL API stores (most of) the data structures for the -doubly-linked list with the respective elements; only the 'head' and 'tail' -pointers are stored "elsewhere" --- and the application needs to provide the -locations of head and tail to each of the calls in the MDLL API. The key -difference for the MDLL API is that the "next" and "previous" pointers in the -struct can no longer be simply called "next" and "prev" --- after all, the -element may be in multiple doubly-linked lists, so we cannot just have one -"next" and one "prev" pointer! - -The solution is to have multiple fields that must have a name of the format -"next_XX" and "prev_XX" where "XX" is the name of one of the doubly-linked -lists. Here is a simple example: -@example -struct MyMultiListElement @{ struct -MyMultiListElement *next_ALIST; struct MyMultiListElement *prev_ALIST; struct -MyMultiListElement *next_BLIST; struct MyMultiListElement *prev_BLIST; void -*data; @}; -@end example - - -Note that by convention, we use all-uppercase letters for the list names. In -addition, the program needs to have a location for the head and tail pointers -for both lists, for example: -@example -static struct MyMultiListElement -*head_ALIST; static struct MyMultiListElement *tail_ALIST; static struct -MyMultiListElement *head_BLIST; static struct MyMultiListElement *tail_BLIST; -@end example - - -Using the MDLL-macros, we can now insert an element into the ALIST: -@example -GNUNET_CONTAINER_MDLL_insert (ALIST, head_ALIST, tail_ALIST, element); -@end example - - -Passing "ALIST" as the first argument to MDLL specifies which of the next/prev -fields in the 'struct MyMultiListElement' should be used. The extra "ALIST" -argument and the "_ALIST" in the names of the next/prev-members are the only -differences between the MDDL and DLL-API. Like the DLL-API, the MDLL-API offers -functions for inserting (at head, at tail, after a given element) and removing -elements from the list. Iterating over the list should be done by directly -accessing the "next_XX" and/or "prev_XX" members. - -@c *************************************************************************** -@node The Automatic Restart Manager (ARM) -@section The Automatic Restart Manager (ARM) -@c %**end of header - -GNUnet's Automated Restart Manager (ARM) is the GNUnet service responsible for -system initialization and service babysitting. ARM starts and halts services, -detects configuration changes and restarts services impacted by the changes as -needed. It's also responsible for restarting services in case of crashes and is -planned to incorporate automatic debugging for diagnosing service crashes -providing developers insights about crash reasons. The purpose of this document -is to give GNUnet developer an idea about how ARM works and how to interact -with it. - -@menu -* Basic functionality:: -* Key configuration options:: -* Availability2:: -* Reliability:: -@end menu - -@c *************************************************************************** -@node Basic functionality -@subsection Basic functionality -@c %**end of header - -@itemize @bullet -@item ARM source code can be found under "src/arm".@ Service processes are -managed by the functions in "gnunet-service-arm.c" which is controlled with -"gnunet-arm.c" (main function in that file is ARM's entry point). - -@item The functions responsible for communicating with ARM , starting and -stopping services -including ARM service itself- are provided by the ARM API -"arm_api.c".@ Function: GNUNET_ARM_connect() returns to the caller an ARM -handle after setting it to the caller's context (configuration and scheduler in -use). This handle can be used afterwards by the caller to communicate with ARM. -Functions GNUNET_ARM_start_service() and GNUNET_ARM_stop_service() are used for -starting and stopping services respectively. - -@item A typical example of using these basic ARM services can be found in file -test_arm_api.c. The test case connects to ARM, starts it, then uses it to start -a service "resolver", stops the "resolver" then stops "ARM". -@end itemize - -@c *************************************************************************** -@node Key configuration options -@subsection Key configuration options -@c %**end of header - -Configurations for ARM and services should be available in a .conf file (As an -example, see test_arm_api_data.conf). When running ARM, the configuration file -to use should be passed to the command:@ @code{@ $ gnunet-arm -s -c -configuration_to_use.conf@ }@ If no configuration is passed, the default -configuration file will be used (see GNUNET_PREFIX/share/gnunet/defaults.conf -which is created from contrib/defaults.conf).@ Each of the services is having a -section starting by the service name between square brackets, for example: -"[arm]". The following options configure how ARM configures or interacts with -the various services: - -@table @asis - -@item PORT Port number on which the service is listening for incoming TCP -connections. ARM will start the services should it notice a request at this -port. - -@item HOSTNAME Specifies on which host the service is deployed. Note -that ARM can only start services that are running on the local system (but will -not check that the hostname matches the local machine name). This option is -used by the @code{gnunet_client_lib.h} implementation to determine which system -to connect to. The default is "localhost". - -@item BINARY The name of the service binary file. - -@item OPTIONS To be passed to the service. - -@item PREFIX A command to pre-pend to the actual command, for example, running -a service with "valgrind" or "gdb" - -@item DEBUG Run in debug mode (much verbosity). - -@item AUTOSTART ARM will listen to UNIX domain socket and/or TCP port of the -service and start the service on-demand. - -@item FORCESTART ARM will always -start this service when the peer is started. - -@item ACCEPT_FROM IPv4 addresses the service accepts connections from. - -@item ACCEPT_FROM6 IPv6 addresses the service accepts connections from. - -@end table - - -Options that impact the operation of ARM overall are in the "[arm]" section. -ARM is a normal service and has (except for AUTOSTART) all of the options that -other services do. In addition, ARM has the following options: -@table @asis - -@item GLOBAL_PREFIX Command to be pre-pended to all services that are going to -run.@ - -@item GLOBAL_POSTFIX Global option that will be supplied to all the services -that are going to run.@ - -@end table - -@c *************************************************************************** -@node Availability2 -@subsection Availability2 -@c %**end of header - -As mentioned before, one of the features provided by ARM is starting services -on demand. Consider the example of one service "client" that wants to connect -to another service a "server". The "client" will ask ARM to run the "server". -ARM starts the "server". The "server" starts listening to incoming connections. -The "client" will establish a connection with the "server". And then, they will -start to communicate together.@ One problem with that scheme is that it's -slow!@ The "client" service wants to communicate with the "server" service at -once and is not willing wait for it to be started and listening to incoming -connections before serving its request.@ One solution for that problem will be -that ARM starts all services as default services. That solution will solve the -problem, yet, it's not quite practical, for some services that are going to be -started can never be used or are going to be used after a relatively long -time.@ The approach followed by ARM to solve this problem is as follows: -@itemize @bullet - - -@item For each service having a PORT field in the configuration file and that -is not one of the default services ( a service that accepts incoming -connections from clients), ARM creates listening sockets for all addresses -associated with that service. - -@item The "client" will immediately establish a connection with the "server". - -@item ARM --- pretending to be the "server" --- will listen on the respective -port and notice the incoming connection from the "client" (but not accept it), -instead - -@item Once there is an incoming connection, ARM will start the "server", -passing on the listen sockets (now, the service is started and can do its -work). - -@item Other client services now can directly connect directly to the "server". -@end itemize - -@c *************************************************************************** -@node Reliability -@subsection Reliability - -One of the features provided by ARM, is the automatic restart of crashed -services.@ ARM needs to know which of the running services died. Function -"gnunet-service-arm.c/maint_child_death()" is responsible for that. The -function is scheduled to run upon receiving a SIGCHLD signal. The function, -then, iterates ARM's list of services running and monitors which service has -died (crashed). For all crashing services, ARM restarts them.@ Now, considering -the case of a service having a serious problem causing it to crash each time -it's started by ARM. If ARM keeps blindly restarting such a service, we are -going to have the pattern: start-crash-restart-crash-restart-crash and so -forth!! Which is of course not practical.@ For that reason, ARM schedules the -service to be restarted after waiting for some delay that grows exponentially -with each crash/restart of that service.@ To clarify the idea, considering the -following example: -@itemize @bullet - - -@item Service S crashed. - -@item ARM receives the SIGCHLD and inspects its list of services to find the -dead one(s). - -@item ARM finds S dead and schedules it for restarting after "backoff" time -which is initially set to 1ms. ARM will double the backoff time correspondent -to S (now backoff(S) = 2ms) - -@item Because there is a severe problem with S, it crashed again. - -@item Again ARM receives the SIGCHLD and detects that it's S again that's -crashed. ARM schedules it for restarting but after its new backoff time (which -became 2ms), and doubles its backoff time (now backoff(S) = 4). - -@item and so on, until backoff(S) reaches a certain threshold -(EXPONENTIAL_BACKOFF_THRESHOLD is set to half an hour), after reaching it, -backoff(S) will remain half an hour, hence ARM won't be busy for a lot of time -trying to restart a problematic service. -@end itemize - -@c *************************************************************************** -@node GNUnet's TRANSPORT Subsystem -@section GNUnet's TRANSPORT Subsystem -@c %**end of header - -This chapter documents how the GNUnet transport subsystem works. The GNUnet -transport subsystem consists of three main components: the transport API (the -interface used by the rest of the system to access the transport service), the -transport service itself (most of the interesting functions, such as choosing -transports, happens here) and the transport plugins. A transport plugin is a -concrete implementation for how two GNUnet peers communicate; many plugins -exist, for example for communication via TCP, UDP, HTTP, HTTPS and others. -Finally, the transport subsystem uses supporting code, especially the NAT/UPnP -library to help with tasks such as NAT traversal. - -Key tasks of the transport service include: -@itemize @bullet - - -@item Create our HELLO message, notify clients and neighbours if our HELLO -changes (using NAT library as necessary) - -@item Validate HELLOs from other peers (send PING), allow other peers to -validate our HELLO's addresses (send PONG) - -@item Upon request, establish connections to other peers (using address -selection from ATS subsystem) and maintain them (again using PINGs and PONGs) -as long as desired - -@item Accept incoming connections, give ATS service the opportunity to switch -communication channels - -@item Notify clients about peers that have connected to us or that have been -disconnected from us - -@item If a (stateful) connection goes down unexpectedly (without explicit -DISCONNECT), quickly attempt to recover (without notifying clients) but do -notify clients quickly if reconnecting fails - -@item Send (payload) messages arriving from clients to other peers via -transport plugins and receive messages from other peers, forwarding those to -clients - -@item Enforce inbound traffic limits (using flow-control if it is applicable); -outbound traffic limits are enforced by CORE, not by us (!) - -@item Enforce restrictions on P2P connection as specified by the blacklist -configuration and blacklisting clients -@end itemize - - -Note that the term "clients" in the list above really refers to the GNUnet-CORE -service, as CORE is typically the only client of the transport service. - -@menu -* Address validation protocol:: -@end menu - -@node Address validation protocol -@subsection Address validation protocol -@c %**end of header - -This section documents how the GNUnet transport service validates connections -with other peers. It is a high-level description of the protocol necessary to -understand the details of the implementation. It should be noted that when we -talk about PING and PONG messages in this section, we refer to transport-level -PING and PONG messages, which are different from core-level PING and PONG -messages (both in implementation and function). - -The goal of transport-level address validation is to minimize the chances of a -successful man-in-the-middle attack against GNUnet peers on the transport -level. Such an attack would not allow the adversary to decrypt the P2P -transmissions, but a successful attacker could at least measure traffic volumes -and latencies (raising the adversaries capablities by those of a global passive -adversary in the worst case). The scenarios we are concerned about is an -attacker, Mallory, giving a HELLO to Alice that claims to be for Bob, but -contains Mallory's IP address instead of Bobs (for some transport). Mallory -would then forward the traffic to Bob (by initiating a connection to Bob and -claiming to be Alice). As a further complication, the scheme has to work even -if say Alice is behind a NAT without traversal support and hence has no address -of her own (and thus Alice must always initiate the connection to Bob). - -An additional constraint is that HELLO messages do not contain a cryptographic -signature since other peers must be able to edit (i.e. remove) addresses from -the HELLO at any time (this was not true in GNUnet 0.8.x). A basic -@strong{assumption} is that each peer knows the set of possible network -addresses that it @strong{might} be reachable under (so for example, the -external IP address of the NAT plus the LAN address(es) with the respective -ports). - -The solution is the following. If Alice wants to validate that a given address -for Bob is valid (i.e. is actually established @strong{directly} with the -intended target), it sends a PING message over that connection to Bob. Note -that in this case, Alice initiated the connection so only she knows which -address was used for sure (Alice maybe behind NAT, so whatever address Bob -sees may not be an address Alice knows she has). Bob checks that the address -given in the PING is actually one of his addresses (does not belong to -Mallory), and if it is, sends back a PONG (with a signature that says that Bob -owns/uses the address from the PING). Alice checks the signature and is happy -if it is valid and the address in the PONG is the address she used. This is -similar to the 0.8.x protocol where the HELLO contained a signature from Bob -for each address used by Bob. Here, the purpose code for the signature is -@code{GNUNET_SIGNATURE_PURPOSE_TRANSPORT_PONG_OWN}. After this, Alice will -remember Bob's address and consider the address valid for a while (12h in the -current implementation). Note that after this exchange, Alice only considers -Bob's address to be valid, the connection itself is not considered -'established'. In particular, Alice may have many addresses for Bob that she -considers valid. - -The PONG message is protected with a nonce/challenge against replay attacks -and uses an expiration time for the signature (but those are almost -implementation details). - -@node NAT library -@section NAT library -@c %**end of header - -The goal of the GNUnet NAT library is to provide a general-purpose API for NAT -traversal @strong{without} third-party support. So protocols that involve -contacting a third peer to help establish a connection between two peers are -outside of the scope of this API. That does not mean that GNUnet doesn't -support involving a third peer (we can do this with the distance-vector -transport or using application-level protocols), it just means that the NAT API -is not concerned with this possibility. The API is written so that it will work -for IPv6-NAT in the future as well as current IPv4-NAT. Furthermore, the NAT -API is always used, even for peers that are not behind NAT --- in that case, -the mapping provided is simply the identity. - -NAT traversal is initiated by calling @code{GNUNET_NAT_register}. Given a set -of addresses that the peer has locally bound to (TCP or UDP), the NAT library -will return (via callback) a (possibly longer) list of addresses the peer -@strong{might} be reachable under. Internally, depending on the configuration, -the NAT library will try to punch a hole (using UPnP) or just "know" that the -NAT was manually punched and generate the respective external IP address (the -one that should be globally visible) based on the given information. - -The NAT library also supports ICMP-based NAT traversal. Here, the other peer -can request connection-reversal by this peer (in this special case, the peer is -even allowed to configure a port number of zero). If the NAT library detects a -connection-reversal request, it returns the respective target address to the -client as well. It should be noted that connection-reversal is currently only -intended for TCP, so other plugins @strong{must} pass @code{NULL} for the -reversal callback. Naturally, the NAT library also supports requesting -connection reversal from a remote peer (@code{GNUNET_NAT_run_client}). - -Once initialized, the NAT handle can be used to test if a given address is -possibly a valid address for this peer (@code{GNUNET_NAT_test_address}). This -is used for validating our addresses when generating PONGs. - -Finally, the NAT library contains an API to test if our NAT configuration is -correct. Using @code{GNUNET_NAT_test_start} @strong{before} binding to the -respective port, the NAT library can be used to test if the configuration -works. The test function act as a local client, initialize the NAT traversal -and then contact a @code{gnunet-nat-server} (running by default on -@code{gnunet.org}) and ask for a connection to be established. This way, it is -easy to test if the current NAT configuration is valid. - -@node Distance-Vector plugin -@section Distance-Vector plugin -@c %**end of header - -The Distance Vector (DV) transport is a transport mechanism that allows peers -to act as relays for each other, thereby connecting peers that would otherwise -be unable to connect. This gives a larger connection set to applications that -may work better with more peers to choose from (for example, File Sharing -and/or DHT). - -The Distance Vector transport essentially has two functions. The first is -"gossiping" connection information about more distant peers to directly -connected peers. The second is taking messages intended for non-directly -connected peers and encapsulating them in a DV wrapper that contains the -required information for routing the message through forwarding peers. Via -gossiping, optimal routes through the known DV neighborhood are discovered and -utilized and the message encapsulation provides some benefits in addition to -simply getting the message from the correct source to the proper destination. - -The gossiping function of DV provides an up to date routing table of peers that -are available up to some number of hops. We call this a fisheye view of the -network (like a fish, nearby objects are known while more distant ones -unknown). Gossip messages are sent only to directly connected peers, but they -are sent about other knowns peers within the "fisheye distance". Whenever two -peers connect, they immediately gossip to each other about their appropriate -other neighbors. They also gossip about the newly connected peer to previously -connected neighbors. In order to keep the routing tables up to date, disconnect -notifications are propogated as gossip as well (because disconnects may not be -sent/received, timeouts are also used remove stagnant routing table entries). - -Routing of messages via DV is straightforward. When the DV transport is -notified of a message destined for a non-direct neighbor, the appropriate -forwarding peer is selected, and the base message is encapsulated in a DV -message which contains information about the initial peer and the intended -recipient. At each forwarding hop, the initial peer is validated (the -forwarding peer ensures that it has the initial peer in its neighborhood, -otherwise the message is dropped). Next the base message is re-encapsulated in -a new DV message for the next hop in the forwarding chain (or delivered to the -current peer, if it has arrived at the destination). - -Assume a three peer network with peers Alice, Bob and Carol. Assume that Alice -<-> Bob and Bob <-> Carol are direct (e.g. over TCP or UDP transports) -connections, but that Alice cannot directly connect to Carol. This may be the -case due to NAT or firewall restrictions, or perhaps based on one of the peers -respective configurations. If the Distance Vector transport is enabled on all -three peers, it will automatically discover (from the gossip protocol) that -Alice and Carol can connect via Bob and provide a "virtual" Alice <-> Carol -connection. Routing between Alice and Carol happens as follows; Alice creates a -message destined for Carol and notifies the DV transport about it. The DV -transport at Alice looks up Carol in the routing table and finds that the -message must be sent through Bob for Carol. The message is encapsulated setting -Alice as the initiator and Carol as the destination and sent to Bob. Bob -receives the messages, verifies both Alice and Carol are known to Bob, and -re-wraps the message in a new DV message for Carol. The DV transport at Carol -receives this message, unwraps the original message, and delivers it to Carol -as though it came directly from Alice. - -@node SMTP plugin -@section SMTP plugin -@c %**end of header - -This page describes the new SMTP transport plugin for GNUnet as it exists in -the 0.7.x and 0.8.x branch. SMTP support is currently not available in GNUnet -0.9.x. This page also describes the transport layer abstraction (as it existed -in 0.7.x and 0.8.x) in more detail and gives some benchmarking results. The -performance results presented are quite old and maybe outdated at this point. -@itemize @bullet -@item Why use SMTP for a peer-to-peer transport? -@item SMTPHow does it work? -@item How do I configure my peer? -@item How do I test if it works? -@item How fast is it? -@item Is there any additional documentation? -@end itemize - - -@menu -* Why use SMTP for a peer-to-peer transport?:: -* How does it work?:: -* How do I configure my peer?:: -* How do I test if it works?:: -* How fast is it?:: -@end menu - -@node Why use SMTP for a peer-to-peer transport? -@subsection Why use SMTP for a peer-to-peer transport? -@c %**end of header - -There are many reasons why one would not want to use SMTP: -@itemize @bullet -@item SMTP is using more bandwidth than TCP, UDP or HTTP -@item SMTP has a much higher latency. -@item SMTP requires significantly more computation (encoding and decoding time) -for the peers. -@item SMTP is significantly more complicated to configure. -@item SMTP may be abused by tricking GNUnet into sending mail to@ -non-participating third parties. -@end itemize - -So why would anybody want to use SMTP? -@itemize @bullet -@item SMTP can be used to contact peers behind NAT boxes (in virtual private -networks). -@item SMTP can be used to circumvent policies that limit or prohibit -peer-to-peer traffic by masking as "legitimate" traffic. -@item SMTP uses E-mail addresses which are independent of a specific IP, which -can be useful to address peers that use dynamic IP addresses. -@item SMTP can be used to initiate a connection (e.g. initial address exchange) -and peers can then negotiate the use of a more efficient protocol (e.g. TCP) -for the actual communication. -@end itemize - -In summary, SMTP can for example be used to send a message to a peer behind a -NAT box that has a dynamic IP to tell the peer to establish a TCP connection -to a peer outside of the private network. Even an extraordinary overhead for -this first message would be irrelevant in this type of situation. - -@node How does it work? -@subsection How does it work? -@c %**end of header - -When a GNUnet peer needs to send a message to another GNUnet peer that has -advertised (only) an SMTP transport address, GNUnet base64-encodes the message -and sends it in an E-mail to the advertised address. The advertisement -contains a filter which is placed in the E-mail header, such that the -receiving host can filter the tagged E-mails and forward it to the GNUnet peer -process. The filter can be specified individually by each peer and be changed -over time. This makes it impossible to censor GNUnet E-mail messages by -searching for a generic filter. - -@node How do I configure my peer? -@subsection How do I configure my peer? -@c %**end of header - -First, you need to configure @code{procmail} to filter your inbound E-mail for -GNUnet traffic. The GNUnet messages must be delivered into a pipe, for example -@code{/tmp/gnunet.smtp}. You also need to define a filter that is used by -procmail to detect GNUnet messages. You are free to choose whichever filter -you like, but you should make sure that it does not occur in your other -E-mail. In our example, we will use @code{X-mailer: GNUnet}. The -@code{~/.procmailrc} configuration file then looks like this: -@example -:0: -* ^X-mailer: GNUnet -/tmp/gnunet.smtp -# where do you want your other e-mail delivered to (default: /var/spool/mail/) -:0: /var/spool/mail/ -@end example - -After adding this file, first make sure that your regular E-mail still works -(e.g. by sending an E-mail to yourself). Then edit the GNUnet configuration. -In the section @code{SMTP} you need to specify your E-mail address under -@code{EMAIL}, your mail server (for outgoing mail) under @code{SERVER}, the -filter (X-mailer: GNUnet in the example) under @code{FILTER} and the name of -the pipe under @code{PIPE}.@ The completed section could then look like this: -@example -EMAIL = me@@mail.gnu.org MTU = 65000 SERVER = mail.gnu.org:25 FILTER = -"X-mailer: GNUnet" PIPE = /tmp/gnunet.smtp -@end example - -Finally, you need to add @code{smtp} to the list of @code{TRANSPORTS} in the -@code{GNUNETD} section. GNUnet peers will use the E-mail address that you -specified to contact your peer until the advertisement times out. Thus, if you -are not sure if everything works properly or if you are not planning to be -online for a long time, you may want to configure this timeout to be short, -e.g. just one hour. For this, set @code{HELLOEXPIRES} to @code{1} in the -@code{GNUNETD} section. - -This should be it, but you may probably want to test it first.@ -@node How do I test if it works? -@subsection How do I test if it works? -@c %**end of header - -Any transport can be subjected to some rudimentary tests using the -@code{gnunet-transport-check} tool. The tool sends a message to the local node -via the transport and checks that a valid message is received. While this test -does not involve other peers and can not check if firewalls or other network -obstacles prohibit proper operation, this is a great testcase for the SMTP -transport since it tests pretty much nearly all of the functionality. - -@code{gnunet-transport-check} should only be used without running -@code{gnunetd} at the same time. By default, @code{gnunet-transport-check} -tests all transports that are specified in the configuration file. But you can -specifically test SMTP by giving the option @code{--transport=smtp}. - -Note that this test always checks if a transport can receive and send. While -you can configure most transports to only receive or only send messages, this -test will only work if you have configured the transport to send and receive -messages. - -@node How fast is it? -@subsection How fast is it? -@c %**end of header - -We have measured the performance of the UDP, TCP and SMTP transport layer -directly and when used from an application using the GNUnet core. Measureing -just the transport layer gives the better view of the actual overhead of the -protocol, whereas evaluating the transport from the application puts the -overhead into perspective from a practical point of view. - -The loopback measurements of the SMTP transport were performed on three -different machines spanning a range of modern SMTP configurations. We used a -PIII-800 running RedHat 7.3 with the Purdue Computer Science configuration -which includes filters for spam. We also used a Xenon 2 GHZ with a vanilla -RedHat 8.0 sendmail configuration. Furthermore, we used qmail on a PIII-1000 -running Sorcerer GNU Linux (SGL). The numbers for UDP and TCP are provided -using the SGL configuration. The qmail benchmark uses qmail's internal -filtering whereas the sendmail benchmarks relies on procmail to filter and -deliver the mail. We used the transport layer to send a message of b bytes -(excluding transport protocol headers) directly to the local machine. This -way, network latency and packet loss on the wire have no impact on the -timings. n messages were sent sequentially over the transport layer, sending -message i+1 after the i-th message was received. All messages were sent over -the same connection and the time to establish the connection was not taken -into account since this overhead is miniscule in practice --- as long as a -connection is used for a significant number of messages. - -@multitable @columnfractions .20 .15 .15 .15 .15 .15 -@headitem Transport @tab UDP @tab TCP @tab SMTP (Purdue sendmail) @tab SMTP (RH 8.0) @tab SMTP (SGL qmail) -@item 11 bytes @tab 31 ms @tab 55 ms @tab 781 s @tab 77 s @tab 24 s -@item 407 bytes @tab 37 ms @tab 62 ms @tab 789 s @tab 78 s @tab 25 s -@item 1,221 bytes @tab 46 ms @tab 73 ms @tab 804 s @tab 78 s @tab 25 s -@end multitable - -The benchmarks show that UDP and TCP are, as expected, both significantly -faster compared with any of the SMTP services. Among the SMTP implementations, -there can be significant differences depending on the SMTP configuration. -Filtering with an external tool like procmail that needs to re-parse its -configuration for each mail can be very expensive. Applying spam filters can -also significantly impact the performance of the underlying SMTP -implementation. The microbenchmark shows that SMTP can be a viable solution -for initiating peer-to-peer sessions: a couple of seconds to connect to a peer -are probably not even going to be noticed by users. The next benchmark -measures the possible throughput for a transport. Throughput can be measured -by sending multiple messages in parallel and measuring packet loss. Note that -not only UDP but also the TCP transport can actually loose messages since the -TCP implementation drops messages if the @code{write} to the socket would -block. While the SMTP protocol never drops messages itself, it is often so -slow that only a fraction of the messages can be sent and received in the -given time-bounds. For this benchmark we report the message loss after -allowing t time for sending m messages. If messages were not sent (or -received) after an overall timeout of t, they were considered lost. The -benchmark was performed using two Xeon 2 GHZ machines running RedHat 8.0 with -sendmail. The machines were connected with a direct 100 MBit ethernet -connection.@ Figures udp1200, tcp1200 and smtp-MTUs show that the throughput -for messages of size 1,200 octects is 2,343 kbps, 3,310 kbps and 6 kbps for -UDP, TCP and SMTP respectively. The high per-message overhead of SMTP can be -improved by increasing the MTU, for example, an MTU of 12,000 octets improves -the throughput to 13 kbps as figure smtp-MTUs shows. Our research paper) has -some more details on the benchmarking results. - -@node Bluetooth plugin -@section Bluetooth plugin -@c %**end of header - -This page describes the new Bluetooth transport plugin for GNUnet. The plugin -is still in the testing stage so don't expect it to work perfectly. If you -have any questions or problems just post them here or ask on the IRC channel. -@itemize @bullet -@item What do I need to use the Bluetooth plugin transport? -@item BluetoothHow does it work? -@item What possible errors should I be aware of? -@item How do I configure my peer? -@item How can I test it? -@end itemize - - - -@menu -* What do I need to use the Bluetooth plugin transport?:: -* How does it work2?:: -* What possible errors should I be aware of?:: -* How do I configure my peer2?:: -* How can I test it?:: -* The implementation of the Bluetooth transport plugin:: -@end menu - -@node What do I need to use the Bluetooth plugin transport? -@subsection What do I need to use the Bluetooth plugin transport? -@c %**end of header - -If you are a Linux user and you want to use the Bluetooth transport plugin you -should install the BlueZ development libraries (if they aren't already -installed). For instructions about how to install the libraries you should -check out the BlueZ site (@uref{http://www.bluez.org/, http://www.bluez.org}). -If you don't know if you have the necesarry libraries, don't worry, just run -the GNUnet configure script and you will be able to see a notification at the -end which will warn you if you don't have the necessary libraries. - -If you are a Windows user you should have installed the -@emph{MinGW}/@emph{MSys2} with the latest updates (especially the -@emph{ws2bth} header). If this is your first build of GNUnet on Windows you -should check out the SBuild repository. It will semi-automatically assembles a -@emph{MinGW}/@emph{MSys2} installation with a lot of extra packages which are -needed for the GNUnet build. So this will ease your work!@ Finally you just -have to be sure that you have the correct drivers for your Bluetooth device -installed and that your device is on and in a discoverable mode. The Windows -Bluetooth Stack supports only the RFCOMM protocol so we cannot turn on your -device programatically! - -@node How does it work2? -@subsection How does it work2? -@c %**end of header - -The Bluetooth transport plugin uses virtually the same code as the WLAN plugin -and only the helper binary is different. The helper takes a single argument, -which represents the interface name and is specified in the configuration -file. Here are the basic steps that are followed by the helper binary used on -Linux: - -@itemize @bullet -@item it verifies if the name corresponds to a Bluetooth interface name -@item it verifies if the iterface is up (if it is not, it tries to bring it up) -@item it tries to enable the page and inquiry scan in order to make the device -discoverable and to accept incoming connection requests -@emph{The above operations require root access so you should start the -transport plugin with root privileges.} -@item it finds an available port number and registers a SDP service which will -be used to find out on which port number is the server listening on and switch -the socket in listening mode -@item it sends a HELLO message with its address -@item finally it forwards traffic from the reading sockets to the STDOUT and -from the STDIN to the writing socket -@end itemize - -Once in a while the device will make an inquiry scan to discover the nearby -devices and it will send them randomly HELLO messages for peer discovery. - -@node What possible errors should I be aware of? -@subsection What possible errors should I be aware of? -@c %**end of header - -@emph{This section is dedicated for Linux users} - -Well there are many ways in which things could go wrong but I will try to -present some tools that you could use to debug and some scenarios. -@itemize @bullet - -@item @code{bluetoothd -n -d} : use this command to enable logging in the -foreground and to print the logging messages - -@item @code{hciconfig}: can be used to configure the Bluetooth devices. If you -run it without any arguments it will print information about the state of the -interfaces. So if you receive an error that the device couldn't be brought up -you should try to bring it manually and to see if it works (use @code{hciconfig --a hciX up}). If you can't and the Bluetooth address has the form -00:00:00:00:00:00 it means that there is something wrong with the D-Bus daemon -or with the Bluetooth daemon. Use @code{bluetoothd} tool to see the logs - -@item @code{sdptool} can be used to control and interogate SDP servers. If you -encounter problems regarding the SDP server (like the SDP server is down) you -should check out if the D-Bus daemon is running correctly and to see if the -Bluetooth daemon started correctly(use @code{bluetoothd} tool). Also, sometimes -the SDP service could work but somehow the device couldn't register his -service. Use @code{sdptool browse [dev-address]} to see if the service is -registered. There should be a service with the name of the interface and GNUnet -as provider. - -@item @code{hcitool} : another useful tool which can be used to configure the -device and to send some particular commands to it. - -@item @code{hcidump} : could be used for low level debugging -@end itemize - -@node How do I configure my peer2? -@subsection How do I configure my peer2? -@c %**end of header - -On Linux, you just have to be sure that the interface name corresponds to the -one that you want to use. Use the @code{hciconfig} tool to check that. By -default it is set to hci0 but you can change it. - -A basic configuration looks like this: -@example -[transport-bluetooth] -# Name of the interface (typically hciX) -INTERFACE = hci0 -# Real hardware, no testing -TESTMODE = 0 TESTING_IGNORE_KEYS = ACCEPT_FROM; -@end example - - -In order to use the Bluetooth transport plugin when the transport service is -started, you must add the plugin name to the default transport service plugins -list. For example: -@example -[transport] ... PLUGINS = dns bluetooth ... -@end example - -If you want to use only the Bluetooth plugin set @emph{PLUGINS = bluetooth} - -On Windows, you cannot specify which device to use. The only thing that you -should do is to add @emph{bluetooth} on the plugins list of the transport -service. - -@node How can I test it? -@subsection How can I test it? -@c %**end of header - -If you have two Bluetooth devices on the same machine which use Linux you -must: -@itemize @bullet - -@item create two different file configuration (one which will use the first -interface (@emph{hci0}) and the other which will use the second interface -(@emph{hci1})). Let's name them @emph{peer1.conf} and @emph{peer2.conf}. - -@item run @emph{gnunet-peerinfo -c peerX.conf -s} in order to generate the -peers private keys. The @strong{X} must be replace with 1 or 2. - -@item run @emph{gnunet-arm -c peerX.conf -s -i=transport} in order to start the -transport service. (Make sure that you have "bluetooth" on the transport -plugins list if the Bluetooth transport service doesn't start.) - -@item run @emph{gnunet-peerinfo -c peer1.conf -s} to get the first peer's ID. -If you already know your peer ID (you saved it from the first command), this -can be skipped. - -@item run @emph{gnunet-transport -c peer2.conf -p=PEER1_ID -s} to start sending -data for benchmarking to the other peer. -@end itemize - - -This scenario will try to connect the second peer to the first one and then -start sending data for benchmarking. - -On Windows you cannot test the plugin functionality using two Bluetooth devices -from the same machine because after you install the drivers there will occur -some conflicts between the Bluetooth stacks. (At least that is what happend on -my machine : I wasn't able to use the Bluesoleil stack and the WINDCOMM one in -the same time). - -If you have two different machines and your configuration files are good you -can use the same scenario presented on the begining of this section. - -Another way to test the plugin functionality is to create your own application -which will use the GNUnet framework with the Bluetooth transport service. - -@node The implementation of the Bluetooth transport plugin -@subsection The implementation of the Bluetooth transport plugin -@c %**end of header - -This page describes the implementation of the Bluetooth transport plugin. - -First I want to remind you that the Bluetooth transport plugin uses virtually -the same code as the WLAN plugin and only the helper binary is different. Also -the scope of the helper binary from the Bluetooth transport plugin is the same -as the one used for the wlan transport plugin: it acceses the interface and -then it forwards traffic in both directions between the Bluetooth interface -and stdin/stdout of the process involved. - -The Bluetooth plugin transport could be used both on Linux and Windows -platforms. - -@itemize @bullet -@item Linux functionality -@item Windows functionality -@item Pending Features -@end itemize - - - -@menu -* Linux functionality:: -* THE INITIALIZATION:: -* THE LOOP:: -* Details about the broadcast implementation:: -* Windows functionality:: -* Pending features:: -@end menu - -@node Linux functionality -@subsubsection Linux functionality -@c %**end of header - -In order to implement the plugin functionality on Linux I used the BlueZ -stack. For the communication with the other devices I used the RFCOMM -protocol. Also I used the HCI protocol to gain some control over the device. -The helper binary takes a single argument (the name of the Bluetooth -interface) and is separated in two stages: - -@c %** 'THE INITIALIZATION' should be in bigger letters or stand out, not -@c %** starting a new section? -@node THE INITIALIZATION -@subsubsection THE INITIALIZATION - -@itemize @bullet -@item first, it checks if we have root privilegies (@emph{Remember that we need -to have root privilegies in order to be able to bring the interface up if it is -down or to change its state.}). - -@item second, it verifies if the interface with the given name exists. - -@strong{If the interface with that name exists and it is a Bluetooth -interface:} - -@item it creates a RFCOMM socket which will be used for listening and call the -@emph{open_device} method - -On the @emph{open_device} method: -@itemize @bullet -@item creates a HCI socket used to send control events to the the device -@item searches for the device ID using the interface name -@item saves the device MAC address -@item checks if the interface is down and tries to bring it UP -@item checks if the interface is in discoverable mode and tries to make it -discoverable -@item closes the HCI socket and binds the RFCOMM one -@item switches the RFCOMM socket in listening mode -@item registers the SDP service (the service will be used by the other devices -to get the port on which this device is listening on) -@end itemize - -@item drops the root privilegies - -@strong{If the interface is not a Bluetooth interface the helper exits with a -suitable error} -@end itemize - -@c %** Same as for @node entry above -@node THE LOOP -@subsubsection THE LOOP - -The helper binary uses a list where it saves all the connected neighbour -devices (@emph{neighbours.devices}) and two buffers (@emph{write_pout} and -@emph{write_std}). The first message which is send is a control message with -the device's MAC address in order to announce the peer presence to the -neighbours. Here are a short description of what happens in the main loop: - -@itemize @bullet -@item Every time when it receives something from the STDIN it processes the -data and saves the message in the first buffer (@emph{write_pout}). When it has -something in the buffer, it gets the destination address from the buffer, -searches the destination address in the list (if there is no connection with -that device, it creates a new one and saves it to the list) and sends the -message. -@item Every time when it receives something on the listening socket it accepts -the connection and saves the socket on a list with the reading sockets. -@item Every time when it receives something from a reading socket it parses the -message, verifies the CRC and saves it in the @emph{write_std} buffer in order -to be sent later to the STDOUT. -@end itemize - -So in the main loop we use the select function to wait until one of the file -descriptor saved in one of the two file descriptors sets used is ready to use. -The first set (@emph{rfds}) represents the reading set and it could contain the -list with the reading sockets, the STDIN file descriptor or the listening -socket. The second set (@emph{wfds}) is the writing set and it could contain -the sending socket or the STDOUT file descriptor. After the select function -returns, we check which file descriptor is ready to use and we do what is -supposed to do on that kind of event. @emph{For example:} if it is the -listening socket then we accept a new connection and save the socket in the -reading list; if it is the STDOUT file descriptor, then we write to STDOUT the -message from the @emph{write_std} buffer. - -To find out on which port a device is listening on we connect to the local SDP -server and searche the registered service for that device. - -@emph{You should be aware of the fact that if the device fails to connect to -another one when trying to send a message it will attempt one more time. If it -fails again, then it skips the message.} -@emph{Also you should know that the -transport Bluetooth plugin has support for @strong{broadcast messages}.} - -@node Details about the broadcast implementation -@subsubsection Details about the broadcast implementation -@c %**end of header - -First I want to point out that the broadcast functionality for the CONTROL -messages is not implemented in a conventional way. Since the inquiry scan time -is too big and it will take some time to send a message to all the -discoverable devices I decided to tackle the problem in a different way. Here -is how I did it: - -@itemize @bullet -@item If it is the first time when I have to broadcast a message I make an -inquiry scan and save all the devices' addresses to a vector. -@item After the inquiry scan ends I take the first address from the list and I -try to connect to it. If it fails, I try to connect to the next one. If it -succeeds, I save the socket to a list and send the message to the device. -@item When I have to broadcast another message, first I search on the list for -a new device which I'm not connected to. If there is no new device on the list -I go to the beginning of the list and send the message to the old devices. -After 5 cycles I make a new inquiry scan to check out if there are new -discoverable devices and save them to the list. If there are no new -discoverable devices I reset the cycling counter and go again through the old -list and send messages to the devices saved in it. -@end itemize - -@strong{Therefore}: - -@itemize @bullet -@item every time when I have a broadcast message I look up on the list for a -new device and send the message to it -@item if I reached the end of the list for 5 times and I'm connected to all the -devices from the list I make a new inquiry scan. @emph{The number of the list's -cycles after an inquiry scan could be increased by redefining the MAX_LOOPS -variable} -@item when there are no new devices I send messages to the old ones. -@end itemize - -Doing so, the broadcast control messages will reach the devices but with delay. - -@emph{NOTICE:} When I have to send a message to a certain device first I check -on the broadcast list to see if we are connected to that device. If not we try -to connect to it and in case of success we save the address and the socket on -the list. If we are already connected to that device we simply use the socket. - -@node Windows functionality -@subsubsection Windows functionality -@c %**end of header - -For Windows I decided to use the Microsoft Bluetooth stack which has the -advantage of coming standard from Windows XP SP2. The main disadvantage is -that it only supports the RFCOMM protocol so we will not be able to have a low -level control over the Bluetooth device. Therefore it is the user -responsability to check if the device is up and in the discoverable mode. Also -there are no tools which could be used for debugging in order to read the data -coming from and going to a Bluetooth device, which obviously hindered my work. -Another thing that slowed down the implementation of the plugin (besides that -I wasn't too accomodated with the win32 API) was that there were some bugs on -MinGW regarding the Bluetooth. Now they are solved but you should keep in mind -that you should have the latest updates (especially the @emph{ws2bth} header). - -Besides the fact that it uses the Windows Sockets, the Windows implemenation -follows the same principles as the Linux one: - -@itemize @bullet -@item -It has a initalization part where it initializes the Windows Sockets, creates a -RFCOMM socket which will be binded and switched to the listening mode and -registers a SDP service. -In the Microsoft Bluetooth API there are two ways to work with the SDP: -@itemize @bullet -@item an easy way which works with very simple service records -@item a hard way which is useful when you need to update or to delete the -record -@end itemize -@end itemize - -Since I only needed the SDP service to find out on which port the device is -listening on and that did not change, I decided to use the easy way. In order -to register the service I used the @emph{WSASetService} function and I -generated the @emph{Universally Unique Identifier} with the @emph{guidgen.exe} -Windows's tool. - -In the loop section the only difference from the Linux implementation is that -I used the GNUNET_NETWORK library for functions like @emph{accept}, -@emph{bind}, @emph{connect} or @emph{select}. I decided to use the -GNUNET_NETWORK library because I also needed to interact with the STDIN and -STDOUT handles and on Windows the select function is only defined for sockets, -and it will not work for arbitrary file handles. - -Another difference between Linux and Windows implementation is that in Linux, -the Bluetooth address is represented in 48 bits while in Windows is -represented in 64 bits. Therefore I had to do some changes on -@emph{plugin_transport_wlan} header. - -Also, currently on Windows the Bluetooth plugin doesn't have support for -broadcast messages. When it receives a broadcast message it will skip it. - -@node Pending features -@subsubsection Pending features -@c %**end of header - -@itemize @bullet -@item Implement the broadcast functionality on Windows @emph{(currently working -on)} -@item Implement a testcase for the helper :@ @emph{@ The testcase consists of a -program which emaluates the plugin and uses the helper. It will simulate -connections, disconnections and data transfers.@ } -@end itemize - -If you have a new idea about a feature of the plugin or suggestions about how -I could improve the implementation you are welcome to comment or to contact -me. - -@node WLAN plugin -@section WLAN plugin -@c %**end of header - -This section documents how the wlan transport plugin works. Parts which are not -implemented yet or could be better implemented are described at the end. - -@node The ATS Subsystem -@section The ATS Subsystem -@c %**end of header - -ATS stands for "automatic transport selection", and the function of ATS in -GNUnet is to decide on which address (and thus transport plugin) should be used -for two peers to communicate, and what bandwidth limits should be imposed on -such an individual connection. To help ATS make an informed decision, -higher-level services inform the ATS service about their requirements and the -quality of the service rendered. The ATS service also interacts with the -transport service to be appraised of working addresses and to communicate its -resource allocation decisions. Finally, the ATS service's operation can be -observed using a monitoring API. - -The main logic of the ATS service only collects the available addresses, their -performance characteristics and the applications requirements, but does not -make the actual allocation decision. This last critical step is left to an ATS -plugin, as we have implemented (currently three) different allocation -strategies which differ significantly in their performance and maturity, and it -is still unclear if any particular plugin is generally superior. - -@node GNUnet's CORE Subsystem -@section GNUnet's CORE Subsystem -@c %**end of header - -The CORE subsystem in GNUnet is responsible for securing link-layer -communications between nodes in the GNUnet overlay network. CORE builds on the -TRANSPORT subsystem which provides for the actual, insecure, unreliable -link-layer communication (for example, via UDP or WLAN), and then adds -fundamental security to the connections: - -@itemize @bullet -@item confidentiality with so-called perfect forward secrecy; we use -@uref{http://en.wikipedia.org/wiki/Elliptic_curve_Diffie%E2%80%93Hellman, -ECDHE} powered by @uref{http://cr.yp.to/ecdh.html, Curve25519} for the key -exchange and then use symmetric encryption, encrypting with both -@uref{http://en.wikipedia.org/wiki/Rijndael, AES-256} and -@uref{http://en.wikipedia.org/wiki/Twofish, Twofish} -@item @uref{http://en.wikipedia.org/wiki/Authentication, authentication} is -achieved by signing the ephemeral keys using @uref{http://ed25519.cr.yp.to/, -Ed25519}, a deterministic variant of @uref{http://en.wikipedia.org/wiki/ECDSA, -ECDSA} -@item integrity protection (using @uref{http://en.wikipedia.org/wiki/SHA-2, -SHA-512} to do @uref{http://en.wikipedia.org/wiki/Authenticated_encryption, -encrypt-then-MAC)} -@item @uref{http://en.wikipedia.org/wiki/Replay_attack, replay} protection -(using nonces, timestamps, challenge-response, message counters and ephemeral -keys) -@item liveness (keep-alive messages, timeout) -@end itemize - -@menu -* Limitations:: -* When is a peer "connected"?:: -* libgnunetcore:: -* The CORE Client-Service Protocol:: -* The CORE Peer-to-Peer Protocol:: -@end menu - -@node Limitations -@subsection Limitations -@c %**end of header - -CORE does not perform @uref{http://en.wikipedia.org/wiki/Routing, routing}; -using CORE it is only possible to communicate with peers that happen to -already be "directly" connected with each other. CORE also does not have an -API to allow applications to establish such "direct" connections --- for this, -applications can ask TRANSPORT, but TRANSPORT might not be able to establish a -"direct" connection. The TOPOLOGY subsystem is responsible for trying to keep -a few "direct" connections open at all times. Applications that need to talk -to particular peers should use the CADET subsystem, as it can establish -arbitrary "indirect" connections. - -Because CORE does not perform routing, CORE must only be used directly by -applications that either perform their own routing logic (such as anonymous -file-sharing) or that do not require routing, for example because they are -based on flooding the network. CORE communication is unreliable and delivery -is possibly out-of-order. Applications that require reliable communication -should use the CADET service. Each application can only queue one message per -target peer with the CORE service at any time; messages cannot be larger than -approximately 63 kilobytes. If messages are small, CORE may group multiple -messages (possibly from different applications) prior to encryption. If -permitted by the application (using the @uref{http://baus.net/on-tcp_cork/, -cork} option), CORE may delay transmissions to facilitate grouping of multiple -small messages. If cork is not enabled, CORE will transmit the message as soon -as TRANSPORT allows it (TRANSPORT is responsible for limiting bandwidth and -congestion control). CORE does not allow flow control; applications are -expected to process messages at line-speed. If flow control is needed, -applications should use the CADET service. - -@node When is a peer "connected"? -@subsection When is a peer "connected"? -@c %**end of header - -In addition to the security features mentioned above, CORE also provides one -additional key feature to applications using it, and that is a limited form of -protocol-compatibility checking. CORE distinguishes between TRANSPORT-level -connections (which enable communication with other peers) and -application-level connections. Applications using the CORE API will -(typically) learn about application-level connections from CORE, and not about -TRANSPORT-level connections. When a typical application uses CORE, it will -specify a set of message types (from @code{gnunet_protocols.h}) that it -understands. CORE will then notify the application about connections it has -with other peers if and only if those applications registered an intersecting -set of message types with their CORE service. Thus, it is quite possible that -CORE only exposes a subset of the established direct connections to a -particular application --- and different applications running above CORE might -see different sets of connections at the same time. - -A special case are applications that do not register a handler for any message -type. CORE assumes that these applications merely want to monitor connections -(or "all" messages via other callbacks) and will notify those applications -about all connections. This is used, for example, by the @code{gnunet-core} -command-line tool to display the active connections. Note that it is also -possible that the TRANSPORT service has more active connections than the CORE -service, as the CORE service first has to perform a key exchange with -connecting peers before exchanging information about supported message types -and notifying applications about the new connection. - -@node libgnunetcore -@subsection libgnunetcore -@c %**end of header - -The CORE API (defined in @code{gnunet_core_service.h}) is the basic messaging -API used by P2P applications built using GNUnet. It provides applications the -ability to send and receive encrypted messages to the peer's "directly" -connected neighbours. - -As CORE connections are generally "direct" connections,@ applications must not -assume that they can connect to arbitrary peers this way, as "direct" -connections may not always be possible. Applications using CORE are notified -about which peers are connected. Creating new "direct" connections must be -done using the TRANSPORT API. - -The CORE API provides unreliable, out-of-order delivery. While the -implementation tries to ensure timely, in-order delivery, both message losses -and reordering are not detected and must be tolerated by the application. Most -important, the core will NOT perform retransmission if messages could not be -delivered. - -Note that CORE allows applications to queue one message per connected peer. -The rate at which each connection operates is influenced by the preferences -expressed by local application as well as restrictions imposed by the other -peer. Local applications can express their preferences for particular -connections using the "performance" API of the ATS service. - -Applications that require more sophisticated transmission capabilities such as -TCP-like behavior, or if you intend to send messages to arbitrary remote -peers, should use the CADET API. - -The typical use of the CORE API is to connect to the CORE service using -@code{GNUNET_CORE_connect}, process events from the CORE service (such as -peers connecting, peers disconnecting and incoming messages) and send messages -to connected peers using @code{GNUNET_CORE_notify_transmit_ready}. Note that -applications must cancel pending transmission requests if they receive a -disconnect event for a peer that had a transmission pending; furthermore, -queueing more than one transmission request per peer per application using the -service is not permitted. - -The CORE API also allows applications to monitor all communications of the -peer prior to encryption (for outgoing messages) or after decryption (for -incoming messages). This can be useful for debugging, diagnostics or to -establish the presence of cover traffic (for anonymity). As monitoring -applications are often not interested in the payload, the monitoring callbacks -can be configured to only provide the message headers (including the message -type and size) instead of copying the full data stream to the monitoring -client. - -The init callback of the @code{GNUNET_CORE_connect} function is called with -the hash of the public key of the peer. This public key is used to identify -the peer globally in the GNUnet network. Applications are encouraged to check -that the provided hash matches the hash that they are using (as theoretically -the application may be using a different configuration file with a different -private key, which would result in hard to find bugs). - -As with most service APIs, the CORE API isolates applications from crashes of -the CORE service. If the CORE service crashes, the application will see -disconnect events for all existing connections. Once the connections are -re-established, the applications will be receive matching connect events. - -@node The CORE Client-Service Protocol -@subsection The CORE Client-Service Protocol -@c %**end of header - -This section describes the protocol between an application using the CORE -service (the client) and the CORE service process itself. - - -@menu -* Setup2:: -* Notifications:: -* Sending:: -@end menu - -@node Setup2 -@subsubsection Setup2 -@c %**end of header - -When a client connects to the CORE service, it first sends a -@code{InitMessage} which specifies options for the connection and a set of -message type values which are supported by the application. The options -bitmask specifies which events the client would like to be notified about. The -options include: - -@table @asis -@item GNUNET_CORE_OPTION_NOTHING No notifications -@item GNUNET_CORE_OPTION_STATUS_CHANGE Peers connecting and disconnecting -@item GNUNET_CORE_OPTION_FULL_INBOUND All inbound messages (after decryption) with -full payload -@item GNUNET_CORE_OPTION_HDR_INBOUND Just the @code{MessageHeader} -of all inbound messages -@item GNUNET_CORE_OPTION_FULL_OUTBOUND All outbound -messages (prior to encryption) with full payload -@item GNUNET_CORE_OPTION_HDR_OUTBOUND Just the @code{MessageHeader} of all outbound -messages -@end table - -Typical applications will only monitor for connection status changes. - -The CORE service responds to the @code{InitMessage} with an -@code{InitReplyMessage} which contains the peer's identity. Afterwards, both -CORE and the client can send messages. - -@node Notifications -@subsubsection Notifications -@c %**end of header - -The CORE will send @code{ConnectNotifyMessage}s and -@code{DisconnectNotifyMessage}s whenever peers connect or disconnect from the -CORE (assuming their type maps overlap with the message types registered by -the client). When the CORE receives a message that matches the set of message -types specified during the @code{InitMessage} (or if monitoring is enabled in -for inbound messages in the options), it sends a @code{NotifyTrafficMessage} -with the peer identity of the sender and the decrypted payload. The same -message format (except with @code{GNUNET_MESSAGE_TYPE_CORE_NOTIFY_OUTBOUND} -for the message type) is used to notify clients monitoring outbound messages; -here, the peer identity given is that of the receiver. - -@node Sending -@subsubsection Sending -@c %**end of header - -When a client wants to transmit a message, it first requests a transmission -slot by sending a @code{SendMessageRequest} which specifies the priority, -deadline and size of the message. Note that these values may be ignored by -CORE. When CORE is ready for the message, it answers with a -@code{SendMessageReady} response. The client can then transmit the payload -with a @code{SendMessage} message. Note that the actual message size in the -@code{SendMessage} is allowed to be smaller than the size in the original -request. A client may at any time send a fresh @code{SendMessageRequest}, -which then superceeds the previous @code{SendMessageRequest}, which is then no -longer valid. The client can tell which @code{SendMessageRequest} the CORE -service's @code{SendMessageReady} message is for as all of these messages -contain a "unique" request ID (based on a counter incremented by the client -for each request). - -@node The CORE Peer-to-Peer Protocol -@subsection The CORE Peer-to-Peer Protocol -@c %**end of header - - -@menu -* Creating the EphemeralKeyMessage:: -* Establishing a connection:: -* Encryption and Decryption:: -* Type maps:: -@end menu - -@node Creating the EphemeralKeyMessage -@subsubsection Creating the EphemeralKeyMessage -@c %**end of header - -When the CORE service starts, each peer creates a fresh ephemeral (ECC) -public-private key pair and signs the corresponding @code{EphemeralKeyMessage} -with its long-term key (which we usually call the peer's identity; the hash of -the public long term key is what results in a @code{struct -GNUNET_PeerIdentity} in all GNUnet APIs. The ephemeral key is ONLY used for an -@uref{http://en.wikipedia.org/wiki/Elliptic_curve_Diffie%E2%80%93Hellman, -ECDHE} exchange by the CORE service to establish symmetric session keys. A -peer will use the same @code{EphemeralKeyMessage} for all peers for -@code{REKEY_FREQUENCY}, which is usually 12 hours. After that time, it will -create a fresh ephemeral key (forgetting the old one) and broadcast the new -@code{EphemeralKeyMessage} to all connected peers, resulting in fresh -symmetric session keys. Note that peers independently decide on when to -discard ephemeral keys; it is not a protocol violation to discard keys more -often. Ephemeral keys are also never stored to disk; restarting a peer will -thus always create a fresh ephemeral key. The use of ephemeral keys is what -provides @uref{http://en.wikipedia.org/wiki/Forward_secrecy, forward secrecy}. - -Just before transmission, the @code{EphemeralKeyMessage} is patched to reflect -the current sender_status, which specifies the current state of the connection -from the point of view of the sender. The possible values are: - -@table @asis -@item KX_STATE_DOWN Initial value, never used on the network -@item KX_STATE_KEY_SENT We sent our ephemeral key, do not know the key of the other -peer -@item KX_STATE_KEY_RECEIVED This peer has received a valid ephemeral key -of the other peer, but we are waiting for the other peer to confirm it's -authenticity (ability to decode) via challenge-response. -@item KX_STATE_UP The -connection is fully up from the point of view of the sender (now performing -keep-alives) -@item KX_STATE_REKEY_SENT The sender has initiated a rekeying -operation; the other peer has so far failed to confirm a working connection -using the new ephemeral key -@end table - -@node Establishing a connection -@subsubsection Establishing a connection -@c %**end of header - -Peers begin their interaction by sending a @code{EphemeralKeyMessage} to the -other peer once the TRANSPORT service notifies the CORE service about the -connection. A peer receiving an @code{EphemeralKeyMessage} with a status -indicating that the sender does not have the receiver's ephemeral key, the -receiver's @code{EphemeralKeyMessage} is sent in response.@ Additionally, if -the receiver has not yet confirmed the authenticity of the sender, it also -sends an (encrypted)@code{PingMessage} with a challenge (and the identity of -the target) to the other peer. Peers receiving a @code{PingMessage} respond -with an (encrypted) @code{PongMessage} which includes the challenge. Peers -receiving a @code{PongMessage} check the challenge, and if it matches set the -connection to @code{KX_STATE_UP}. - -@node Encryption and Decryption -@subsubsection Encryption and Decryption -@c %**end of header - -All functions related to the key exchange and encryption/decryption of -messages can be found in @code{gnunet-service-core_kx.c} (except for the -cryptographic primitives, which are in @code{util/crypto*.c}).@ Given the key -material from ECDHE, a -@uref{http://en.wikipedia.org/wiki/Key_derivation_function, Key derivation -function} is used to derive two pairs of encryption and decryption keys for -AES-256 and TwoFish, as well as initialization vectors and authentication keys -(for @uref{http://en.wikipedia.org/wiki/HMAC, HMAC}). The HMAC is computed -over the encrypted payload. Encrypted messages include an iv_seed and the HMAC -in the header. - -Each encrypted message in the CORE service includes a sequence number and a -timestamp in the encrypted payload. The CORE service remembers the largest -observed sequence number and a bit-mask which represents which of the previous -32 sequence numbers were already used. Messages with sequence numbers lower -than the largest observed sequence number minus 32 are discarded. Messages -with a timestamp that is less than @code{REKEY_TOLERANCE} off (5 minutes) are -also discarded. This of course means that system clocks need to be reasonably -synchronized for peers to be able to communicate. Additionally, as the -ephemeral key changes every 12h, a peer would not even be able to decrypt -messages older than 12h. - -@node Type maps -@subsubsection Type maps -@c %**end of header - -Once an encrypted connection has been established, peers begin to exchange -type maps. Type maps are used to allow the CORE service to determine which -(encrypted) connections should be shown to which applications. A type map is -an array of 65536 bits representing the different types of messages understood -by applications using the CORE service. Each CORE service maintains this map, -simply by setting the respective bit for each message type supported by any of -the applications using the CORE service. Note that bits for message types -embedded in higher-level protocols (such as MESH) will not be included in -these type maps. - -Typically, the type map of a peer will be sparse. Thus, the CORE service -attempts to compress its type map using @code{gzip}-style compression -("deflate") prior to transmission. However, if the compression fails to -compact the map, the map may also be transmitted without compression -(resulting in @code{GNUNET_MESSAGE_TYPE_CORE_COMPRESSED_TYPE_MAP} or -@code{GNUNET_MESSAGE_TYPE_CORE_BINARY_TYPE_MAP} messages respectively). Upon -receiving a type map, the respective CORE service notifies applications about -the connection to the other peer if they support any message type indicated in -the type map (or no message type at all). If the CORE service experience a -connect or disconnect event from an application, it updates its type map -(setting or unsetting the respective bits) and notifies its neighbours about -the change. The CORE services of the neighbours then in turn generate connect -and disconnect events for the peer that sent the type map for their respective -applications. As CORE messages may be lost, the CORE service confirms -receiving a type map by sending back a -@code{GNUNET_MESSAGE_TYPE_CORE_CONFIRM_TYPE_MAP}. If such a confirmation (with -the correct hash of the type map) is not received, the sender will retransmit -the type map (with exponential back-off). - -@node GNUnet's CADET subsystem -@section GNUnet's CADET subsystem - -The CADET subsystem in GNUnet is responsible for secure end-to-end -communications between nodes in the GNUnet overlay network. CADET builds on the -CORE subsystem which provides for the link-layer communication and then adds -routing, forwarding and additional security to the connections. CADET offers -the same cryptographic services as CORE, but on an end-to-end level. This is -done so peers retransmitting traffic on behalf of other peers cannot access the -payload data. - -@itemize @bullet -@item CADET provides confidentiality with so-called perfect forward secrecy; we -use ECDHE powered by Curve25519 for the key exchange and then use symmetric -encryption, encrypting with both AES-256 and Twofish -@item authentication is achieved by signing the ephemeral keys using Ed25519, a -deterministic variant of ECDSA -@item integrity protection (using SHA-512 to do encrypt-then-MAC, although only -256 bits are sent to reduce overhead) -@item replay protection (using nonces, timestamps, challenge-response, message -counters and ephemeral keys) -@item liveness (keep-alive messages, timeout) -@end itemize - -Additional to the CORE-like security benefits, CADET offers other properties -that make it a more universal service than CORE. - -@itemize @bullet -@item CADET can establish channels to arbitrary peers in GNUnet. If a peer is -not immediately reachable, CADET will find a path through the network and ask -other peers to retransmit the traffic on its behalf. -@item CADET offers (optional) reliability mechanisms. In a reliable channel -traffic is guaranteed to arrive complete, unchanged and in-order. -@item CADET takes care of flow and congestion control mechanisms, not allowing -the sender to send more traffic than the receiver or the network are able to -process. -@end itemize - -@menu -* libgnunetcadet:: -@end menu - -@node libgnunetcadet -@subsection libgnunetcadet - - -The CADET API (defined in gnunet_cadet_service.h) is the messaging API used by -P2P applications built using GNUnet. It provides applications the ability to -send and receive encrypted messages to any peer participating in GNUnet. The -API is heavily base on the CORE API. - -CADET delivers messages to other peers in "channels". A channel is a permanent -connection defined by a destination peer (identified by its public key) and a -port number. Internally, CADET tunnels all channels towards a destiantion peer -using one session key and relays the data on multiple "connections", -independent from the channels. - -Each channel has optional paramenters, the most important being the reliability -flag. Should a message get lost on TRANSPORT/CORE level, if a channel is -created with as reliable, CADET will retransmit the lost message and deliver it -in order to the destination application. - -To communicate with other peers using CADET, it is necessary to first connect -to the service using @code{GNUNET_CADET_connect}. This function takes several -parameters in form of callbacks, to allow the client to react to various -events, like incoming channels or channels that terminate, as well as specify a -list of ports the client wishes to listen to (at the moment it is not possible -to start listening on further ports once connected, but nothing prevents a -client to connect several times to CADET, even do one connection per listening -port). The function returns a handle which has to be used for any further -interaction with the service. - -To connect to a remote peer a client has to call the -@code{GNUNET_CADET_channel_create} function. The most important parameters -given are the remote peer's identity (it public key) and a port, which -specifies which application on the remote peer to connect to, similar to -TCP/UDP ports. CADET will then find the peer in the GNUnet network and -establish the proper low-level connections and do the necessary key exchanges -to assure and authenticated, secure and verified communication. Similar to -@code{GNUNET_CADET_connect},@code{GNUNET_CADET_create_channel} returns a handle -to interact with the created channel. - -For every message the client wants to send to the remote application, -@code{GNUNET_CADET_notify_transmit_ready} must be called, indicating the -channel on which the message should be sent and the size of the message (but -not the message itself!). Once CADET is ready to send the message, the provided -callback will fire, and the message contents are provided to this callback. - -Please note the CADET does not provide an explicit notification of when a -channel is connected. In loosely connected networks, like big wireless mesh -networks, this can take several seconds, even minutes in the worst case. To be -alerted when a channel is online, a client can call -@code{GNUNET_CADET_notify_transmit_ready} immediately after -@code{GNUNET_CADET_create_channel}. When the callback is activated, it means -that the channel is online. The callback can give 0 bytes to CADET if no -message is to be sent, this is ok. - -If a transmission was requested but before the callback fires it is no longer -needed, it can be cancelled with -@code{GNUNET_CADET_notify_transmit_ready_cancel}, which uses the handle given -back by @code{GNUNET_CADET_notify_transmit_ready}. As in the case of CORE, only -one message can be requested at a time: a client must not call -@code{GNUNET_CADET_notify_transmit_ready} again until the callback is called or -the request is cancelled. - -When a channel is no longer needed, a client can call -@code{GNUNET_CADET_channel_destroy} to get rid of it. Note that CADET will try -to transmit all pending traffic before notifying the remote peer of the -destruction of the channel, including retransmitting lost messages if the -channel was reliable. - -Incoming channels, channels being closed by the remote peer, and traffic on any -incoming or outgoing channels are given to the client when CADET executes the -callbacks given to it at the time of @code{GNUNET_CADET_connect}. - -Finally, when an application no longer wants to use CADET, it should call -@code{GNUNET_CADET_disconnect}, but first all channels and pending -transmissions must be closed (otherwise CADET will complain). - -@node GNUnet's NSE subsystem -@section GNUnet's NSE subsystem - - -NSE stands for Network Size Estimation. The NSE subsystem provides other -subsystems and users with a rough estimate of the number of peers currently -participating in the GNUnet overlay. The computed value is not a precise number -as producing a precise number in a decentralized, efficient and secure way is -impossible. While NSE's estimate is inherently imprecise, NSE also gives the -expected range. For a peer that has been running in a stable network for a -while, the real network size will typically (99.7% of the time) be in the range -of [2/3 estimate, 3/2 estimate]. We will now give an overview of the algorithm -used to calcualte the estimate; all of the details can be found in this -technical report. - -@menu -* Motivation:: -* Principle:: -* libgnunetnse:: -* The NSE Client-Service Protocol:: -* The NSE Peer-to-Peer Protocol:: -@end menu - -@node Motivation -@subsection Motivation - - -Some subsytems, like DHT, need to know the size of the GNUnet network to -optimize some parameters of their own protocol. The decentralized nature of -GNUnet makes efficient and securely counting the exact number of peers -infeasable. Although there are several decentralized algorithms to count the -number of peers in a system, so far there is none to do so securely. Other -protocols may allow any malicious peer to manipulate the final result or to -take advantage of the system to perform DoS (Denial of Service) attacks against -the network. GNUnet's NSE protocol avoids these drawbacks. - - - -@menu -* Security:: -@end menu - -@node Security -@subsubsection Security - - -The NSE subsystem is designed to be resilient against these attacks. It uses -@uref{http://en.wikipedia.org/wiki/Proof-of-work_system, proofs of work} to -prevent one peer from impersonating a large number of participants, which would -otherwise allow an adversary to artifically inflate the estimate. The DoS -protection comes from the time-based nature of the protocol: the estimates are -calculated periodically and out-of-time traffic is either ignored or stored for -later retransmission by benign peers. In particular, peers cannot trigger -global network communication at will. - -@node Principle -@subsection Principle - - -The algorithm calculates the estimate by finding the globally closest peer ID -to a random, time-based value. - -The idea is that the closer the ID is to the random value, the more "densely -packed" the ID space is, and therefore, more peers are in the network. - - - -@menu -* Example:: -* Algorithm:: -* Target value:: -* Timing:: -* Controlled Flooding:: -* Calculating the estimate:: -@end menu - -@node Example -@subsubsection Example - - -Suppose all peers have IDs between 0 and 100 (our ID space), and the random -value is 42. If the closest peer has the ID 70 we can imagine that the average -"distance" between peers is around 30 and therefore the are around 3 peers in -the whole ID space. On the other hand, if the closest peer has the ID 44, we -can imagine that the space is rather packed with peers, maybe as much as 50 of -them. Naturally, we could have been rather unlucky, and there is only one peer -and happens to have the ID 44. Thus, the current estimate is calculated as the -average over multiple rounds, and not just a single sample. - -@node Algorithm -@subsubsection Algorithm - - -Given that example, one can imagine that the job of the subsystem is to -efficiently communicate the ID of the closest peer to the target value to all -the other peers, who will calculate the estimate from it. - -@node Target value -@subsubsection Target value - -@c %**end of header - -The target value itself is generated by hashing the current time, rounded down -to an agreed value. If the rounding amount is 1h (default) and the time is -12:34:56, the time to hash would be 12:00:00. The process is repeated each -rouning amount (in this example would be every hour). Every repetition is -called a round. - -@node Timing -@subsubsection Timing -@c %**end of header - -The NSE subsystem has some timing control to avoid everybody broadcasting its -ID all at one. Once each peer has the target random value, it compares its own -ID to the target and calculates the hypothetical size of the network if that -peer were to be the closest. Then it compares the hypothetical size with the -estimate from the previous rounds. For each value there is an assiciated point -in the period, let's call it "broadcast time". If its own hypothetical estimate -is the same as the previous global estimate, its "broadcast time" will be in -the middle of the round. If its bigger it will be earlier and if its smaler -(the most likely case) it will be later. This ensures that the peers closests -to the target value start broadcasting their ID the first. - -@node Controlled Flooding -@subsubsection Controlled Flooding - -@c %**end of header - -When a peer receives a value, first it verifies that it is closer than the -closest value it had so far, otherwise it answers the incoming message with a -message containing the better value. Then it checks a proof of work that must -be included in the incoming message, to ensure that the other peer's ID is not -made up (otherwise a malicious peer could claim to have an ID of exactly the -target value every round). Once validated, it compares the brodcast time of the -received value with the current time and if it's not too early, sends the -received value to its neighbors. Otherwise it stores the value until the -correct broadcast time comes. This prevents unnecessary traffic of sub-optimal -values, since a better value can come before the broadcast time, rendering the -previous one obsolete and saving the traffic that would have been used to -broadcast it to the neighbors. - -@node Calculating the estimate -@subsubsection Calculating the estimate - -@c %**end of header - -Once the closest ID has been spread across the network each peer gets the exact -distance betweed this ID and the target value of the round and calculates the -estimate with a mathematical formula described in the tech report. The estimate -generated with this method for a single round is not very precise. Remember the -case of the example, where the only peer is the ID 44 and we happen to generate -the target value 42, thinking there are 50 peers in the network. Therefore, the -NSE subsystem remembers the last 64 estimates and calculates an average over -them, giving a result of which usually has one bit of uncertainty (the real -size could be half of the estimate or twice as much). Note that the actual -network size is calculated in powers of two of the raw input, thus one bit of -uncertainty means a factor of two in the size estimate. - -@node libgnunetnse -@subsection libgnunetnse - -@c %**end of header - -The NSE subsystem has the simplest API of all services, with only two calls: -@code{GNUNET_NSE_connect} and @code{GNUNET_NSE_disconnect}. - -The connect call gets a callback function as a parameter and this function is -called each time the network agrees on an estimate. This usually is once per -round, with some exceptions: if the closest peer has a late local clock and -starts spreading his ID after everyone else agreed on a value, the callback -might be activated twice in a round, the second value being always bigger than -the first. The default round time is set to 1 hour. - -The disconnect call disconnects from the NSE subsystem and the callback is no -longer called with new estimates. - - - -@menu -* Results:: -* Examples2:: -@end menu - -@node Results -@subsubsection Results - -@c %**end of header - -The callback provides two values: the average and the -@uref{http://en.wikipedia.org/wiki/Standard_deviation, standard deviation} of -the last 64 rounds. The values provided by the callback function are -logarithmic, this means that the real estimate numbers can be obtained by -calculating 2 to the power of the given value (2average). From a statistics -point of view this means that: - -@itemize @bullet -@item 68% of the time the real size is included in the interval -[(2average-stddev), 2] -@item 95% of the time the real size is included in the interval -[(2average-2*stddev, 2^average+2*stddev] -@item 99.7% of the time the real size is included in the interval -[(2average-3*stddev, 2average+3*stddev] -@end itemize - -The expected standard variation for 64 rounds in a network of stable size is -0.2. Thus, we can say that normally: - -@itemize @bullet -@item 68% of the time the real size is in the range [-13%, +15%] -@item 95% of the time the real size is in the range [-24%, +32%] -@item 99.7% of the time the real size is in the range [-34%, +52%] -@end itemize - -As said in the introduction, we can be quite sure that usually the real size is -between one third and three times the estimate. This can of course vary with -network conditions. Thus, applications may want to also consider the provided -standard deviation value, not only the average (in particular, if the standard -veriation is very high, the average maybe meaningless: the network size is -changing rapidly). - -@node Examples2 -@subsubsection Examples2 - -@c %**end of header - -Let's close with a couple examples. - -@table @asis - -@item Average: 10, std dev: 1 Here the estimate would be 2^10 = 1024 peers.@ -The range in which we can be 95% sure is: [2^8, 2^12] = [256, 4096]. We can be -very (>99.7%) sure that the network is not a hundred peers and absolutely sure -that it is not a million peers, but somewhere around a thousand. - -@item Average 22, std dev: 0.2 Here the estimate would be 2^22 = 4 Million peers.@ -The range in which we can be 99.7% sure is: [2^21.4, 2^22.6] = [2.8M, 6.3M]. -We can be sure that the network size is around four million, with absolutely -way of it being 1 million. - -@end table - -To put this in perspective, if someone remembers the LHC Higgs boson results, -were announced with "5 sigma" and "6 sigma" certainties. In this case a 5 sigma -minimum would be 2 million and a 6 sigma minimum, 1.8 million. - -@node The NSE Client-Service Protocol -@subsection The NSE Client-Service Protocol - -@c %**end of header - -As with the API, the client-service protocol is very simple, only has 2 -different messages, defined in @code{src/nse/nse.h}: - -@itemize @bullet -@item @code{GNUNET_MESSAGE_TYPE_NSE_START}@ This message has no parameters and -is sent from the client to the service upon connection. -@item @code{GNUNET_MESSAGE_TYPE_NSE_ESTIMATE}@ This message is sent from the -service to the client for every new estimate and upon connection. Contains a -timestamp for the estimate, the average and the standard deviation for the -respective round. -@end itemize - -When the @code{GNUNET_NSE_disconnect} API call is executed, the client simply -disconnects from the service, with no message involved. - -@node The NSE Peer-to-Peer Protocol -@subsection The NSE Peer-to-Peer Protocol - -@c %**end of header - -The NSE subsystem only has one message in the P2P protocol, the -@code{GNUNET_MESSAGE_TYPE_NSE_P2P_FLOOD} message. - -This message key contents are the timestamp to identify the round (differences -in system clocks may cause some peers to send messages way too early or way too -late, so the timestamp allows other peers to identify such messages easily), -the @uref{http://en.wikipedia.org/wiki/Proof-of-work_system, proof of work} -used to make it difficult to mount a -@uref{http://en.wikipedia.org/wiki/Sybil_attack, Sybil attack}, and the public -key, which is used to verify the signature on the message. - -Every peer stores a message for the previous, current and next round. The -messages for the previous and current round are given to peers that connect to -us. The message for the next round is simply stored until our system clock -advances to the next round. The message for the current round is what we are -flooding the network with right now. At the beginning of each round the peer -does the following: - -@itemize @bullet -@item calculates his own distance to the target value -@item creates, signs and stores the message for the current round (unless it -has a better message in the "next round" slot which came early in the previous -round) -@item calculates, based on the stored round message (own or received) when to -stard flooding it to its neighbors -@end itemize - -Upon receiving a message the peer checks the validity of the message (round, -proof of work, signature). The next action depends on the contents of the -incoming message: - -@itemize @bullet -@item if the message is worse than the current stored message, the peer sends -the current message back immediately, to stop the other peer from spreading -suboptimal results -@item if the message is better than the current stored message, the peer stores -the new message and calculates the new target time to start spreading it to its -neighbors (excluding the one the message came from) -@item if the message is for the previous round, it is compared to the message -stored in the "previous round slot", which may then be updated -@item if the message is for the next round, it is compared to the message -stored in the "next round slot", which again may then be updated -@end itemize - -Finally, when it comes to send the stored message for the current round to the -neighbors there is a random delay added for each neighbor, to avoid traffic -spikes and minimize cross-messages. - -@node GNUnet's HOSTLIST subsystem -@section GNUnet's HOSTLIST subsystem - -@c %**end of header - -Peers in the GNUnet overlay network need address information so that they can -connect with other peers. GNUnet uses so called HELLO messages to store and -exchange peer addresses. GNUnet provides several methods for peers to obtain -this information: - -@itemize @bullet -@item out-of-band exchange of HELLO messages (manually, using for example -gnunet-peerinfo) -@item HELLO messages shipped with GNUnet (automatic with distribution) -@item UDP neighbor discovery in LAN (IPv4 broadcast, IPv6 multicast) -@item topology gossiping (learning from other peers we already connected to), -and -@item the HOSTLIST daemon covered in this section, which is particularly -relevant for bootstrapping new peers. -@end itemize - -New peers have no existing connections (and thus cannot learn from gossip among -peers), may not have other peers in their LAN and might be started with an -outdated set of HELLO messages from the distribution. In this case, getting new -peers to connect to the network requires either manual effort or the use of a -HOSTLIST to obtain HELLOs. - -@menu -* HELLOs:: -* Overview for the HOSTLIST subsystem:: -* Interacting with the HOSTLIST daemon:: -* Hostlist security address validation:: -* The HOSTLIST daemon:: -* The HOSTLIST server:: -* The HOSTLIST client:: -* Usage:: -@end menu - -@node HELLOs -@subsection HELLOs - -@c %**end of header - -The basic information peers require to connect to other peers are contained in -so called HELLO messages you can think of as a business card. Besides the -identity of the peer (based on the cryptographic public key) a HELLO message -may contain address information that specifies ways to contact a peer. By -obtaining HELLO messages, a peer can learn how to contact other peers. - -@node Overview for the HOSTLIST subsystem -@subsection Overview for the HOSTLIST subsystem - -@c %**end of header - -The HOSTLIST subsystem provides a way to distribute and obtain contact -information to connect to other peers using a simple HTTP GET request. It's -implementation is split in three parts, the main file for the daemon itself -(gnunet-daemon-hostlist.c), the HTTP client used to download peer information -(hostlist-client.c) and the server component used to provide this information -to other peers (hostlist-server.c). The server is basically a small HTTP web -server (based on GNU libmicrohttpd) which provides a list of HELLOs known to -the local peer for download. The client component is basically a HTTP client -(based on libcurl) which can download hostlists from one or more websites. The -hostlist format is a binary blob containing a sequence of HELLO messages. Note -that any HTTP server can theoretically serve a hostlist, the build-in hostlist -server makes it simply convenient to offer this service. - - -@menu -* Features:: -* Limitations2:: -@end menu - -@node Features -@subsubsection Features - -@c %**end of header - -The HOSTLIST daemon can: - -@itemize @bullet -@item provide HELLO messages with validated addresses obtained from PEERINFO to -download for other peers -@item download HELLO messages and forward these message to the TRANSPORT -subsystem for validation -@item advertises the URL of this peer's hostlist address to other peers via -gossip -@item automatically learn about hostlist servers from the gossip of other peers -@end itemize - -@node Limitations2 -@subsubsection Limitations2 - -@c %**end of header - -The HOSTLIST daemon does not: - -@itemize @bullet -@item verify the cryptographic information in the HELLO messages -@item verify the address information in the HELLO messages -@end itemize - -@node Interacting with the HOSTLIST daemon -@subsection Interacting with the HOSTLIST daemon - -@c %**end of header - -The HOSTLIST subsystem is currently implemented as a daemon, so there is no -need for the user to interact with it and therefore there is no command line -tool and no API to communicate with the daemon. In the future, we can envision -changing this to allow users to manually trigger the download of a hostlist. - -Since there is no command line interface to interact with HOSTLIST, the only -way to interact with the hostlist is to use STATISTICS to obtain or modify -information about the status of HOSTLIST: -@example -$ gnunet-statistics -s hostlist -@end example - -In particular, HOSTLIST includes a @strong{persistent} value in statistics that -specifies when the hostlist server might be queried next. As this value is -exponentially increasing during runtime, developers may want to reset or -manually adjust it. Note that HOSTLIST (but not STATISTICS) needs to be -shutdown if changes to this value are to have any effect on the daemon (as -HOSTLIST does not monitor STATISTICS for changes to the download -frequency). - -@node Hostlist security address validation -@subsection Hostlist security address validation - -@c %**end of header - -Since information obtained from other parties cannot be trusted without -validation, we have to distinguish between @emph{validated} and @emph{not -validated} addresses. Before using (and so trusting) information from other -parties, this information has to be double-checked (validated). Address -validation is not done by HOSTLIST but by the TRANSPORT service. - -The HOSTLIST component is functionally located between the PEERINFO and the -TRANSPORT subsystem. When acting as a server, the daemon obtains valid -(@emph{validated}) peer information (HELLO messages) from the PEERINFO service -and provides it to other peers. When acting as a client, it contacts the -HOSTLIST servers specified in the configuration, downloads the (unvalidated) -list of HELLO messages and forwards these information to the TRANSPORT server -to validate the addresses. - -@node The HOSTLIST daemon -@subsection The HOSTLIST daemon - -@c %**end of header - -The hostlist daemon is the main component of the HOSTLIST subsystem. It is -started by the ARM service and (if configured) starts the HOSTLIST client and -server components. - -If the daemon provides a hostlist itself it can advertise it's own hostlist to -other peers. To do so it sends a GNUNET_MESSAGE_TYPE_HOSTLIST_ADVERTISEMENT -message to other peers when they connect to this peer on the CORE level. This -hostlist advertisement message contains the URL to access the HOSTLIST HTTP -server of the sender. The daemon may also subscribe to this type of message -from CORE service, and then forward these kind of message to the HOSTLIST -client. The client then uses all available URLs to download peer information -when necessary. - -When starting, the HOSTLIST daemon first connects to the CORE subsystem and if -hostlist learning is enabled, registers a CORE handler to receive this kind of -messages. Next it starts (if configured) the client and server. It passes -pointers to CORE connect and disconnect and receive handlers where the client -and server store their functions, so the daemon can notify them about CORE -events. - -To clean up on shutdown, the daemon has a cleaning task, shutting down all -subsystems and disconnecting from CORE. - -@node The HOSTLIST server -@subsection The HOSTLIST server - -@c %**end of header - -The server provides a way for other peers to obtain HELLOs. Basically it is a -small web server other peers can connect to and download a list of HELLOs using -standard HTTP; it may also advertise the URL of the hostlist to other peers -connecting on CORE level. - - -@menu -* The HTTP Server:: -* Advertising the URL:: -@end menu - -@node The HTTP Server -@subsubsection The HTTP Server - -@c %**end of header - -During startup, the server starts a web server listening on the port specified -with the HTTPPORT value (default 8080). In addition it connects to the PEERINFO -service to obtain peer information. The HOSTLIST server uses the -GNUNET_PEERINFO_iterate function to request HELLO information for all peers and -adds their information to a new hostlist if they are suitable (expired -addresses and HELLOs without addresses are both not suitable) and the maximum -size for a hostlist is not exceeded (MAX_BYTES_PER_HOSTLISTS = 500000). When -PEERINFO finishes (with a last NULL callback), the server destroys the previous -hostlist response available for download on the web server and replaces it with -the updated hostlist. The hostlist format is basically a sequence of HELLO -messages (as obtained from PEERINFO) without any special tokenization. Since -each HELLO message contains a size field, the response can easily be split into -separate HELLO messages by the client. - -A HOSTLIST client connecting to the HOSTLIST server will receive the hostlist -as a HTTP response and the the server will terminate the connection with the -result code HTTP 200 OK. The connection will be closed immediately if no -hostlist is available. - -@node Advertising the URL -@subsubsection Advertising the URL - -@c %**end of header - -The server also advertises the URL to download the hostlist to other peers if -hostlist advertisement is enabled. When a new peer connects and has hostlist -learning enabled, the server sends a GNUNET_MESSAGE_TYPE_HOSTLIST_ADVERTISEMENT -message to this peer using the CORE service. - -@node The HOSTLIST client -@subsection The HOSTLIST client - -@c %**end of header - -The client provides the functionality to download the list of HELLOs from a set -of URLs. It performs a standard HTTP request to the URLs configured and learned -from advertisement messages received from other peers. When a HELLO is -downloaded, the HOSTLIST client forwards the HELLO to the TRANSPORT service for -validation. - -The client supports two modes of operation: download of HELLOs (bootstrapping) -and learning of URLs. - - -@menu -* Bootstrapping:: -* Learning:: -@end menu - -@node Bootstrapping -@subsubsection Bootstrapping - -@c %**end of header - -For bootstrapping, it schedules a task to download the hostlist from the set of -known URLs. The downloads are only performed if the number of current -connections is smaller than a minimum number of connections (at the moment 4). -The interval between downloads increases exponentially; however, the -exponential growth is limited if it becomes longer than an hour. At that point, -the frequency growth is capped at (#number of connections * 1h). - -Once the decision has been taken to download HELLOs, the daemon chooses a -random URL from the list of known URLs. URLs can be configured in the -configuration or be learned from advertisement messages. The client uses a HTTP -client library (libcurl) to initiate the download using the libcurl multi -interface. Libcurl passes the data to the callback_download function which -stores the data in a buffer if space is available and the maximum size for a -hostlist download is not exceeded (MAX_BYTES_PER_HOSTLISTS = 500000). When a -full HELLO was downloaded, the HOSTLIST client offers this HELLO message to the -TRANSPORT service for validation. When the download is finished or failed, -statistical information about the quality of this URL is updated. - -@node Learning -@subsubsection Learning - -@c %**end of header - -The client also manages hostlist advertisements from other peers. The HOSTLIST -daemon forwards GNUNET_MESSAGE_TYPE_HOSTLIST_ADVERTISEMENT messages to the -client subsystem, which extracts the URL from the message. Next, a test of the -newly obtained URL is performed by triggering a download from the new URL. If -the URL works correctly, it is added to the list of working URLs. - -The size of the list of URLs is restricted, so if an additional server is added -and the list is full, the URL with the worst quality ranking (determined -through successful downloads and number of HELLOs e.g.) is discarded. During -shutdown the list of URLs is saved to a file for persistance and loaded on -startup. URLs from the configuration file are never discarded. - -@node Usage -@subsection Usage - -@c %**end of header - -To start HOSTLIST by default, it has to be added to the DEFAULTSERVICES section -for the ARM services. This is done in the default configuration. - -For more information on how to configure the HOSTLIST subsystem see the -installation handbook:@ Configuring the hostlist to bootstrap@ Configuring your -peer to provide a hostlist - -@node GNUnet's IDENTITY subsystem -@section GNUnet's IDENTITY subsystem - -@c %**end of header - -Identities of "users" in GNUnet are called egos. Egos can be used as pseudonyms -(fake names) or be tied to an organization (for example, GNU) or even the -actual identity of a human. GNUnet users are expected to have many egos. They -might have one tied to their real identity, some for organizations they manage, -and more for different domains where they want to operate under a pseudonym. - -The IDENTITY service allows users to manage their egos. The identity service -manages the private keys egos of the local user; it does not manage identities -of other users (public keys). Public keys for other users need names to become -manageable. GNUnet uses the GNU Name System (GNS) to give names to other users -and manage their public keys securely. This chapter is about the IDENTITY -service, which is about the management of private keys. - -On the network, an ego corresponds to an ECDSA key (over Curve25519, using RFC -6979, as required by GNS). Thus, users can perform actions under a particular -ego by using (signing with) a particular private key. Other users can then -confirm that the action was really performed by that ego by checking the -signature against the respective public key. - -The IDENTITY service allows users to associate a human-readable name with each -ego. This way, users can use names that will remind them of the purpose of a -particular ego. The IDENTITY service will store the respective private keys and -allows applications to access key information by name. Users can change the -name that is locally (!) associated with an ego. Egos can also be deleted, -which means that the private key will be removed and it thus will not be -possible to perform actions with that ego in the future. - -Additionally, the IDENTITY subsystem can associate service functions with egos. -For example, GNS requires the ego that should be used for the shorten zone. GNS -will ask IDENTITY for an ego for the "gns-short" service. The IDENTITY service -has a mapping of such service strings to the name of the ego that the user -wants to use for this service, for example "my-short-zone-ego". - -Finally, the IDENTITY API provides access to a special ego, the anonymous ego. -The anonymous ego is special in that its private key is not really private, but -fixed and known to everyone. Thus, anyone can perform actions as anonymous. -This can be useful as with this trick, code does not have to contain a special -case to distinguish between anonymous and pseudonymous egos. - -@menu -* libgnunetidentity:: -* The IDENTITY Client-Service Protocol:: -@end menu - -@node libgnunetidentity -@subsection libgnunetidentity -@c %**end of header - - -@menu -* Connecting to the service:: -* Operations on Egos:: -* The anonymous Ego:: -* Convenience API to lookup a single ego:: -* Associating egos with service functions:: -@end menu - -@node Connecting to the service -@subsubsection Connecting to the service - -@c %**end of header - -First, typical clients connect to the identity service using -@code{GNUNET_IDENTITY_connect}. This function takes a callback as a parameter. -If the given callback parameter is non-null, it will be invoked to notify the -application about the current state of the identities in the system. - -@itemize @bullet -@item First, it will be invoked on all known egos at the time of the -connection. For each ego, a handle to the ego and the user's name for the ego -will be passed to the callback. Furthermore, a @code{void **} context argument -will be provided which gives the client the opportunity to associate some state -with the ego. -@item Second, the callback will be invoked with NULL for the ego, the name and -the context. This signals that the (initial) iteration over all egos has -completed. -@item Then, the callback will be invoked whenever something changes about an -ego. If an ego is renamed, the callback is invoked with the ego handle of the -ego that was renamed, and the new name. If an ego is deleted, the callback is -invoked with the ego handle and a name of NULL. In the deletion case, the -application should also release resources stored in the context. -@item When the application destroys the connection to the identity service -using @code{GNUNET_IDENTITY_disconnect}, the callback is again invoked with the -ego and a name of NULL (equivalent to deletion of the egos). This should again -be used to clean up the per-ego context. -@end itemize - -The ego handle passed to the callback remains valid until the callback is -invoked with a name of NULL, so it is safe to store a reference to the ego's -handle. - -@node Operations on Egos -@subsubsection Operations on Egos - -@c %**end of header - -Given an ego handle, the main operations are to get its associated private key -using @code{GNUNET_IDENTITY_ego_get_private_key} or its associated public key -using @code{GNUNET_IDENTITY_ego_get_public_key}. - -The other operations on egos are pretty straightforward. Using -@code{GNUNET_IDENTITY_create}, an application can request the creation of an -ego by specifying the desired name. The operation will fail if that name is -already in use. Using @code{GNUNET_IDENTITY_rename} the name of an existing ego -can be changed. Finally, egos can be deleted using -@code{GNUNET_IDENTITY_delete}. All of these operations will trigger updates to -the callback given to the @code{GNUNET_IDENTITY_connect} function of all -applications that are connected with the identity service at the time. -@code{GNUNET_IDENTITY_cancel} can be used to cancel the operations before the -respective continuations would be called. It is not guaranteed that the -operation will not be completed anyway, only the continuation will no longer be -called. - -@node The anonymous Ego -@subsubsection The anonymous Ego - -@c %**end of header - -A special way to obtain an ego handle is to call -@code{GNUNET_IDENTITY_ego_get_anonymous}, which returns an ego for the -"anonymous" user --- anyone knows and can get the private key for this user, so -it is suitable for operations that are supposed to be anonymous but require -signatures (for example, to avoid a special path in the code). The anonymous -ego is always valid and accessing it does not require a connection to the -identity service. - -@node Convenience API to lookup a single ego -@subsubsection Convenience API to lookup a single ego - - -As applications commonly simply have to lookup a single ego, there is a -convenience API to do just that. Use @code{GNUNET_IDENTITY_ego_lookup} to -lookup a single ego by name. Note that this is the user's name for the ego, not -the service function. The resulting ego will be returned via a callback and -will only be valid during that callback. The operation can be cancelled via -@code{GNUNET_IDENTITY_ego_lookup_cancel} (cancellation is only legal before the -callback is invoked). - -@node Associating egos with service functions -@subsubsection Associating egos with service functions - - -The @code{GNUNET_IDENTITY_set} function is used to associate a particular ego -with a service function. The name used by the service and the ego are given as -arguments. Afterwards, the service can use its name to lookup the associated -ego using @code{GNUNET_IDENTITY_get}. - -@node The IDENTITY Client-Service Protocol -@subsection The IDENTITY Client-Service Protocol - -@c %**end of header - -A client connecting to the identity service first sends a message with type -@code{GNUNET_MESSAGE_TYPE_IDENTITY_START} to the service. After that, the -client will receive information about changes to the egos by receiving messages -of type @code{GNUNET_MESSAGE_TYPE_IDENTITY_UPDATE}. Those messages contain the -private key of the ego and the user's name of the ego (or zero bytes for the -name to indicate that the ego was deleted). A special bit @code{end_of_list} is -used to indicate the end of the initial iteration over the identity service's -egos. - -The client can trigger changes to the egos by sending CREATE, RENAME or DELETE -messages. The CREATE message contains the private key and the desired name. The -RENAME message contains the old name and the new name. The DELETE message only -needs to include the name of the ego to delete. The service responds to each of -these messages with a RESULT_CODE message which indicates success or error of -the operation, and possibly a human-readable error message. - -Finally, the client can bind the name of a service function to an ego by -sending a SET_DEFAULT message with the name of the service function and the -private key of the ego. Such bindings can then be resolved using a GET_DEFAULT -message, which includes the name of the service function. The identity service -will respond to a GET_DEFAULT request with a SET_DEFAULT message containing the -respective information, or with a RESULT_CODE to indicate an error. - -@node GNUnet's NAMESTORE Subsystem -@section GNUnet's NAMESTORE Subsystem - -@c %**end of header - -The NAMESTORE subsystem provides persistent storage for local GNS zone -information. All local GNS zone information are managed by NAMESTORE. It -provides both the functionality to administer local GNS information (e.g. -delete and add records) as well as to retrieve GNS information (e.g to list -name information in a client). NAMESTORE does only manage the persistent -storage of zone information belonging to the user running the service: GNS -information from other users obtained from the DHT are stored by the NAMECACHE -subsystem. - -NAMESTORE uses a plugin-based database backend to store GNS information with -good performance. Here sqlite, MySQL and PostgreSQL are supported database -backends. NAMESTORE clients interact with the IDENTITY subsystem to obtain -cryptographic information about zones based on egos as described with the -IDENTITY subsystem., but internally NAMESTORE refers to zones using the ECDSA -private key. In addition, it collaborates with the NAMECACHE subsystem and -stores zone information when local information are modified in the GNS cache to -increase look-up performance for local information. - -NAMESTORE provides functionality to look-up and store records, to iterate over -a specific or all zones and to monitor zones for changes. NAMESTORE -functionality can be accessed using the NAMESTORE api or the NAMESTORE command -line tool. - -@menu -* libgnunetnamestore:: -@end menu - -@node libgnunetnamestore -@subsection libgnunetnamestore - -@c %**end of header - -To interact with NAMESTORE clients first connect to the NAMESTORE service using -the @code{GNUNET_NAMESTORE_connect} passing a configuration handle. As a result -they obtain a NAMESTORE handle, they can use for operations, or NULL is -returned if the connection failed. - -To disconnect from NAMESTORE, clients use @code{GNUNET_NAMESTORE_disconnect} -and specify the handle to disconnect. - -NAMESTORE internally uses the ECDSA private key to refer to zones. These -private keys can be obtained from the IDENTITY subsytem. Here @emph{egos@emph{ -can be used to refer to zones or the default ego assigned to the GNS subsystem -can be used to obtained the master zone's private key.}} - - -@menu -* Editing Zone Information:: -* Iterating Zone Information:: -* Monitoring Zone Information:: -@end menu - -@node Editing Zone Information -@subsubsection Editing Zone Information - -@c %**end of header - -NAMESTORE provides functions to lookup records stored under a label in a zone -and to store records under a label in a zone. - -To store (and delete) records, the client uses the -@code{GNUNET_NAMESTORE_records_store} function and has to provide namestore -handle to use, the private key of the zone, the label to store the records -under, the records and number of records plus an callback function. After the -operation is performed NAMESTORE will call the provided callback function with -the result GNUNET_SYSERR on failure (including timeout/queue drop/failure to -validate), GNUNET_NO if content was already there or not found GNUNET_YES (or -other positive value) on success plus an additional error message. - -Records are deleted by using the store command with 0 records to store. It is -important to note, that records are not merged when records exist with the -label. So a client has first to retrieve records, merge with existing records -and then store the result. - -To perform a lookup operation, the client uses the -@code{GNUNET_NAMESTORE_records_store} function. Here he has to pass the -namestore handle, the private key of the zone and the label. He also has to -provide a callback function which will be called with the result of the lookup -operation: the zone for the records, the label, and the records including the -number of records included. - -A special operation is used to set the preferred nickname for a zone. This -nickname is stored with the zone and is automatically merged with all labels -and records stored in a zone. Here the client uses the -@code{GNUNET_NAMESTORE_set_nick} function and passes the private key of the -zone, the nickname as string plus a the callback with the result of the -operation. - -@node Iterating Zone Information -@subsubsection Iterating Zone Information - -@c %**end of header - -A client can iterate over all information in a zone or all zones managed by -NAMESTORE. Here a client uses the @code{GNUNET_NAMESTORE_zone_iteration_start} -function and passes the namestore handle, the zone to iterate over and a -callback function to call with the result. If the client wants to iterate over -all the, he passes NULL for the zone. A @code{GNUNET_NAMESTORE_ZoneIterator} -handle is returned to be used to continue iteration. - -NAMESTORE calls the callback for every result and expects the client to call@ -@code{GNUNET_NAMESTORE_zone_iterator_next} to continue to iterate or -@code{GNUNET_NAMESTORE_zone_iterator_stop} to interrupt the iteration. When -NAMESTORE reached the last item it will call the callback with a NULL value to -indicate. - -@node Monitoring Zone Information -@subsubsection Monitoring Zone Information - -@c %**end of header - -Clients can also monitor zones to be notified about changes. Here the clients -uses the @code{GNUNET_NAMESTORE_zone_monitor_start} function and passes the -private key of the zone and and a callback function to call with updates for a -zone. The client can specify to obtain zone information first by iterating over -the zone and specify a synchronization callback to be called when the client -and the namestore are synced. - -On an update, NAMESTORE will call the callback with the private key of the -zone, the label and the records and their number. - -To stop monitoring, the client call @code{GNUNET_NAMESTORE_zone_monitor_stop} -and passes the handle obtained from the function to start the monitoring. - -@node GNUnet's PEERINFO subsystem -@section GNUnet's PEERINFO subsystem - -@c %**end of header - -The PEERINFO subsystem is used to store verified (validated) information about -known peers in a persistent way. It obtains these addresses for example from -TRANSPORT service which is in charge of address validation. Validation means -that the information in the HELLO message are checked by connecting to the -addresses and performing a cryptographic handshake to authenticate the peer -instance stating to be reachable with these addresses. Peerinfo does not -validate the HELLO messages itself but only stores them and gives them to -interested clients. - -As future work, we think about moving from storing just HELLO messages to -providing a generic persistent per-peer information store. More and more -subsystems tend to need to store per-peer information in persistent way. To not -duplicate this functionality we plan to provide a PEERSTORE service providing -this functionality - -@menu -* Features2:: -* Limitations3:: -* DeveloperPeer Information:: -* Startup:: -* Managing Information:: -* Obtaining Information:: -* The PEERINFO Client-Service Protocol:: -* libgnunetpeerinfo:: -@end menu - -@node Features2 -@subsection Features2 - -@c %**end of header - -@itemize @bullet -@item Persistent storage -@item Client notification mechanism on update -@item Periodic clean up for expired information -@item Differentiation between public and friend-only HELLO -@end itemize - -@node Limitations3 -@subsection Limitations3 - - -@itemize @bullet -@item Does not perform HELLO validation -@end itemize - -@node DeveloperPeer Information -@subsection DeveloperPeer Information - -@c %**end of header - -The PEERINFO subsystem stores these information in the form of HELLO messages -you can think of as business cards. These HELLO messages contain the public key -of a peer and the addresses a peer can be reached under. The addresses include -an expiration date describing how long they are valid. This information is -updated regularly by the TRANSPORT service by revalidating the address. If an -address is expired and not renewed, it can be removed from the HELLO message. - -Some peer do not want to have their HELLO messages distributed to other peers , -especially when GNUnet's friend-to-friend modus is enabled. To prevent this -undesired distribution. PEERINFO distinguishes between @emph{public} and -@emph{friend-only} HELLO messages. Public HELLO messages can be freely -distributed to other (possibly unknown) peers (for example using the hostlist, -gossiping, broadcasting), whereas friend-only HELLO messages may not be -distributed to other peers. Friend-only HELLO messages have an additional flag -@code{friend_only} set internally. For public HELLO message this flag is not -set. PEERINFO does and cannot not check if a client is allowed to obtain a -specific HELLO type. - -The HELLO messages can be managed using the GNUnet HELLO library. Other GNUnet -systems can obtain these information from PEERINFO and use it for their -purposes. Clients are for example the HOSTLIST component providing these -information to other peers in form of a hostlist or the TRANSPORT subsystem -using these information to maintain connections to other peers. - -@node Startup -@subsection Startup - -@c %**end of header - -During startup the PEERINFO services loads persistent HELLOs from disk. First -PEERINFO parses the directory configured in the HOSTS value of the -@code{PEERINFO} configuration section to store PEERINFO information.@ For all -files found in this directory valid HELLO messages are extracted. In addition -it loads HELLO messages shipped with the GNUnet distribution. These HELLOs are -used to simplify network bootstrapping by providing valid peer information with -the distribution. The use of these HELLOs can be prevented by setting the -@code{USE_INCLUDED_HELLOS} in the @code{PEERINFO} configuration section to -@code{NO}. Files containing invalid information are removed. - -@node Managing Information -@subsection Managing Information - -@c %**end of header - -The PEERINFO services stores information about known PEERS and a single HELLO -message for every peer. A peer does not need to have a HELLO if no information -are available. HELLO information from different sources, for example a HELLO -obtained from a remote HOSTLIST and a second HELLO stored on disk, are combined -and merged into one single HELLO message per peer which will be given to -clients. During this merge process the HELLO is immediately written to disk to -ensure persistence. - -PEERINFO in addition periodically scans the directory where information are -stored for empty HELLO messages with expired TRANSPORT addresses.@ This -periodic task scans all files in the directory and recreates the HELLO messages -it finds. Expired TRANSPORT addresses are removed from the HELLO and if the -HELLO does not contain any valid addresses, it is discarded and removed from -disk. - -@node Obtaining Information -@subsection Obtaining Information - -@c %**end of header - -When a client requests information from PEERINFO, PEERINFO performs a lookup -for the respective peer or all peers if desired and transmits this information -to the client. The client can specify if friend-only HELLOs have to be included -or not and PEERINFO filters the respective HELLO messages before transmitting -information. - -To notify clients about changes to PEERINFO information, PEERINFO maintains a -list of clients interested in this notifications. Such a notification occurs if -a HELLO for a peer was updated (due to a merge for example) or a new peer was -added. - -@node The PEERINFO Client-Service Protocol -@subsection The PEERINFO Client-Service Protocol - -@c %**end of header - -To connect and disconnect to and from the PEERINFO Service PEERINFO utilizes -the util client/server infrastructure, so no special messages types are used -here. - -To add information for a peer, the plain HELLO message is transmitted to the -service without any wrapping. Alle information required are stored within the -HELLO message. The PEERINFO service provides a message handler accepting and -processing these HELLO messages. - -When obtaining PEERINFO information using the iterate functionality specific -messages are used. To obtain information for all peers, a @code{struct -ListAllPeersMessage} with message type -@code{GNUNET_MESSAGE_TYPE_PEERINFO_GET_ALL} and a flag include_friend_only to -indicate if friend-only HELLO messages should be included are transmitted. If -information for a specific peer is required a @code{struct ListAllPeersMessage} -with @code{GNUNET_MESSAGE_TYPE_PEERINFO_GET} containing the peer identity is -used. - -For both variants the PEERINFO service replies for each HELLO message he wants -to transmit with a @code{struct ListAllPeersMessage} with type -@code{GNUNET_MESSAGE_TYPE_PEERINFO_INFO} containing the plain HELLO. The final -message is @code{struct GNUNET_MessageHeader} with type -@code{GNUNET_MESSAGE_TYPE_PEERINFO_INFO}. If the client receives this message, -he can proceed with the next request if any is pending - -@node libgnunetpeerinfo -@subsection libgnunetpeerinfo - -@c %**end of header - -The PEERINFO API consists mainly of three different functionalities: -maintaining a connection to the service, adding new information and retrieving -information form the PEERINFO service. - - -@menu -* Connecting to the Service:: -* Adding Information:: -* Obtaining Information2:: -@end menu - -@node Connecting to the Service -@subsubsection Connecting to the Service - -@c %**end of header - -To connect to the PEERINFO service the function @code{GNUNET_PEERINFO_connect} -is used, taking a configuration handle as an argument, and to disconnect from -PEERINFO the function @code{GNUNET_PEERINFO_disconnect}, taking the PEERINFO -handle returned from the connect function has to be called. - -@node Adding Information -@subsubsection Adding Information - -@c %**end of header - -@code{GNUNET_PEERINFO_add_peer} adds a new peer to the PEERINFO subsystem -storage. This function takes the PEERINFO handle as an argument, the HELLO -message to store and a continuation with a closure to be called with the result -of the operation. The @code{GNUNET_PEERINFO_add_peer} returns a handle to this -operation allowing to cancel the operation with the respective cancel function -@code{GNUNET_PEERINFO_add_peer_cancel}. To retrieve information from PEERINFO -you can iterate over all information stored with PEERINFO or you can tell -PEERINFO to notify if new peer information are available. - -@node Obtaining Information2 -@subsubsection Obtaining Information2 - -@c %**end of header - -To iterate over information in PEERINFO you use @code{GNUNET_PEERINFO_iterate}. -This function expects the PEERINFO handle, a flag if HELLO messages intended -for friend only mode should be included, a timeout how long the operation -should take and a callback with a callback closure to be called for the -results. If you want to obtain information for a specific peer, you can specify -the peer identity, if this identity is NULL, information for all peers are -returned. The function returns a handle to allow to cancel the operation using -@code{GNUNET_PEERINFO_iterate_cancel}. - -To get notified when peer information changes, you can use -@code{GNUNET_PEERINFO_notify}. This function expects a configuration handle and -a flag if friend-only HELLO messages should be included. The PEERINFO service -will notify you about every change and the callback function will be called to -notify you about changes. The function returns a handle to cancel notifications -with @code{GNUNET_PEERINFO_notify_cancel}. - - -@node GNUnet's PEERSTORE subsystem -@section GNUnet's PEERSTORE subsystem - -@c %**end of header - -GNUnet's PEERSTORE subsystem offers persistent per-peer storage for other -GNUnet subsystems. GNUnet subsystems can use PEERSTORE to persistently store -and retrieve arbitrary data. Each data record stored with PEERSTORE contains -the following fields: - -@itemize @bullet -@item subsystem: Name of the subsystem responsible for the record. -@item peerid: Identity of the peer this record is related to. -@item key: a key string identifying the record. -@item value: binary record value. -@item expiry: record expiry date. -@end itemize - -@menu -* Functionality:: -* Architecture:: -* libgnunetpeerstore:: -@end menu - -@node Functionality -@subsection Functionality - -@c %**end of header - -Subsystems can store any type of value under a (subsystem, peerid, key) -combination. A "replace" flag set during store operations forces the PEERSTORE -to replace any old values stored under the same (subsystem, peerid, key) -combination with the new value. Additionally, an expiry date is set after which -the record is *possibly* deleted by PEERSTORE. - -Subsystems can iterate over all values stored under any of the following -combination of fields: - -@itemize @bullet -@item (subsystem) -@item (subsystem, peerid) -@item (subsystem, key) -@item (subsystem, peerid, key) -@end itemize - -Subsystems can also request to be notified about any new values stored under a -(subsystem, peerid, key) combination by sending a "watch" request to -PEERSTORE. - -@node Architecture -@subsection Architecture - -@c %**end of header - -PEERSTORE implements the following components: - -@itemize @bullet -@item PEERSTORE service: Handles store, iterate and watch operations. -@item PEERSTORE API: API to be used by other subsystems to communicate and -issue commands to the PEERSTORE service. -@item PEERSTORE plugins: Handles the persistent storage. At the moment, only an -"sqlite" plugin is implemented. -@end itemize - -@node libgnunetpeerstore -@subsection libgnunetpeerstore - -@c %**end of header - -libgnunetpeerstore is the library containing the PEERSTORE API. Subsystems -wishing to communicate with the PEERSTORE service use this API to open a -connection to PEERSTORE. This is done by calling -@code{GNUNET_PEERSTORE_connect} which returns a handle to the newly created -connection. This handle has to be used with any further calls to the API. - -To store a new record, the function @code{GNUNET_PEERSTORE_store} is to be used -which requires the record fields and a continuation function that will be -called by the API after the STORE request is sent to the PEERSTORE service. -Note that calling the continuation function does not mean that the record is -successfully stored, only that the STORE request has been successfully sent to -the PEERSTORE service. @code{GNUNET_PEERSTORE_store_cancel} can be called to -cancel the STORE request only before the continuation function has been called. - -To iterate over stored records, the function @code{GNUNET_PEERSTORE_iterate} is -to be used. @emph{peerid} and @emph{key} can be set to NULL. An iterator -callback function will be called with each matching record found and a NULL -record at the end to signal the end of result set. -@code{GNUNET_PEERSTORE_iterate_cancel} can be used to cancel the ITERATE -request before the iterator callback is called with a NULL record. - -To be notified with new values stored under a (subsystem, peerid, key) -combination, the function @code{GNUNET_PEERSTORE_watch} is to be used. This -will register the watcher with the PEERSTORE service, any new records matching -the given combination will trigger the callback function passed to -@code{GNUNET_PEERSTORE_watch}. This continues until -@code{GNUNET_PEERSTORE_watch_cancel} is called or the connection to the service -is destroyed. - -After the connection is no longer needed, the function -@code{GNUNET_PEERSTORE_disconnect} can be called to disconnect from the -PEERSTORE service. Any pending ITERATE or WATCH requests will be destroyed. If -the @code{sync_first} flag is set to @code{GNUNET_YES}, the API will delay the -disconnection until all pending STORE requests are sent to the PEERSTORE -service, otherwise, the pending STORE requests will be destroyed as well. - -@node GNUnet's SET Subsystem -@section GNUnet's SET Subsystem - -@c %**end of header - -The SET service implements efficient set operations between two peers over a -mesh tunnel. Currently, set union and set intersection are the only supported -operations. Elements of a set consist of an @emph{element type} and arbitrary -binary @emph{data}. The size of an element's data is limited to around 62 -KB. - -@menu -* Local Sets:: -* Set Modifications:: -* Set Operations:: -* Result Elements:: -* libgnunetset:: -* The SET Client-Service Protocol:: -* The SET Intersection Peer-to-Peer Protocol:: -* The SET Union Peer-to-Peer Protocol:: -@end menu - -@node Local Sets -@subsection Local Sets - -@c %**end of header - -Sets created by a local client can be modified and reused for multiple -operations. As each set operation requires potentially expensive special -auxilliary data to be computed for each element of a set, a set can only -participate in one type of set operation (i.e. union or intersection). The type -of a set is determined upon its creation. If a the elements of a set are needed -for an operation of a different type, all of the set's element must be copied -to a new set of appropriate type. - -@node Set Modifications -@subsection Set Modifications - -@c %**end of header - -Even when set operations are active, one can add to and remove elements from a -set. However, these changes will only be visible to operations that have been -created after the changes have taken place. That is, every set operation only -sees a snapshot of the set from the time the operation was started. This -mechanism is @emph{not} implemented by copying the whole set, but by attaching -@emph{generation information} to each element and operation. - -@node Set Operations -@subsection Set Operations - -@c %**end of header - -Set operations can be started in two ways: Either by accepting an operation -request from a remote peer, or by requesting a set operation from a remote -peer. Set operations are uniquely identified by the involved @emph{peers}, an -@emph{application id} and the @emph{operation type}. - -The client is notified of incoming set operations by @emph{set listeners}. A -set listener listens for incoming operations of a specific operation type and -application id. Once notified of an incoming set request, the client can -accept the set request (providing a local set for the operation) or reject -it. - -@node Result Elements -@subsection Result Elements - -@c %**end of header - -The SET service has three @emph{result modes} that determine how an operation's -result set is delivered to the client: - -@itemize @bullet -@item @strong{Full Result Set.} All elements of set resulting from the set -operation are returned to the client. -@item @strong{Added Elements.} Only elements that result from the operation and -are not already in the local peer's set are returned. Note that for some -operations (like set intersection) this result mode will never return any -elements. This can be useful if only the remove peer is actually interested in -the result of the set operation. -@item @strong{Removed Elements.} Only elements that are in the local peer's -initial set but not in the operation's result set are returned. Note that for -some operations (like set union) this result mode will never return any -elements. This can be useful if only the remove peer is actually interested in -the result of the set operation. -@end itemize - -@node libgnunetset -@subsection libgnunetset - -@c %**end of header - -@menu -* Sets:: -* Listeners:: -* Operations:: -* Supplying a Set:: -* The Result Callback:: -@end menu - -@node Sets -@subsubsection Sets - -@c %**end of header - -New sets are created with @code{GNUNET_SET_create}. Both the local peer's -configuration (as each set has its own client connection) and the operation -type must be specified. The set exists until either the client calls -@code{GNUNET_SET_destroy} or the client's connection to the service is -disrupted. In the latter case, the client is notified by the return value of -functions dealing with sets. This return value must always be checked. - -Elements are added and removed with @code{GNUNET_SET_add_element} and -@code{GNUNET_SET_remove_element}. - -@node Listeners -@subsubsection Listeners - -@c %**end of header - -Listeners are created with @code{GNUNET_SET_listen}. Each time time a remote -peer suggests a set operation with an application id and operation type -matching a listener, the listener's callack is invoked. The client then must -synchronously call either @code{GNUNET_SET_accept} or @code{GNUNET_SET_reject}. -Note that the operation will not be started until the client calls -@code{GNUNET_SET_commit} (see Section "Supplying a Set"). - -@node Operations -@subsubsection Operations - -@c %**end of header - -Operations to be initiated by the local peer are created with -@code{GNUNET_SET_prepare}. Note that the operation will not be started until -the client calls @code{GNUNET_SET_commit} (see Section "Supplying a -Set"). - -@node Supplying a Set -@subsubsection Supplying a Set - -@c %**end of header - -To create symmetry between the two ways of starting a set operation (accepting -and nitiating it), the operation handles returned by @code{GNUNET_SET_accept} -and @code{GNUNET_SET_prepare} do not yet have a set to operate on, thus they -can not do any work yet. - -The client must call @code{GNUNET_SET_commit} to specify a set to use for an -operation. @code{GNUNET_SET_commit} may only be called once per set -operation. - -@node The Result Callback -@subsubsection The Result Callback - -@c %**end of header - -Clients must specify both a result mode and a result callback with -@code{GNUNET_SET_accept} and @code{GNUNET_SET_prepare}. The result callback -with a status indicating either that an element was received, or the operation -failed or succeeded. The interpretation of the received element depends on the -result mode. The callback needs to know which result mode it is used in, as the -arguments do not indicate if an element is part of the full result set, or if -it is in the difference between the original set and the final set. - -@node The SET Client-Service Protocol -@subsection The SET Client-Service Protocol - -@c %**end of header - -@menu -* Creating Sets:: -* Listeners2:: -* Initiating Operations:: -* Modifying Sets:: -* Results and Operation Status:: -* Iterating Sets:: -@end menu - -@node Creating Sets -@subsubsection Creating Sets - -@c %**end of header - -For each set of a client, there exists a client connection to the service. Sets -are created by sending the @code{GNUNET_SERVICE_SET_CREATE} message over a new -client connection. Multiple operations for one set are multiplexed over one -client connection, using a request id supplied by the client. - -@node Listeners2 -@subsubsection Listeners2 - -@c %**end of header - -Each listener also requires a seperate client connection. By sending the -@code{GNUNET_SERVICE_SET_LISTEN} message, the client notifies the service of -the application id and operation type it is interested in. A client rejects an -incoming request by sending @code{GNUNET_SERVICE_SET_REJECT} on the listener's -client connection. In contrast, when accepting an incoming request, a a -@code{GNUNET_SERVICE_SET_ACCEPT} message must be sent over the@ set that is -supplied for the set operation. - -@node Initiating Operations -@subsubsection Initiating Operations - -@c %**end of header - -Operations with remote peers are initiated by sending a -@code{GNUNET_SERVICE_SET_EVALUATE} message to the service. The@ client -connection that this message is sent by determines the set to use. - -@node Modifying Sets -@subsubsection Modifying Sets - -@c %**end of header - -Sets are modified with the @code{GNUNET_SERVICE_SET_ADD} and -@code{GNUNET_SERVICE_SET_REMOVE} messages. - - -@c %@menu -@c %* Results and Operation Status:: -@c %* Iterating Sets:: -@c %@end menu - -@node Results and Operation Status -@subsubsection Results and Operation Status -@c %**end of header - -The service notifies the client of result elements and success/failure of a set -operation with the @code{GNUNET_SERVICE_SET_RESULT} message. - -@node Iterating Sets -@subsubsection Iterating Sets - -@c %**end of header - -All elements of a set can be requested by sending -@code{GNUNET_SERVICE_SET_ITER_REQUEST}. The server responds with -@code{GNUNET_SERVICE_SET_ITER_ELEMENT} and eventually terminates the iteration -with @code{GNUNET_SERVICE_SET_ITER_DONE}. After each received element, the -client@ must send @code{GNUNET_SERVICE_SET_ITER_ACK}. Note that only one set -iteration may be active for a set at any given time. - -@node The SET Intersection Peer-to-Peer Protocol -@subsection The SET Intersection Peer-to-Peer Protocol - -@c %**end of header - -The intersection protocol operates over CADET and starts with a -GNUNET_MESSAGE_TYPE_SET_P2P_OPERATION_REQUEST being sent by the peer initiating -the operation to the peer listening for inbound requests. It includes the -number of elements of the initiating peer, which is used to decide which side -will send a Bloom filter first. - -The listening peer checks if the operation type and application identifier are -acceptable for its current state. If not, it responds with a -GNUNET_MESSAGE_TYPE_SET_RESULT and a status of GNUNET_SET_STATUS_FAILURE (and -terminates the CADET channel). - -If the application accepts the request, the listener sends back a@ -GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_ELEMENT_INFO if it has more elements -in the set than the client. Otherwise, it immediately starts with the Bloom -filter exchange. If the initiator receives a -GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_ELEMENT_INFO response, it beings the -Bloom filter exchange, unless the set size is indicated to be zero, in which -case the intersection is considered finished after just the initial -handshake. - - -@menu -* The Bloom filter exchange:: -* Salt:: -@end menu - -@node The Bloom filter exchange -@subsubsection The Bloom filter exchange - -@c %**end of header - -In this phase, each peer transmits a Bloom filter over the remaining keys of -the local set to the other peer using a -GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_BF message. This message additionally -includes the number of elements left in the sender's set, as well as the XOR -over all of the keys in that set. - -The number of bits 'k' set per element in the Bloom filter is calculated based -on the relative size of the two sets. Furthermore, the size of the Bloom filter -is calculated based on 'k' and the number of elements in the set to maximize -the amount of data filtered per byte transmitted on the wire (while avoiding an -excessively high number of iterations). - -The receiver of the message removes all elements from its local set that do not -pass the Bloom filter test. It then checks if the set size of the sender and -the XOR over the keys match what is left of his own set. If they do, he sends -a@ GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_DONE back to indicate that the -latest set is the final result. Otherwise, the receiver starts another Bloom -fitler exchange, except this time as the sender. - -@node Salt -@subsubsection Salt - -@c %**end of header - -Bloomfilter operations are probablistic: With some non-zero probability the -test may incorrectly say an element is in the set, even though it is not. - -To mitigate this problem, the intersection protocol iterates exchanging Bloom -filters using a different random 32-bit salt in each iteration (the salt is -also included in the message). With different salts, set operations may fail -for different elements. Merging the results from the executions, the -probability of failure drops to zero. - -The iterations terminate once both peers have established that they have sets -of the same size, and where the XOR over all keys computes the same 512-bit -value (leaving a failure probability of 2-511). - -@node The SET Union Peer-to-Peer Protocol -@subsection The SET Union Peer-to-Peer Protocol - -@c %**end of header - -The SET union protocol is based on Eppstein's efficient set reconciliation -without prior context. You should read this paper first if you want to -understand the protocol. - -The union protocol operates over CADET and starts with a -GNUNET_MESSAGE_TYPE_SET_P2P_OPERATION_REQUEST being sent by the peer initiating -the operation to the peer listening for inbound requests. It includes the -number of elements of the initiating peer, which is currently not used. - -The listening peer checks if the operation type and application identifier are -acceptable for its current state. If not, it responds with a -GNUNET_MESSAGE_TYPE_SET_RESULT and a status of GNUNET_SET_STATUS_FAILURE (and -terminates the CADET channel). - -If the application accepts the request, it sends back a strata estimator using -a message of type GNUNET_MESSAGE_TYPE_SET_UNION_P2P_SE. The initiator evaluates -the strata estimator and initiates the exchange of invertible Bloom filters, -sending a GNUNET_MESSAGE_TYPE_SET_UNION_P2P_IBF. - -During the IBF exchange, if the receiver cannot invert the Bloom filter or -detects a cycle, it sends a larger IBF in response (up to a defined maximum -limit; if that limit is reached, the operation fails). Elements decoded while -processing the IBF are transmitted to the other peer using -GNUNET_MESSAGE_TYPE_SET_P2P_ELEMENTS, or requested from the other peer using -GNUNET_MESSAGE_TYPE_SET_P2P_ELEMENT_REQUESTS messages, depending on the sign -observed during decoding of the IBF. Peers respond to a -GNUNET_MESSAGE_TYPE_SET_P2P_ELEMENT_REQUESTS message with the respective -element in a GNUNET_MESSAGE_TYPE_SET_P2P_ELEMENTS message. If the IBF fully -decodes, the peer responds with a GNUNET_MESSAGE_TYPE_SET_UNION_P2P_DONE -message instead of another GNUNET_MESSAGE_TYPE_SET_UNION_P2P_IBF. - -All Bloom filter operations use a salt to mingle keys before hasing them into -buckets, such that future iterations have a fresh chance of succeeding if they -failed due to collisions before. - -@node GNUnet's STATISTICS subsystem -@section GNUnet's STATISTICS subsystem - -@c %**end of header - -In GNUnet, the STATISTICS subsystem offers a central place for all subsystems -to publish unsigned 64-bit integer run-time statistics. Keeping this -information centrally means that there is a unified way for the user to obtain -data on all subsystems, and individual subsystems do not have to always include -a custom data export method for performance metrics and other statistics. For -example, the TRANSPORT system uses STATISTICS to update information about the -number of directly connected peers and the bandwidth that has been consumed by -the various plugins. This information is valuable for diagnosing connectivity -and performance issues. - -Following the GNUnet service architecture, the STATISTICS subsystem is divided -into an API which is exposed through the header -@strong{gnunet_statistics_service.h} and the STATISTICS service -@strong{gnunet-service-statistics}. The @strong{gnunet-statistics} command-line -tool can be used to obtain (and change) information about the values stored by -the STATISTICS service. The STATISTICS service does not communicate with other -peers. - -Data is stored in the STATISTICS service in the form of tuples -@strong{(subsystem, name, value, persistence)}. The subsystem determines to -which other GNUnet's subsystem the data belongs. name is the name through which -value is associated. It uniquely identifies the record from among other records -belonging to the same subsystem. In some parts of the code, the pair -@strong{(subsystem, name)} is called a @strong{statistic} as it identifies the -values stored in the STATISTCS service.The persistence flag determines if the -record has to be preserved across service restarts. A record is said to be -persistent if this flag is set for it; if not, the record is treated as a -non-persistent record and it is lost after service restart. Persistent records -are written to and read from the file @strong{statistics.data} before shutdown -and upon startup. The file is located in the HOME directory of the peer. - -An anomaly of the STATISTICS service is that it does not terminate immediately -upon receiving a shutdown signal if it has any clients connected to it. It -waits for all the clients that are not monitors to close their connections -before terminating itself. This is to prevent the loss of data during peer -shutdown --- delaying the STATISTICS service shutdown helps other services to -store important data to STATISTICS during shutdown. - -@menu -* libgnunetstatistics:: -* The STATISTICS Client-Service Protocol:: -@end menu - -@node libgnunetstatistics -@subsection libgnunetstatistics - -@c %**end of header - -@strong{libgnunetstatistics} is the library containing the API for the -STATISTICS subsystem. Any process requiring to use STATISTICS should use this -API by to open a connection to the STATISTICS service. This is done by calling -the function @code{GNUNET_STATISTICS_create()}. This function takes the -subsystem's name which is trying to use STATISTICS and a configuration. All -values written to STATISTICS with this connection will be placed in the section -corresponding to the given subsystem's name. The connection to STATISTICS can -be destroyed with the function GNUNET_STATISTICS_destroy(). This function -allows for the connection to be destroyed immediately or upon transferring all -pending write requests to the service. - -Note: STATISTICS subsystem can be disabled by setting @code{DISABLE = YES} -under the @code{[STATISTICS]} section in the configuration. With such a -configuration all calls to @code{GNUNET_STATISTICS_create()} return @code{NULL} -as the STATISTICS subsystem is unavailable and no other functions from the API -can be used. - - -@menu -* Statistics retrieval:: -* Setting statistics and updating them:: -* Watches:: -@end menu - -@node Statistics retrieval -@subsubsection Statistics retrieval - -@c %**end of header - -Once a connection to the statistics service is obtained, information about any -other system which uses statistics can be retrieved with the function -GNUNET_STATISTICS_get(). This function takes the connection handle, the name of -the subsystem whose information we are interested in (a @code{NULL} value will -retrieve information of all available subsystems using STATISTICS), the name of -the statistic we are interested in (a @code{NULL} value will retrieve all -available statistics), a continuation callback which is called when all of -requested information is retrieved, an iterator callback which is called for -each parameter in the retrieved information and a closure for the -aforementioned callbacks. The library then invokes the iterator callback for -each value matching the request. - -Call to @code{GNUNET_STATISTICS_get()} is asynchronous and can be canceled with -the function @code{GNUNET_STATISTICS_get_cancel()}. This is helpful when -retrieving statistics takes too long and especially when we want to shutdown -and cleanup everything. - -@node Setting statistics and updating them -@subsubsection Setting statistics and updating them - -@c %**end of header - -So far we have seen how to retrieve statistics, here we will learn how we can -set statistics and update them so that other subsystems can retrieve them. - -A new statistic can be set using the function @code{GNUNET_STATISTICS_set()}. -This function takes the name of the statistic and its value and a flag to make -the statistic persistent. The value of the statistic should be of the type -@code{uint64_t}. The function does not take the name of the subsystem; it is -determined from the previous @code{GNUNET_STATISTICS_create()} invocation. If -the given statistic is already present, its value is overwritten. - -An existing statistics can be updated, i.e its value can be increased or -decreased by an amount with the function @code{GNUNET_STATISTICS_update()}. The -parameters to this function are similar to @code{GNUNET_STATISTICS_set()}, -except that it takes the amount to be changed as a type @code{int64_t} instead -of the value. - -The library will combine multiple set or update operations into one message if -the client performs requests at a rate that is faster than the available IPC -with the STATISTICS service. Thus, the client does not have to worry about -sending requests too quickly. - -@node Watches -@subsubsection Watches - -@c %**end of header - -As interesting feature of STATISTICS lies in serving notifications whenever a -statistic of our interest is modified. This is achieved by registering a watch -through the function @code{GNUNET_STATISTICS_watch()}. The parameters of this -function are similar to those of @code{GNUNET_STATISTICS_get()}. Changes to the -respective statistic's value will then cause the given iterator callback to be -called. Note: A watch can only be registered for a specific statistic. Hence -the subsystem name and the parameter name cannot be @code{NULL} in a call to -@code{GNUNET_STATISTICS_watch()}. - -A registered watch will keep notifying any value changes until -@code{GNUNET_STATISTICS_watch_cancel()} is called with the same parameters that -are used for registering the watch. - -@node The STATISTICS Client-Service Protocol -@subsection The STATISTICS Client-Service Protocol -@c %**end of header - - -@menu -* Statistics retrieval2:: -* Setting and updating statistics:: -* Watching for updates:: -@end menu - -@node Statistics retrieval2 -@subsubsection Statistics retrieval2 - -@c %**end of header - -To retrieve statistics, the client transmits a message of type -@code{GNUNET_MESSAGE_TYPE_STATISTICS_GET} containing the given subsystem name -and statistic parameter to the STATISTICS service. The service responds with a -message of type @code{GNUNET_MESSAGE_TYPE_STATISTICS_VALUE} for each of the -statistics parameters that match the client request for the client. The end of -information retrieved is signaled by the service by sending a message of type -@code{GNUNET_MESSAGE_TYPE_STATISTICS_END}. - -@node Setting and updating statistics -@subsubsection Setting and updating statistics - -@c %**end of header - -The subsystem name, parameter name, its value and the persistence flag are -communicated to the service through the message -@code{GNUNET_MESSAGE_TYPE_STATISTICS_SET}. - -When the service receives a message of type -@code{GNUNET_MESSAGE_TYPE_STATISTICS_SET}, it retrieves the subsystem name and -checks for a statistic parameter with matching the name given in the message. -If a statistic parameter is found, the value is overwritten by the new value -from the message; if not found then a new statistic parameter is created with -the given name and value. - -In addition to just setting an absolute value, it is possible to perform a -relative update by sending a message of type -@code{GNUNET_MESSAGE_TYPE_STATISTICS_SET} with an update flag -(@code{GNUNET_STATISTICS_SETFLAG_RELATIVE}) signifying that the value in the -message should be treated as an update value. - -@node Watching for updates -@subsubsection Watching for updates - -@c %**end of header - -The function registers the watch at the service by sending a message of type -@code{GNUNET_MESSAGE_TYPE_STATISTICS_WATCH}. The service then sends -notifications through messages of type -@code{GNUNET_MESSAGE_TYPE_STATISTICS_WATCH_VALUE} whenever the statistic -parameter's value is changed. - -@node GNUnet's Distributed Hash Table (DHT) -@section GNUnet's Distributed Hash Table (DHT) - -@c %**end of header - -GNUnet includes a generic distributed hash table that can be used by developers -building P2P applications in the framework. This section documents high-level -features and how developers are expected to use the DHT. We have a research -paper detailing how the DHT works. Also, Nate's thesis includes a detailed -description and performance analysis (in chapter 6). - -Key features of GNUnet's DHT include: - -@itemize @bullet -@item stores key-value pairs with values up to (approximately) 63k in size -@item works with many underlay network topologies (small-world, random graph), -underlay does not need to be a full mesh / clique -@item support for extended queries (more than just a simple 'key'), filtering -duplicate replies within the network (bloomfilter) and content validation (for -details, please read the subsection on the block library) -@item can (optionally) return paths taken by the PUT and GET operations to the -application -@item provides content replication to handle churn -@end itemize - -GNUnet's DHT is randomized and unreliable. Unreliable means that there is no -strict guarantee that a value stored in the DHT is always found --- values are -only found with high probability. While this is somewhat true in all P2P DHTs, -GNUnet developers should be particularly wary of this fact (this will help you -write secure, fault-tolerant code). Thus, when writing any application using -the DHT, you should always consider the possibility that a value stored in the -DHT by you or some other peer might simply not be returned, or returned with a -significant delay. Your application logic must be written to tolerate this -(naturally, some loss of performance or quality of service is expected in this -case). - -@menu -* Block library and plugins:: -* libgnunetdht:: -* The DHT Client-Service Protocol:: -* The DHT Peer-to-Peer Protocol:: -@end menu - -@node Block library and plugins -@subsection Block library and plugins - -@c %**end of header - -@menu -* What is a Block?:: -* The API of libgnunetblock:: -* Queries:: -* Sample Code:: -* Conclusion2:: -@end menu - -@node What is a Block? -@subsubsection What is a Block? - -@c %**end of header - -Blocks are small (< 63k) pieces of data stored under a key (struct -GNUNET_HashCode). Blocks have a type (enum GNUNET_BlockType) which defines -their data format. Blocks are used in GNUnet as units of static data exchanged -between peers and stored (or cached) locally. Uses of blocks include -file-sharing (the files are broken up into blocks), the VPN (DNS information is -stored in blocks) and the DHT (all information in the DHT and meta-information -for the maintenance of the DHT are both stored using blocks). The block -subsystem provides a few common functions that must be available for any type -of block. - -@node The API of libgnunetblock -@subsubsection The API of libgnunetblock - -@c %**end of header - -The block library requires for each (family of) block type(s) a block plugin -(implementing gnunet_block_plugin.h) that provides basic functions that are -needed by the DHT (and possibly other subsystems) to manage the block. These -block plugins are typically implemented within their respective subsystems.@ -The main block library is then used to locate, load and query the appropriate -block plugin. Which plugin is appropriate is determined by the block type -(which is just a 32-bit integer). Block plugins contain code that specifies -which block types are supported by a given plugin. The block library loads all -block plugins that are installed at the local peer and forwards the application -request to the respective plugin. - -The central functions of the block APIs (plugin and main library) are to allow -the mapping of blocks to their respective key (if possible) and the ability to -check that a block is well-formed and matches a given request (again, if -possible). This way, GNUnet can avoid storing invalid blocks, storing blocks -under the wrong key and forwarding blocks in response to a query that they do -not answer. - -One key function of block plugins is that it allows GNUnet to detect duplicate -replies (via the Bloom filter). All plugins MUST support detecting duplicate -replies (by adding the current response to the Bloom filter and rejecting it if -it is encountered again). If a plugin fails to do this, responses may loop in -the network. - -@node Queries -@subsubsection Queries -@c %**end of header - -The query format for any block in GNUnet consists of four main components. -First, the type of the desired block must be specified. Second, the query must -contain a hash code. The hash code is used for lookups in hash tables and -databases and must not be unique for the block (however, if possible a unique -hash should be used as this would be best for performance). Third, an optional -Bloom filter can be specified to exclude known results; replies that hash to -the bits set in the Bloom filter are considered invalid. False-positives can be -eliminated by sending the same query again with a different Bloom filter -mutator value, which parameterizes the hash function that is used. Finally, an -optional application-specific "eXtended query" (xquery) can be specified to -further constrain the results. It is entirely up to the type-specific plugin to -determine whether or not a given block matches a query (type, hash, Bloom -filter, and xquery). Naturally, not all xquery's are valid and some types of -blocks may not support Bloom filters either, so the plugin also needs to check -if the query is valid in the first place. - -Depending on the results from the plugin, the DHT will then discard the -(invalid) query, forward the query, discard the (invalid) reply, cache the -(valid) reply, and/or forward the (valid and non-duplicate) reply. - -@node Sample Code -@subsubsection Sample Code - -@c %**end of header - -The source code in @strong{plugin_block_test.c} is a good starting point for -new block plugins --- it does the minimal work by implementing a plugin that -performs no validation at all. The respective @strong{Makefile.am} shows how to -build and install a block plugin. - -@node Conclusion2 -@subsubsection Conclusion2 - -@c %**end of header - -In conclusion, GNUnet subsystems that want to use the DHT need to define a -block format and write a plugin to match queries and replies. For testing, the -"GNUNET_BLOCK_TYPE_TEST" block type can be used; it accepts any query as valid -and any reply as matching any query. This type is also used for the DHT command -line tools. However, it should NOT be used for normal applications due to the -lack of error checking that results from this primitive implementation. - -@node libgnunetdht -@subsection libgnunetdht - -@c %**end of header - -The DHT API itself is pretty simple and offers the usual GET and PUT functions -that work as expected. The specified block type refers to the block library -which allows the DHT to run application-specific logic for data stored in the -network. - - -@menu -* GET:: -* PUT:: -* MONITOR:: -* DHT Routing Options:: -@end menu - -@node GET -@subsubsection GET - -@c %**end of header - -When using GET, the main consideration for developers (other than the block -library) should be that after issuing a GET, the DHT will continuously cause -(small amounts of) network traffic until the operation is explicitly canceled. -So GET does not simply send out a single network request once; instead, the -DHT will continue to search for data. This is needed to achieve good success -rates and also handles the case where the respective PUT operation happens -after the GET operation was started. Developers should not cancel an existing -GET operation and then explicitly re-start it to trigger a new round of -network requests; this is simply inefficient, especially as the internal -automated version can be more efficient, for example by filtering results in -the network that have already been returned. - -If an application that performs a GET request has a set of replies that it -already knows and would like to filter, it can call@ -@code{GNUNET_DHT_get_filter_known_results} with an array of hashes over the -respective blocks to tell the DHT that these results are not desired (any -more). This way, the DHT will filter the respective blocks using the block -library in the network, which may result in a significant reduction in -bandwidth consumption. - -@node PUT -@subsubsection PUT - -@c %**end of header - -In contrast to GET operations, developers @strong{must} manually re-run PUT -operations periodically (if they intend the content to continue to be -available). Content stored in the DHT expires or might be lost due to churn. -Furthermore, GNUnet's DHT typically requires multiple rounds of PUT operations -before a key-value pair is consistently available to all peers (the DHT -randomizes paths and thus storage locations, and only after multiple rounds of -PUTs there will be a sufficient number of replicas in large DHTs). An explicit -PUT operation using the DHT API will only cause network traffic once, so in -order to ensure basic availability and resistance to churn (and adversaries), -PUTs must be repeated. While the exact frequency depends on the application, a -rule of thumb is that there should be at least a dozen PUT operations within -the content lifetime. Content in the DHT typically expires after one day, so -DHT PUT operations should be repeated at least every 1-2 hours. - -@node MONITOR -@subsubsection MONITOR - -@c %**end of header - -The DHT API also allows applications to monitor messages crossing the local -DHT service. The types of messages used by the DHT are GET, PUT and RESULT -messages. Using the monitoring API, applications can choose to monitor these -requests, possibly limiting themselves to requests for a particular block -type. - -The monitoring API is not only usefu only for diagnostics, it can also be used -to trigger application operations based on PUT operations. For example, an -application may use PUTs to distribute work requests to other peers. The -workers would then monitor for PUTs that give them work, instead of looking -for work using GET operations. This can be beneficial, especially if the -workers have no good way to guess the keys under which work would be stored. -Naturally, additional protocols might be needed to ensure that the desired -number of workers will process the distributed workload. - -@node DHT Routing Options -@subsubsection DHT Routing Options - -@c %**end of header - -There are two important options for GET and PUT requests: - -@table @asis -@item GNUNET_DHT_RO_DEMULITPLEX_EVERYWHERE This option means that all peers -should process the request, even if their peer ID is not closest to the key. -For a PUT request, this means that all peers that a request traverses may make -a copy of the data. Similarly for a GET request, all peers will check their -local database for a result. Setting this option can thus significantly improve -caching and reduce bandwidth consumption --- at the expense of a larger DHT -database. If in doubt, we recommend that this option should be used. -@item GNUNET_DHT_RO_RECORD_ROUTE This option instructs the DHT to record the path -that a GET or a PUT request is taking through the overlay network. The -resulting paths are then returned to the application with the respective -result. This allows the receiver of a result to construct a path to the -originator of the data, which might then be used for routing. Naturally, -setting this option requires additional bandwidth and disk space, so -applications should only set this if the paths are needed by the application -logic. -@item GNUNET_DHT_RO_FIND_PEER This option is an internal option used by -the DHT's peer discovery mechanism and should not be used by applications. -@item GNUNET_DHT_RO_BART This option is currently not implemented. It may in -the future offer performance improvements for clique topologies. -@end table - -@node The DHT Client-Service Protocol -@subsection The DHT Client-Service Protocol - -@c %**end of header - -@menu -* PUTting data into the DHT:: -* GETting data from the DHT:: -* Monitoring the DHT:: -@end menu - -@node PUTting data into the DHT -@subsubsection PUTting data into the DHT - -@c %**end of header - -To store (PUT) data into the DHT, the client sends a@ @code{struct -GNUNET_DHT_ClientPutMessage} to the service. This message specifies the block -type, routing options, the desired replication level, the expiration time, key, -value and a 64-bit unique ID for the operation. The service responds with a@ -@code{struct GNUNET_DHT_ClientPutConfirmationMessage} with the same 64-bit -unique ID. Note that the service sends the confirmation as soon as it has -locally processed the PUT request. The PUT may still be propagating through the -network at this time. - -In the future, we may want to change this to provide (limited) feedback to the -client, for example if we detect that the PUT operation had no effect because -the same key-value pair was already stored in the DHT. However, changing this -would also require additional state and messages in the P2P -interaction. - -@node GETting data from the DHT -@subsubsection GETting data from the DHT - -@c %**end of header - -To retrieve (GET) data from the DHT, the client sends a@ @code{struct -GNUNET_DHT_ClientGetMessage} to the service. The message specifies routing -options, a replication level (for replicating the GET, not the content), the -desired block type, the key, the (optional) extended query and unique 64-bit -request ID. - -Additionally, the client may send any number of@ @code{struct -GNUNET_DHT_ClientGetResultSeenMessage}s to notify the service about results -that the client is already aware of. These messages consist of the key, the -unique 64-bit ID of the request, and an arbitrary number of hash codes over the -blocks that the client is already aware of. As messages are restricted to 64k, -a client that already knows more than about a thousand blocks may need to send -several of these messages. Naturally, the client should transmit these messages -as quickly as possible after the original GET request such that the DHT can -filter those results in the network early on. Naturally, as these messages are -send after the original request, it is conceivalbe that the DHT service may -return blocks that match those already known to the client anyway. - -In response to a GET request, the service will send @code{struct -GNUNET_DHT_ClientResultMessage}s to the client. These messages contain the -block type, expiration, key, unique ID of the request and of course the value -(a block). Depending on the options set for the respective operations, the -replies may also contain the path the GET and/or the PUT took through the -network. - -A client can stop receiving replies either by disconnecting or by sending a -@code{struct GNUNET_DHT_ClientGetStopMessage} which must contain the key and -the 64-bit unique ID of the original request. Using an explicit "stop" message -is more common as this allows a client to run many concurrent GET operations -over the same connection with the DHT service --- and to stop them -individually. - -@node Monitoring the DHT -@subsubsection Monitoring the DHT - -@c %**end of header - -To begin monitoring, the client sends a @code{struct -GNUNET_DHT_MonitorStartStop} message to the DHT service. In this message, flags -can be set to enable (or disable) monitoring of GET, PUT and RESULT messages -that pass through a peer. The message can also restrict monitoring to a -particular block type or a particular key. Once monitoring is enabled, the DHT -service will notify the client about any matching event using @code{struct -GNUNET_DHT_MonitorGetMessage}s for GET events, @code{struct -GNUNET_DHT_MonitorPutMessage} for PUT events and@ @code{struct -GNUNET_DHT_MonitorGetRespMessage} for RESULTs. Each of these messages contains -all of the information about the event. - -@node The DHT Peer-to-Peer Protocol -@subsection The DHT Peer-to-Peer Protocol -@c %**end of header - - -@menu -* Routing GETs or PUTs:: -* PUTting data into the DHT2:: -* GETting data from the DHT2:: -@end menu - -@node Routing GETs or PUTs -@subsubsection Routing GETs or PUTs - -@c %**end of header - -When routing GETs or PUTs, the DHT service selects a suitable subset of -neighbours for forwarding. The exact number of neighbours can be zero or more -and depends on the hop counter of the query (initially zero) in relation to the -(log of) the network size estimate, the desired replication level and the -peer's connectivity. Depending on the hop counter and our network size -estimate, the selection of the peers maybe randomized or by proximity to the -key. Furthermore, requests include a set of peers that a request has already -traversed; those peers are also excluded from the selection. - -@node PUTting data into the DHT2 -@subsubsection PUTting data into the DHT2 - -@c %**end of header - -To PUT data into the DHT, the service sends a @code{struct PeerPutMessage} of -type @code{GNUNET_MESSAGE_TYPE_DHT_P2P_PUT} to the respective neighbour. In -addition to the usual information about the content (type, routing options, -desired replication level for the content, expiration time, key and value), the -message contains a fixed-size Bloom filter with information about which peers -(may) have already seen this request. This Bloom filter is used to ensure that -DHT messages never loop back to a peer that has already processed the request. -Additionally, the message includes the current hop counter and, depending on -the routing options, the message may include the full path that the message has -taken so far. The Bloom filter should already contain the identity of the -previous hop; however, the path should not include the identity of the previous -hop and the receiver should append the identity of the sender to the path, not -its own identity (this is done to reduce bandwidth). - -@node GETting data from the DHT2 -@subsubsection GETting data from the DHT2 - -@c %**end of header - -A peer can search the DHT by sending @code{struct PeerGetMessage}s of type -@code{GNUNET_MESSAGE_TYPE_DHT_P2P_GET} to other peers. In addition to the usual -information about the request (type, routing options, desired replication level -for the request, the key and the extended query), a GET request also again -contains a hop counter, a Bloom filter over the peers that have processed the -request already and depending on the routing options the full path traversed by -the GET. Finally, a GET request includes a variable-size second Bloom filter -and a so-called Bloom filter mutator value which together indicate which -replies the sender has already seen. During the lookup, each block that matches -they block type, key and extended query is additionally subjected to a test -against this Bloom filter. The block plugin is expected to take the hash of the -block and combine it with the mutator value and check if the result is not yet -in the Bloom filter. The originator of the query will from time to time modify -the mutator to (eventually) allow false-positives filtered by the Bloom filter -to be returned. - -Peers that receive a GET request perform a local lookup (depending on their -proximity to the key and the query options) and forward the request to other -peers. They then remember the request (including the Bloom filter for blocking -duplicate results) and when they obtain a matching, non-filtered response a -@code{struct PeerResultMessage} of type@ -@code{GNUNET_MESSAGE_TYPE_DHT_P2P_RESULT} is forwarded to the previous hop. -Whenver a result is forwarded, the block plugin is used to update the Bloom -filter accordingly, to ensure that the same result is never forwarded more than -once. The DHT service may also cache forwarded results locally if the -"CACHE_RESULTS" option is set to "YES" in the configuration. - -@node The GNU Name System (GNS) -@section The GNU Name System (GNS) - -@c %**end of header - -The GNU Name System (GNS) is a decentralized database that enables users to -securely resolve names to values. Names can be used to identify other users -(for example, in social networking), or network services (for example, VPN -services running at a peer in GNUnet, or purely IP-based services on the -Internet). Users interact with GNS by typing in a hostname that ends in ".gnu" -or ".zkey". - -Videos giving an overview of most of the GNS and the motivations behind it is -available here and here. The remainder of this chapter targets developers that -are familiar with high level concepts of GNS as presented in these talks. - -GNS-aware applications should use the GNS resolver to obtain the respective -records that are stored under that name in GNS. Each record consists of a type, -value, expiration time and flags. - -The type specifies the format of the value. Types below 65536 correspond to DNS -record types, larger values are used for GNS-specific records. Applications can -define new GNS record types by reserving a number and implementing a plugin -(which mostly needs to convert the binary value representation to a -human-readable text format and vice-versa). The expiration time specifies how -long the record is to be valid. The GNS API ensures that applications are only -given non-expired values. The flags are typically irrelevant for applications, -as GNS uses them internally to control visibility and validity of records. - -Records are stored along with a signature. The signature is generated using the -private key of the authoritative zone. This allows any GNS resolver to verify -the correctness of a name-value mapping. - -Internally, GNS uses the NAMECACHE to cache information obtained from other -users, the NAMESTORE to store information specific to the local users, and the -DHT to exchange data between users. A plugin API is used to enable applications -to define new GNS record types. - -@menu -* libgnunetgns:: -* libgnunetgnsrecord:: -* GNS plugins:: -* The GNS Client-Service Protocol:: -* Hijacking the DNS-Traffic using gnunet-service-dns:: -* Serving DNS lookups via GNS on W32:: -@end menu - -@node libgnunetgns -@subsection libgnunetgns - -@c %**end of header - -The GNS API itself is extremely simple. Clients first connec to the GNS service -using @code{GNUNET_GNS_connect}. They can then perform lookups using -@code{GNUNET_GNS_lookup} or cancel pending lookups using -@code{GNUNET_GNS_lookup_cancel}. Once finished, clients disconnect using -@code{GNUNET_GNS_disconnect}. - - -@menu -* Looking up records:: -* Accessing the records:: -* Creating records:: -* Future work:: -@end menu - -@node Looking up records -@subsubsection Looking up records - -@c %**end of header - -@code{GNUNET_GNS_lookup} takes a number of arguments: - -@table @asis -@item handle This is simply the GNS connection handle from -@code{GNUNET_GNS_connect}. -@item name The client needs to specify the name to -be resolved. This can be any valid DNS or GNS hostname. -@item zone The client -needs to specify the public key of the GNS zone against which the resolution -should be done (the ".gnu" zone). Note that a key must be provided, even if the -name ends in ".zkey". This should typically be the public key of the -master-zone of the user. -@item type This is the desired GNS or DNS record type -to look for. While all records for the given name will be returned, this can be -important if the client wants to resolve record types that themselves delegate -resolution, such as CNAME, PKEY or GNS2DNS. Resolving a record of any of these -types will only work if the respective record type is specified in the request, -as the GNS resolver will otherwise follow the delegation and return the records -from the respective destination, instead of the delegating record. -@item only_cached This argument should typically be set to @code{GNUNET_NO}. Setting -it to @code{GNUNET_YES} disables resolution via the overlay network. -@item shorten_zone_key If GNS encounters new names during resolution, their -respective zones can automatically be learned and added to the "shorten zone". -If this is desired, clients must pass the private key of the shorten zone. If -NULL is passed, shortening is disabled. -@item proc This argument identifies -the function to call with the result. It is given proc_cls, the number of -records found (possilby zero) and the array of the records as arguments. proc -will only be called once. After proc,> has been called, the lookup must no -longer be cancelled. -@item proc_cls The closure for proc. -@end table - -@node Accessing the records -@subsubsection Accessing the records - -@c %**end of header - -The @code{libgnunetgnsrecord} library provides an API to manipulate the GNS -record array that is given to proc. In particular, it offers functions such as -converting record values to human-readable strings (and back). However, most -@code{libgnunetgnsrecord} functions are not interesting to GNS client -applications. - -For DNS records, the @code{libgnunetdnsparser} library provides functions for -parsing (and serializing) common types of DNS records. - -@node Creating records -@subsubsection Creating records - -@c %**end of header - -Creating GNS records is typically done by building the respective record -information (possibly with the help of @code{libgnunetgnsrecord} and -@code{libgnunetdnsparser}) and then using the @code{libgnunetnamestore} to -publish the information. The GNS API is not involved in this -operation. - -@node Future work -@subsubsection Future work - -@c %**end of header - -In the future, we want to expand @code{libgnunetgns} to allow applications to -observe shortening operations performed during GNS resolution, for example so -that users can receive visual feedback when this happens. - -@node libgnunetgnsrecord -@subsection libgnunetgnsrecord - -@c %**end of header - -The @code{libgnunetgnsrecord} library is used to manipulate GNS records (in -plaintext or in their encrypted format). Applications mostly interact with -@code{libgnunetgnsrecord} by using the functions to convert GNS record values -to strings or vice-versa, or to lookup a GNS record type number by name (or -vice-versa). The library also provides various other functions that are mostly -used internally within GNS, such as converting keys to names, checking for -expiration, encrypting GNS records to GNS blocks, verifying GNS block -signatures and decrypting GNS records from GNS blocks. - -We will now discuss the four commonly used functions of the API.@ -@code{libgnunetgnsrecord} does not perform these operations itself, but instead -uses plugins to perform the operation. GNUnet includes plugins to support -common DNS record types as well as standard GNS record types. - - -@menu -* Value handling:: -* Type handling:: -@end menu - -@node Value handling -@subsubsection Value handling - -@c %**end of header - -@code{GNUNET_GNSRECORD_value_to_string} can be used to convert the (binary) -representation of a GNS record value to a human readable, 0-terminated UTF-8 -string. NULL is returned if the specified record type is not supported by any -available plugin. - -@code{GNUNET_GNSRECORD_string_to_value} can be used to try to convert a human -readable string to the respective (binary) representation of a GNS record -value. - -@node Type handling -@subsubsection Type handling - -@c %**end of header - -@code{GNUNET_GNSRECORD_typename_to_number} can be used to obtain the numeric -value associated with a given typename. For example, given the typename "A" -(for DNS A reocrds), the function will return the number 1. A list of common -DNS record types is -@uref{http://en.wikipedia.org/wiki/List_of_DNS_record_types, here. Note that -not all DNS record types are supported by GNUnet GNSRECORD plugins at this -time.} - -@code{GNUNET_GNSRECORD_number_to_typename} can be used to obtain the typename -associated with a given numeric value. For example, given the type number 1, -the function will return the typename "A". - -@node GNS plugins -@subsection GNS plugins - -@c %**end of header - -Adding a new GNS record type typically involves writing (or extending) a -GNSRECORD plugin. The plugin needs to implement the -@code{gnunet_gnsrecord_plugin.h} API which provides basic functions that are -needed by GNSRECORD to convert typenames and values of the respective record -type to strings (and back). These gnsrecord plugins are typically implemented -within their respective subsystems. Examples for such plugins can be found in -the GNSRECORD, GNS and CONVERSATION subsystems. - -The @code{libgnunetgnsrecord} library is then used to locate, load and query -the appropriate gnsrecord plugin. Which plugin is appropriate is determined by -the record type (which is just a 32-bit integer). The @code{libgnunetgnsrecord} -library loads all block plugins that are installed at the local peer and -forwards the application request to the plugins. If the record type is not -supported by the plugin, it should simply return an error code. - -The central functions of the block APIs (plugin and main library) are the same -four functions for converting between values and strings, and typenames and -numbers documented in the previous subsection. - -@node The GNS Client-Service Protocol -@subsection The GNS Client-Service Protocol - -@c %**end of header - -The GNS client-service protocol consists of two simple messages, the -@code{LOOKUP} message and the @code{LOOKUP_RESULT}. Each @code{LOOKUP} message -contains a unique 32-bit identifier, which will be included in the -corresponding response. Thus, clients can send many lookup requests in parallel -and receive responses out-of-order. A @code{LOOKUP} request also includes the -public key of the GNS zone, the desired record type and fields specifying -whether shortening is enabled or networking is disabled. Finally, the -@code{LOOKUP} message includes the name to be resolved. - -The response includes the number of records and the records themselves in the -format created by @code{GNUNET_GNSRECORD_records_serialize}. They can thus be -deserialized using @code{GNUNET_GNSRECORD_records_deserialize}. - -@node Hijacking the DNS-Traffic using gnunet-service-dns -@subsection Hijacking the DNS-Traffic using gnunet-service-dns - -@c %**end of header - -This section documents how the gnunet-service-dns (and the gnunet-helper-dns) -intercepts DNS queries from the local system.@ This is merely one method for -how we can obtain GNS queries. It is also possible to change @code{resolv.conf} -to point to a machine running @code{gnunet-dns2gns} or to modify libc's name -system switch (NSS) configuration to include a GNS resolution plugin. The -method described in this chaper is more of a last-ditch catch-all approach. - -@code{gnunet-service-dns} enables intercepting DNS traffic using policy based -routing. We MARK every outgoing DNS-packet if it was not sent by our -application. Using a second routing table in the Linux kernel these marked -packets are then routed through our virtual network interface and can thus be -captured unchanged. - -Our application then reads the query and decides how to handle it: A query to -an address ending in ".gnu" or ".zkey" is hijacked by @code{gnunet-service-gns} -and resolved internally using GNS. In the future, a reverse query for an -address of the configured virtual network could be answered with records kept -about previous forward queries. Queries that are not hijacked by some -application using the DNS service will be sent to the original recipient. The -answer to the query will always be sent back through the virtual interface with -the original nameserver as source address. - - -@menu -* Network Setup Details:: -@end menu - -@node Network Setup Details -@subsubsection Network Setup Details - -@c %**end of header - -The DNS interceptor adds the following rules to the Linux kernel: -@example -iptables -t mangle -I OUTPUT 1 -p udp --sport $LOCALPORT --dport 53 -j -ACCEPT iptables -t mangle -I OUTPUT 2 -p udp --dport 53 -j MARK --set-mark 3 ip -rule add fwmark 3 table2 ip route add default via $VIRTUALDNS table2 -@end example - -Line 1 makes sure that all packets coming from a port our application opened -beforehand (@code{$LOCALPORT}) will be routed normally. Line 2 marks every -other packet to a DNS-Server with mark 3 (chosen arbitrarily). The third line -adds a routing policy based on this mark 3 via the routing table. - -@node Serving DNS lookups via GNS on W32 -@subsection Serving DNS lookups via GNS on W32 - -@c %**end of header - -This section documents how the libw32nsp (and gnunet-gns-helper-service-w32) do -DNS resolutions of DNS queries on the local system. This only applies to GNUnet -running on W32. - -W32 has a concept of "Namespaces" and "Namespace providers". These are used to -present various name systems to applications in a generic way. Namespaces -include DNS, mDNS, NLA and others. For each namespace any number of providers -could be registered, and they are queried in an order of priority (which is -adjustable). - -Applications can resolve names by using WSALookupService*() family of -functions. - -However, these are WSA-only facilities. Common BSD socket functions for -namespace resolutions are gethostbyname and getaddrinfo (among others). These -functions are implemented internally (by default - by mswsock, which also -implements the default DNS provider) as wrappers around WSALookupService*() -functions (see "Sample Code for a Service Provider" on MSDN). - -On W32 GNUnet builds a libw32nsp - a namespace provider, which can then be -installed into the system by using w32nsp-install (and uninstalled by -w32nsp-uninstall), as described in "Installation Handbook". - -libw32nsp is very simple and has almost no dependencies. As a response to -NSPLookupServiceBegin(), it only checks that the provider GUID passed to it by -the caller matches GNUnet DNS Provider GUID, checks that name being resolved -ends in ".gnu" or ".zkey", then connects to gnunet-gns-helper-service-w32 at -127.0.0.1:5353 (hardcoded) and sends the name resolution request there, -returning the connected socket to the caller. - -When the caller invokes NSPLookupServiceNext(), libw32nsp reads a completely -formed reply from that socket, unmarshalls it, then gives it back to the -caller. - -At the moment gnunet-gns-helper-service-w32 is implemented to ever give only -one reply, and subsequent calls to NSPLookupServiceNext() will fail with -WSA_NODATA (first call to NSPLookupServiceNext() might also fail if GNS failed -to find the name, or there was an error connecting to it). - -gnunet-gns-helper-service-w32 does most of the processing: - -@itemize @bullet -@item Maintains a connection to GNS. -@item Reads GNS config and loads appropriate keys. -@item Checks service GUID and decides on the type of record to look up, -refusing to make a lookup outright when unsupported service GUID is passed. -@item Launches the lookup -@end itemize - -When lookup result arrives, gnunet-gns-helper-service-w32 forms a complete -reply (including filling a WSAQUERYSETW structure and, possibly, a binary blob -with a hostent structure for gethostbyname() client), marshalls it, and sends -it back to libw32nsp. If no records were found, it sends an empty header. - -This works for most normal applications that use gethostbyname() or -getaddrinfo() to resolve names, but fails to do anything with applications that -use alternative means of resolving names (such as sending queries to a DNS -server directly by themselves). This includes some of well known utilities, -like "ping" and "nslookup". - -@node The GNS Namecache -@section The GNS Namecache - -@c %**end of header - -The NAMECACHE subsystem is responsible for caching (encrypted) resolution -results of the GNU Name System (GNS). GNS makes zone information available to -other users via the DHT. However, as accessing the DHT for every lookup is -expensive (and as the DHT's local cache is lost whenever the peer is -restarted), GNS uses the NAMECACHE as a more persistent cache for DHT lookups. -Thus, instead of always looking up every name in the DHT, GNS first checks if -the result is already available locally in the NAMECACHE. Only if there is no -result in the NAMECACHE, GNS queries the DHT. The NAMECACHE stores data in the -same (encrypted) format as the DHT. It thus makes no sense to iterate over all -items in the NAMECACHE --- the NAMECACHE does not have a way to provide the -keys required to decrypt the entries. - -Blocks in the NAMECACHE share the same expiration mechanism as blocks in the -DHT --- the block expires wheneever any of the records in the (encrypted) block -expires. The expiration time of the block is the only information stored in -plaintext. The NAMECACHE service internally performs all of the required work -to expire blocks, clients do not have to worry about this. Also, given that -NAMECACHE stores only GNS blocks that local users requested, there is no -configuration option to limit the size of the NAMECACHE. It is assumed to be -always small enough (a few MB) to fit on the drive. - -The NAMECACHE supports the use of different database backends via a plugin API. - -@menu -* libgnunetnamecache:: -* The NAMECACHE Client-Service Protocol:: -* The NAMECACHE Plugin API:: -@end menu - -@node libgnunetnamecache -@subsection libgnunetnamecache - -@c %**end of header - -The NAMECACHE API consists of five simple functions. First, there is -@code{GNUNET_NAMECACHE_connect} to connect to the NAMECACHE service. This -returns the handle required for all other operations on the NAMECACHE. Using -@code{GNUNET_NAMECACHE_block_cache} clients can insert a block into the cache. -@code{GNUNET_NAMECACHE_lookup_block} can be used to lookup blocks that were -stored in the NAMECACHE. Both operations can be cancelled using -@code{GNUNET_NAMECACHE_cancel}. Note that cancelling a -@code{GNUNET_NAMECACHE_block_cache} operation can result in the block being -stored in the NAMECACHE --- or not. Cancellation primarily ensures that the -continuation function with the result of the operation will no longer be -invoked. Finally, @code{GNUNET_NAMECACHE_disconnect} closes the connection to -the NAMECACHE. - -The maximum size of a block that can be stored in the NAMECACHE is -@code{GNUNET_NAMECACHE_MAX_VALUE_SIZE}, which is defined to be 63 kB. - -@node The NAMECACHE Client-Service Protocol -@subsection The NAMECACHE Client-Service Protocol - -@c %**end of header - -All messages in the NAMECACHE IPC protocol start with the @code{struct -GNUNET_NAMECACHE_Header} which adds a request ID (32-bit integer) to the -standard message header. The request ID is used to match requests with the -respective responses from the NAMECACHE, as they are allowed to happen -out-of-order. - - -@menu -* Lookup:: -* Store:: -@end menu - -@node Lookup -@subsubsection Lookup - -@c %**end of header - -The @code{struct LookupBlockMessage} is used to lookup a block stored in the -cache. It contains the query hash. The NAMECACHE always responds with a -@code{struct LookupBlockResponseMessage}. If the NAMECACHE has no response, it -sets the expiration time in the response to zero. Otherwise, the response is -expected to contain the expiration time, the ECDSA signature, the derived key -and the (variable-size) encrypted data of the block. - -@node Store -@subsubsection Store - -@c %**end of header - -The @code{struct BlockCacheMessage} is used to cache a block in the NAMECACHE. -It has the same structure as the @code{struct LookupBlockResponseMessage}. The -service responds with a @code{struct BlockCacheResponseMessage} which contains -the result of the operation (success or failure). In the future, we might want -to make it possible to provide an error message as well. - -@node The NAMECACHE Plugin API -@subsection The NAMECACHE Plugin API -@c %**end of header - -The NAMECACHE plugin API consists of two functions, @code{cache_block} to store -a block in the database, and @code{lookup_block} to lookup a block in the -database. - - -@menu -* Lookup2:: -* Store2:: -@end menu - -@node Lookup2 -@subsubsection Lookup2 - -@c %**end of header - -The @code{lookup_block} function is expected to return at most one block to the -iterator, and return @code{GNUNET_NO} if there were no non-expired results. If -there are multiple non-expired results in the cache, the lookup is supposed to -return the result with the largest expiration time. - -@node Store2 -@subsubsection Store2 - -@c %**end of header - -The @code{cache_block} function is expected to try to store the block in the -database, and return @code{GNUNET_SYSERR} if this was not possible for any -reason. Furthermore, @code{cache_block} is expected to implicitly perform cache -maintenance and purge blocks from the cache that have expired. Note that -@code{cache_block} might encounter the case where the database already has -another block stored under the same key. In this case, the plugin must ensure -that the block with the larger expiration time is preserved. Obviously, this -can done either by simply adding new blocks and selecting for the most recent -expiration time during lookup, or by checking which block is more recent during -the store operation. - -@node The REVOCATION Subsystem -@section The REVOCATION Subsystem -@c %**end of header - -The REVOCATION subsystem is responsible for key revocation of Egos. If a user -learns that his private key has been compromised or has lost it, he can use the -REVOCATION system to inform all of the other users that this private key is no -longer valid. The subsystem thus includes ways to query for the validity of -keys and to propagate revocation messages. - -@menu -* Dissemination:: -* Revocation Message Design Requirements:: -* libgnunetrevocation:: -* The REVOCATION Client-Service Protocol:: -* The REVOCATION Peer-to-Peer Protocol:: -@end menu - -@node Dissemination -@subsection Dissemination - -@c %**end of header - -When a revocation is performed, the revocation is first of all disseminated by -flooding the overlay network. The goal is to reach every peer, so that when a -peer needs to check if a key has been revoked, this will be purely a local -operation where the peer looks at his local revocation list. Flooding the -network is also the most robust form of key revocation --- an adversary would -have to control a separator of the overlay graph to restrict the propagation of -the revocation message. Flooding is also very easy to implement --- peers that -receive a revocation message for a key that they have never seen before simply -pass the message to all of their neighbours. - -Flooding can only distribute the revocation message to peers that are online. -In order to notify peers that join the network later, the revocation service -performs efficient set reconciliation over the sets of known revocation -messages whenever two peers (that both support REVOCATION dissemination) -connect. The SET service is used to perform this operation -efficiently. - -@node Revocation Message Design Requirements -@subsection Revocation Message Design Requirements - -@c %**end of header - -However, flooding is also quite costly, creating O(|E|) messages on a network -with |E| edges. Thus, revocation messages are required to contain a -proof-of-work, the result of an expensive computation (which, however, is cheap -to verify). Only peers that have expended the CPU time necessary to provide -this proof will be able to flood the network with the revocation message. This -ensures that an attacker cannot simply flood the network with millions of -revocation messages. The proof-of-work required by GNUnet is set to take days -on a typical PC to compute; if the ability to quickly revoke a key is needed, -users have the option to pre-compute revocation messages to store off-line and -use instantly after their key has expired. - -Revocation messages must also be signed by the private key that is being -revoked. Thus, they can only be created while the private key is in the -possession of the respective user. This is another reason to create a -revocation message ahead of time and store it in a secure location. - -@node libgnunetrevocation -@subsection libgnunetrevocation - -@c %**end of header - -The REVOCATION API consists of two parts, to query and to issue -revocations. - - -@menu -* Querying for revoked keys:: -* Preparing revocations:: -* Issuing revocations:: -@end menu - -@node Querying for revoked keys -@subsubsection Querying for revoked keys - -@c %**end of header - -@code{GNUNET_REVOCATION_query} is used to check if a given ECDSA public key has -been revoked. The given callback will be invoked with the result of the check. -The query can be cancelled using @code{GNUNET_REVOCATION_query_cancel} on the -return value. - -@node Preparing revocations -@subsubsection Preparing revocations - -@c %**end of header - -It is often desirable to create a revocation record ahead-of-time and store it -in an off-line location to be used later in an emergency. This is particularly -true for GNUnet revocations, where performing the revocation operation itself -is computationally expensive and thus is likely to take some time. Thus, if -users want the ability to perform revocations quickly in an emergency, they -must pre-compute the revocation message. The revocation API enables this with -two functions that are used to compute the revocation message, but not trigger -the actual revocation operation. - -@code{GNUNET_REVOCATION_check_pow} should be used to calculate the -proof-of-work required in the revocation message. This function takes the -public key, the required number of bits for the proof of work (which in GNUnet -is a network-wide constant) and finally a proof-of-work number as arguments. -The function then checks if the given proof-of-work number is a valid proof of -work for the given public key. Clients preparing a revocation are expected to -call this function repeatedly (typically with a monotonically increasing -sequence of numbers of the proof-of-work number) until a given number satisfies -the check. That number should then be saved for later use in the revocation -operation. - -@code{GNUNET_REVOCATION_sign_revocation} is used to generate the signature that -is required in a revocation message. It takes the private key that (possibly in -the future) is to be revoked and returns the signature. The signature can again -be saved to disk for later use, which will then allow performing a revocation -even without access to the private key. - -@node Issuing revocations -@subsubsection Issuing revocations - - -Given a ECDSA public key, the signature from @code{GNUNET_REVOCATION_sign} and -the proof-of-work, @code{GNUNET_REVOCATION_revoke} can be used to perform the -actual revocation. The given callback is called upon completion of the -operation. @code{GNUNET_REVOCATION_revoke_cancel} can be used to stop the -library from calling the continuation; however, in that case it is undefined -whether or not the revocation operation will be executed. - -@node The REVOCATION Client-Service Protocol -@subsection The REVOCATION Client-Service Protocol - - -The REVOCATION protocol consists of four simple messages. - -A @code{QueryMessage} containing a public ECDSA key is used to check if a -particular key has been revoked. The service responds with a -@code{QueryResponseMessage} which simply contains a bit that says if the given -public key is still valid, or if it has been revoked. - -The second possible interaction is for a client to revoke a key by passing a -@code{RevokeMessage} to the service. The @code{RevokeMessage} contains the -ECDSA public key to be revoked, a signature by the corresponding private key -and the proof-of-work, The service responds with a -@code{RevocationResponseMessage} which can be used to indicate that the -@code{RevokeMessage} was invalid (i.e. proof of work incorrect), or otherwise -indicates that the revocation has been processed successfully. - -@node The REVOCATION Peer-to-Peer Protocol -@subsection The REVOCATION Peer-to-Peer Protocol - -@c %**end of header - -Revocation uses two disjoint ways to spread revocation information among peers. -First of all, P2P gossip exchanged via CORE-level neighbours is used to quickly -spread revocations to all connected peers. Second, whenever two peers (that -both support revocations) connect, the SET service is used to compute the union -of the respective revocation sets. - -In both cases, the exchanged messages are @code{RevokeMessage}s which contain -the public key that is being revoked, a matching ECDSA signature, and a -proof-of-work. Whenever a peer learns about a new revocation this way, it first -validates the signature and the proof-of-work, then stores it to disk -(typically to a file $GNUNET_DATA_HOME/revocation.dat) and finally spreads the -information to all directly connected neighbours. - -For computing the union using the SET service, the peer with the smaller hashed -peer identity will connect (as a "client" in the two-party set protocol) to the -other peer after one second (to reduce traffic spikes on connect) and initiate -the computation of the set union. All revocation services use a common hash to -identify the SET operation over revocation sets. - -The current implementation accepts revocation set union operations from all -peers at any time; however, well-behaved peers should only initiate this -operation once after establishing a connection to a peer with a larger hashed -peer identity. - -@node GNUnet's File-sharing (FS) Subsystem -@section GNUnet's File-sharing (FS) Subsystem - -@c %**end of header - -This chapter describes the details of how the file-sharing service works. As -with all services, it is split into an API (libgnunetfs), the service process -(gnunet-service-fs) and user interface(s). The file-sharing service uses the -datastore service to store blocks and the DHT (and indirectly datacache) for -lookups for non-anonymous file-sharing.@ Furthermore, the file-sharing service -uses the block library (and the block fs plugin) for validation of DHT -operations. - -In contrast to many other services, libgnunetfs is rather complex since the -client library includes a large number of high-level abstractions; this is -necessary since the Fs service itself largely only operates on the block level. -The FS library is responsible for providing a file-based abstraction to -applications, including directories, meta data, keyword search, verification, -and so on. - -The method used by GNUnet to break large files into blocks and to use keyword -search is called the "Encoding for Censorship Resistant Sharing" (ECRS). ECRS -is largely implemented in the fs library; block validation is also reflected in -the block FS plugin and the FS service. ECRS on-demand encoding is implemented -in the FS service. - -NOTE: The documentation in this chapter is quite incomplete. - -@menu -* Encoding for Censorship-Resistant Sharing (ECRS):: -* File-sharing persistence directory structure:: -@end menu - -@node Encoding for Censorship-Resistant Sharing (ECRS) -@subsection Encoding for Censorship-Resistant Sharing (ECRS) - -@c %**end of header - -When GNUnet shares files, it uses a content encoding that is called ECRS, the -Encoding for Censorship-Resistant Sharing. Most of ECRS is described in the -(so far unpublished) research paper attached to this page. ECRS obsoletes the -previous ESED and ESED II encodings which were used in GNUnet before version -0.7.0.@ @ The rest of this page assumes that the reader is familiar with the -attached paper. What follows is a description of some minor extensions that -GNUnet makes over what is described in the paper. The reason why these -extensions are not in the paper is that we felt that they were obvious or -trivial extensions to the original scheme and thus did not warrant space in -the research report. - - -@menu -* Namespace Advertisements:: -* KSBlocks:: -@end menu - -@node Namespace Advertisements -@subsubsection Namespace Advertisements - -@c %**end of header -@c %**FIXME: all zeroses -> ? - -An @code{SBlock} with identifier all zeros is a signed -advertisement for a namespace. This special @code{SBlock} contains metadata -describing the content of the namespace. Instead of the name of the identifier -for a potential update, it contains the identifier for the root of the -namespace. The URI should always be empty. The @code{SBlock} is signed with -the content provder's RSA private key (just like any other SBlock). Peers -can search for @code{SBlock}s in order to find out more about a namespace. - -@node KSBlocks -@subsubsection KSBlocks - -@c %**end of header - -GNUnet implements @code{KSBlocks} which are @code{KBlocks} that, instead of -encrypting a CHK and metadata, encrypt an @code{SBlock} instead. In other -words, @code{KSBlocks} enable GNUnet to find @code{SBlocks} using the global -keyword search. Usually the encrypted @code{SBlock} is a namespace -advertisement. The rationale behind @code{KSBlock}s and @code{SBlock}s is to -enable peers to discover namespaces via keyword searches, and, to associate -useful information with namespaces. When GNUnet finds @code{KSBlocks} during a -normal keyword search, it adds the information to an internal list of -discovered namespaces. Users looking for interesting namespaces can then -inspect this list, reducing the need for out-of-band discovery of namespaces. -Naturally, namespaces (or more specifically, namespace advertisements) can -also be referenced from directories, but @code{KSBlock}s should make it easier -to advertise namespaces for the owner of the pseudonym since they eliminate -the need to first create a directory. - -Collections are also advertised using @code{KSBlock}s. - -@table @asis -@item Attachment Size -@item ecrs.pdf 270.68 KB -@item https://gnunet.org/sites/default/files/ecrs.pdf -@end table - -@node File-sharing persistence directory structure -@subsection File-sharing persistence directory structure - -@c %**end of header - -This section documents how the file-sharing library implements persistence of -file-sharing operations and specifically the resulting directory structure. -This code is only active if the @code{GNUNET_FS_FLAGS_PERSISTENCE} flag was set -when calling @code{GNUNET_FS_start}. In this case, the file-sharing library -will try hard to ensure that all major operations (searching, downloading, -publishing, unindexing) are persistent, that is, can live longer than the -process itself. More specifically, an operation is supposed to live until it is -explicitly stopped. - -If @code{GNUNET_FS_stop} is called before an operation has been stopped, a -@code{SUSPEND} event is generated and then when the process calls -@code{GNUNET_FS_start} next time, a @code{RESUME} event is generated. -Additionally, even if an application crashes (segfault, SIGKILL, system crash) -and hence @code{GNUNET_FS_stop} is never called and no @code{SUSPEND} events -are generated, operations are still resumed (with @code{RESUME} events). This -is implemented by constantly writing the current state of the file-sharing -operations to disk. Specifically, the current state is always written to disk -whenever anything significant changes (the exception are block-wise progress in -publishing and unindexing, since those operations would be slowed down -significantly and can be resumed cheaply even without detailed accounting). -Note that@ if the process crashes (or is killed) during a serialization -operation, FS does not guarantee that this specific operation is recoverable -(no strict transactional semantics, again for performance reasons). However, -all other unrelated operations should resume nicely. - -Since we need to serialize the state continuously and want to recover as much -as possible even after crashing during a serialization operation, we do not use -one large file for serialization. Instead, several directories are used for the -various operations. When @code{GNUNET_FS_start} executes, the master -directories are scanned for files describing operations to resume. Sometimes, -these operations can refer to related operations in child directories which may -also be resumed at this point. Note that corrupted files are cleaned up -automatically. However, dangling files in child directories (those that are not -referenced by files from the master directories) are not automatically removed. - -Persistence data is kept in a directory that begins with the "STATE_DIR" prefix -from the configuration file (by default, "$SERVICEHOME/persistence/") followed -by the name of the client as given to @code{GNUNET_FS_start} (for example, -"gnunet-gtk") followed by the actual name of the master or child directory. - -The names for the master directories follow the names of the operations: - -@itemize @bullet -@item "search" -@item "download" -@item "publish" -@item "unindex" -@end itemize - -Each of the master directories contains names (chosen at random) for each -active top-level (master) operation. Note that a download that is associated -with a search result is not a top-level operation. - -In contrast to the master directories, the child directories are only consulted -when another operation refers to them. For each search, a subdirectory (named -after the master search synchronization file) contains the search results. -Search results can have an associated download, which is then stored in the -general "download-child" directory. Downloads can be recursive, in which case -children are stored in subdirectories mirroring the structure of the recursive -download (either starting in the master "download" directory or in the -"download-child" directory depending on how the download was initiated). For -publishing operations, the "publish-file" directory contains information about -the individual files and directories that are part of the publication. However, -this directory structure is flat and does not mirror the structure of the -publishing operation. Note that unindex operations cannot have associated child -operations. - -@node GNUnet's REGEX Subsystem -@section GNUnet's REGEX Subsystem - -@c %**end of header - -Using the REGEX subsystem, you can discover peers that offer a particular -service using regular expressions. The peers that offer a service specify it -using a regular expressions. Peers that want to patronize a service search -using a string. The REGEX subsystem will then use the DHT to return a set of -matching offerers to the patrons. - -For the technical details, we have "Max's defense talk and Max's Master's -thesis. An additional publication is under preparation and available to team -members (in Git). - -@menu -* How to run the regex profiler:: -@end menu - -@node How to run the regex profiler -@subsection How to run the regex profiler - -@c %**end of header - -The gnunet-regex-profiler can be used to profile the usage of mesh/regex for a -given set of regular expressions and strings. Mesh/regex allows you to announce -your peer ID under a certain regex and search for peers matching a particular -regex using a string. See https://gnunet.org/szengel2012ms for a full -introduction. - -First of all, the regex profiler uses GNUnet testbed, thus all the implications -for testbed also apply to the regex profiler (for example you need -password-less ssh login to the machines listed in your hosts file). - -@strong{Configuration} - -Moreover, an appropriate configuration file is needed. Generally you can refer -to SVN HEAD: contrib/regex_profiler_infiniband.conf for an example -configuration. In the following paragraph the important details are -highlighted. - -Announcing of the regular expressions is done by the -gnunet-daemon-regexprofiler, therefore you have to make sure it is started, by -adding it to the AUTOSTART set of ARM:@ -@code{ -[regexprofiler]@ -AUTOSTART = YES@ -} - -Furthermore you have to specify the location of the binary: -@example -[regexprofiler] -# Location of the gnunet-daemon-regexprofiler binary. -BINARY = /home/szengel/gnunet/src/mesh/.libs/gnunet-daemon-regexprofiler -# Regex prefix that will be applied to all regular expressions and -# search string. -REGEX_PREFIX = "GNVPN-0001-PAD" -@end example - -When running the profiler with a large scale deployment, you probably want to -reduce the workload of each peer. Use the following options to do this.@ -@example -[dht]@ -# Force network size estimation@ -FORCE_NSE = 1 - -[dhtcache] -DATABASE = heap@ -# Disable RC-file for Bloom filter? (for benchmarking with limited IO -# availability)@ -DISABLE_BF_RC = YES@ -# Disable Bloom filter entirely@ -DISABLE_BF = YES - -[nse]@ -# Minimize proof-of-work CPU consumption by NSE@ -WORKBITS = 1 -@end example - - -@strong{Options} - -To finally run the profiler some options and the input data need to be -specified on the command line. -@code{@ gnunet-regex-profiler -c config-file -d -log-file -n num-links -p@ path-compression-length -s search-delay -t -matching-timeout -a num-search-strings hosts-file policy-dir -search-strings-file@ } - -@code{config-file} the configuration file created earlier.@ @code{log-file} -file where to write statistics output.@ @code{num-links} number of random links -between started peers.@ @code{path-compression-length} maximum path compression -length in the DFA.@ @code{search-delay} time to wait between peers finished -linking and@ starting to match strings.@ @code{matching-timeout} timeout after -witch to cancel the searching.@ @code{num-search-strings} number of strings in -the search-strings-file. - -The @code{hosts-file} should contain a list of hosts for the testbed, one per -line in the following format. @code{user@@host_ip:port}. - -The @code{policy-dir} is a folder containing text files containing one or more -regular expressions. A peer is started for each file in that folder and the -regular expressions in the corresponding file are announced by this peer. - -The @code{search-strings-file} is a text file containing search strings, one in -each line. - -You can create regular expressions and search strings for every AS in the@ -Internet using the attached scripts. You need one of the -@uref{http://data.caida.org/datasets/routing/routeviews-prefix2as/, CAIDA -routeviews prefix2as} data files for this. Run @code{create_regex.py <filename> -<output path>} to create the regular expressions and @code{create_strings.py -<input path> <outfile>} to create a search strings file from the previously -created regular expressions. diff --git a/doc/chapters/installation.texi b/doc/chapters/installation.texi deleted file mode 100644 index 14c10d2b01..0000000000 --- a/doc/chapters/installation.texi +++ /dev/null @@ -1,3625 +0,0 @@ -@node GNUnet Installation Handbook -@chapter GNUnet Installation Handbook - -This handbook describes how to install (build setup, compilation) and setup -(configuration, start) GNUnet 0.10.x. After following these instructions you -should be able to install and then start user-interfaces to interact with the -network. - -This manual is far from complete, and we welcome informed contributions, be it -in the form of new chapters or insightful comments. - - - -@menu -* Dependencies:: -* Pre-installation notes:: -* Generic installation instructions:: -* Build instructions for Ubuntu 12.04 using Git:: -* Build Instructions for Microsoft Windows Platforms:: -* Build instructions for Debian 7.5:: -* Installing GNUnet from Git on Ubuntu 14.4:: -* Build instructions for Debian 8:: -* Outdated build instructions for previous revisions:: -* Portable GNUnet:: -* The graphical configuration interface:: -* How to start and stop a GNUnet peer:: -@end menu - -@node Dependencies -@section Dependencies -@c %**end of header - -This document lists the various known dependencies for GNUnet 0.10.x. -Suggestions for missing dependencies or wrong version numbers are welcome. - - - -@menu -* External dependencies:: -* Fixing libgnurl build issues:: -* Internal dependencies:: -@end menu - -@node External dependencies -@subsection External dependencies -@c %**end of header - -These packages must be installed before a typical GNUnet installation -can be performed: - -@table @asis -@item GNU libmicrohttpd 0.9.30 or higher -@item GNU libextractor 1.0 or higher -@item GNU libtool 2.2 or higher -@item GNU libunistring 0.9.1.1 or higher -@item GNU libidn 1.0.0 or higher -@item @uref{https://gnupg.org/software/libgcrypt/index.html, GNU libgcrypt} -@uref{https://gnupg.org/ftp/gcrypt/libgcrypt/, 1.6.0} or higher -@item @uref{https://gnutls.org/, GnuTLS} -@uref{https://www.gnupg.org/ftp/gcrypt/gnutls/v3.2/, 3.2.7} or higher, -compile with libunbound for DANE support; GnuTLS also requires GNU -nettle 2.7 (update: GnuTLS 3.2.7 appears NOT to work against GNU nettle -> 2.7, due to some API updatings done by nettle. Thus it should be compiled -against nettle 2.7 and, in case you get some error on the reference to -`rpl_strerror' being undefined, follow the instructions on@ -@uref{http://lists.gnupg.org/pipermail/gnutls-devel/2013-November/006588.html, this} -post (and the link inside it)). -@item @uref{https://gnunet.org/gnurl, gnURL} libgnurl 7.34.0 or higher, -must be compiled after @code{GnuTLS} -@item libglpk 4.45 or higher -@item @uref{http://www.openssl.org/, OpenSSL} (binary) 1.0 or higher -@item TeX Live 2012 or higher, optional (for gnunet-bcd) -@item libpulse 2.0 or higher, optional (for gnunet-conversation) -@item libopus 1.0.1 or higher, optional (for gnunet-conversation) -@item libogg 1.3.0 or higher, optional (for gnunet-conversation) -@item certool (binary) -optional for convenient installation of the GNS proxy -(available as part of Debian's libnss3-tools) -@item python-zbar 0.10 or higher, optional (for gnunet-qr) -@item libsqlite 3.8.0 or higher (note that the code will compile and often work with lower -version numbers, but you may get subtle bugs with respect to quota management -in certain rare cases); alternatively, MySQL or Postgres can also be installed, -but those databases will require more complex configurations (not recommended -for first-time users) -@item zlib any version we tested worked -@item Gtk+ 3.0 or higher, optional (for gnunet-gtk) -@item libgladeui must match Gtk+ version, optional (for gnunet-gtk) -@item libqrencode 3.0 or higher, optional (for gnunet-namestore-gtk) -@end table - - -@node Fixing libgnurl build issues -@subsection Fixing libgnurl build issues - -If you have to compile libgnurl from source since the version included in your -distribution is to old you perhaps get an error message while running the -@file{configure} script: - -@code{@ - $ configure@ - ...@ - checking for 64-bit curl_off_t data type... unknown@ - checking for 32-bit curl_off_t data type... unknown@ - checking for 16-bit curl_off_t data type... unknown@ - configure: error: cannot find data type for curl_off_t.@ -} - -Solution: - -Before running the configure script, set: - -@code{CFLAGS="-I. -I$BUILD_ROOT/include" } - - - -@node Internal dependencies -@subsection Internal dependencies - -This section tries to give an overview of what processes a typical GNUnet peer -running a particular application would consist of. All of the processes listed -here should be automatically started by @code{gnunet-arm -s}. The list is given -as a rough first guide to users for failure diagnostics. Ideally, end-users -should never have to worry about these internal dependencies. - -In terms of internal dependencies, a minimum file-sharing system consists of -the following GNUnet processes (in order of dependency): - -@itemize @bullet -@item gnunet-service-arm -@item gnunet-service-resolver (required by all) -@item gnunet-service-statistics (required by all) -@item gnunet-service-peerinfo -@item gnunet-service-transport (requires peerinfo) -@item gnunet-service-core (requires transport) -@item gnunet-daemon-hostlist (requires core) -@item gnunet-daemon-topology (requires hostlist, peerinfo) -@item gnunet-service-datastore -@item gnunet-service-dht (requires core) -@item gnunet-service-identity -@item gnunet-service-fs (requires identity, mesh, dht, datastore, core) -@end itemize - - -A minimum VPN system consists of the following GNUnet processes (in order of -dependency): - -@itemize @bullet -@item gnunet-service-arm -@item gnunet-service-resolver (required by all) -@item gnunet-service-statistics (required by all) -@item gnunet-service-peerinfo -@item gnunet-service-transport (requires peerinfo) -@item gnunet-service-core (requires transport) -@item gnunet-daemon-hostlist (requires core) -@item gnunet-service-dht (requires core) -@item gnunet-service-mesh (requires dht, core) -@item gnunet-service-dns (requires dht) -@item gnunet-service-regex (requires dht) -@item gnunet-service-vpn (requires regex, dns, mesh, dht) -@end itemize - - -A minimum GNS system consists of the following GNUnet processes (in order of -dependency): -@itemize @bullet -@item gnunet-service-arm -@item gnunet-service-resolver (required by all) -@item gnunet-service-statistics (required by all) -@item gnunet-service-peerinfo -@item gnunet-service-transport (requires peerinfo) -@item gnunet-service-core (requires transport) -@item gnunet-daemon-hostlist (requires core) -@item gnunet-service-dht (requires core) -@item gnunet-service-mesh (requires dht, core) -@item gnunet-service-dns (requires dht) -@item gnunet-service-regex (requires dht) -@item gnunet-service-vpn (requires regex, dns, mesh, dht) -@item gnunet-service-identity -@item gnunet-service-namestore (requires identity) -@item gnunet-service-gns (requires vpn, dns, dht, namestore, identity) -@end itemize - -@node Pre-installation notes -@section Pre-installation notes - -Please note that in the code instructions for the installation, -@emph{#} indicates commands run as privileged root user and -@emph{$} shows commands run as unprivileged ("normal") system user. - - -@node Generic installation instructions -@section Generic installation instructions - -First, in addition to the GNUnet sources you must download the latest version -of various dependencies. Most distributions do not include sufficiently recent -versions of these dependencies. Thus, a typically installation on a "modern" -GNU/Linux distribution requires you to install the following -dependencies (ideally in this order): - -@itemize @bullet -@item libgpgerror and libgcrypt -@item libnettle and libunbound (possibly from distribution), GnuTLS -@item libgnurl (read the README) -@item GNU libmicrohttpd -@item GNU libextractor (make sure to first install the various mandatory and optional -dependencies including development headers from your distribution) -@end itemize - -Other dependencies that you should strongly consider to install is a -database (MySQL, sqlite or Postgres). The following instructions will assume -that you installed at least sqlite. For most distributions you should be able -to find pre-build packages for the database. Again, make sure to install the -client libraries and the respective development headers (if they are -packaged separately) as well. - -You can find specific, detailed instructions for installing of the dependencies -(and possibly the rest of the GNUnet installation) in the platform-specific -descriptions, which are linked from the bottom of this page. Please consult -them now. If your distribution is not listed, please study the instructions for -Debian stable carefully as you try to install the dependencies for your own -distribution. Contributing additional instructions for further platforms is -always appreciated. - -Before proceeding further, please double-check the dependency list. Note that -in addition to satisfying the dependencies, you might have to make sure that -development headers for the various libraries are also installed. There maybe -files for other distributions, or you might be able to find equivalent packages -for your distribution. - -While it is possible to build and install GNUnet without having root access, -we will assume that you have full control over your system in these -instructions. First, you should create a system user @emph{gnunet} and an additional -group @emph{gnunetdns}. On Debian and Ubuntu GNU/Linux, type:@ -@code{@ - # adduser --system --home /var/lib/gnunet --group --disabled-password gnunet@ - # addgroup --system gnunetdns@ -}@ - On other Unixes, this should have the same effect:@ -@code{@ - # useradd --system --groups gnunet --home-dir /var/lib/gnunet@ - # addgroup --system gnunetdns@ -}@ - Now compile and install GNUnet using:@ -@code{@ - $ tar xvf gnunet-0.10.?.tar.gz@ - $ cd gnunet-0.10.?@ - $ ./configure --with-sudo=sudo --with-nssdir=/lib@ - $ make@ - $ sudo make install@ -}@ - -If you want to be able to enable DEBUG-level log messages, add -@code{--enable-logging=verbose} to the end of the @code{./configure} command. -DEBUG-level log messages are in English-only and should only be useful for -developers (or for filing really detailed bug reports). - -Finally, you probably want to compile @code{gnunet-gtk}, which includes gnunet-setup -(graphical tool for configuration) and @code{gnunet-fs-gtk} (graphical tool for -file-sharing):@ - -@code{@ - $ tar xvf gnunet-gtk-0.10.?.tar.gz@ - $ cd gnunet-gtk-0.10.?@ - $ ./configure --with-gnunet=/usr/local/@ - $ make@ - $ sudo make install@ - $ cd ..@ - $ sudo ldconfig # just to be safe@ -}@ - Next, edit the file @file{/etc/gnunet.conf} to contain the following:@ -@code{@ - [arm]@ - SYSTEM_ONLY = YES@ - USER_ONLY = NO@ -}@ -You may need to update your ld.so cache to include files installed in -@file{/usr/local/lib}:@ - -@code{@ - # ldconfig@ -}@ - -Then, switch from user root to user gnunet to start the peer:@ - -@code{@ - # su -s /bin/sh - gnunet@ - $ gnunet-arm -c /etc/gnunet.conf -s@ -}@ - -You may also want to add the last line in the gnunet users @file{crontab} -prefixed with @code{@@reboot} so that it is executed whenever the system is -booted:@ - -@code{@ - @@reboot /usr/local/bin/gnunet-arm -c /etc/gnunet.conf -s@ -}@ - -This will only start the system-wide GNUnet services. Type exit to get back -your root shell. Now, you need to configure the per-user part. For each -$USER on the system, run:@ - -@code{@ - # adduser $USER gnunet@ -}@ - -to allow them to access the system-wide GNUnet services. Then, each user should -create a configuration file @file{~/.config/gnunet.conf} with the lines:@ - -@code{@ - [arm]@ - SYSTEM_ONLY = NO@ - USER_ONLY = YES@ - DEFAULTSERVICES = gns@ -}@ - -and start the per-user services using@ - -@code{@ - $ gnunet-arm -c ~/.config/gnunet.conf -s@ -}@ - -Again, adding a @code{crontab} entry to autostart the peer is advised:@ -@code{@ -@@reboot /usr/local/bin/gnunet-arm -c $HOME/.config/gnunet.conf -s@ -}@ - -Note that some GNUnet services (such as SOCKS5 proxies) may need a system-wide -TCP port for each user. For those services, systems with more than one user may -require each user to specify a different port number in their personal -configuration file. - -Finally, the user should perform the basic initial setup for the GNU Name -System. This is done by running two commands:@ - -@example -$ gnunet-gns-import.sh@ -$ gnunet-gns-proxy-setup-ca@ -@end example - -The first generates the default zones, wheras the second setups the GNS -Certificate Authority with the user's browser. Now, to actiave GNS in the -normal DNS resolution process, you need to edit your @file{/etc/nsswitch.conf} -where you should find a line like this: -@example -hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4 -@end example - - -The exact details may differ a bit, which is fine. Add the text -@emph{"gns [NOTFOUND=return]"} after @emph{"files"}: -@example -hosts: files gns [NOTFOUND=return] mdns4_minimal [NOTFOUND=return] dns mdns4 -@end example - - -You might want to make sure that @file{/lib/libnss_gns.so.2} exists on your -system, it should have been created during the installation. - - - -@node Build instructions for Ubuntu 12.04 using Git -@section Build instructions for Ubuntu 12.04 using Git - - -@menu -* Install the required build tools:: -* Install libgcrypt 1.6 and libgpg-error:: -* Install gnutls with DANE support:: -* Install libgnurl:: -* Install libmicrohttpd from Git:: -* Install libextractor from Git:: -* Install GNUnet dependencies:: -* Build GNUnet:: -* Install the GNUnet-gtk user interface from Git:: -@end menu - -@node Install the required build tools -@subsection Install the required build tools - -First, make sure Git is installed on your system:@ - -$ sudo apt-get install git@ - -Install the essential buildtools:@ - -$ sudo apt-get install automake autopoint autoconf libtool - -@node Install libgcrypt 1.6 and libgpg-error -@subsection Install libgcrypt 1.6 and libgpg-error - -$ wget ftp://ftp.gnupg.org/gcrypt/libgpg-error/libgpg-error-1.12.tar.bz2@ -$ tar xf libgpg-error-1.12.tar.bz2@ -$ cd libgpg-error-1.12@ -$ ./configure@ -$ sudo make install@ -$ cd ..@ - -@node Install gnutls with DANE support -@subsection Install gnutls with DANE support - -@example -$ wget http://www.lysator.liu.se/~nisse/archive/nettle-2.7.1.tar.gz@ -$ tar xf nettle-2.7.1.tar.gz@ -$ cd nettle-2.7.1@ -$ ./configure@ -$ sudo make install@ -$ cd .. -@end example - -@example -$ wget https://www.nlnetlabs.nl/downloads/ldns/ldns-1.6.16.tar.gz@ -$ tar xf ldns-1.6.16.tar.gz@ -$ cd ldns-1.6.16@ -$ ./configure@ -$ sudo make install@ -$ cd .. -@end example - -@example -$ wget https://unbound.net/downloads/unbound-1.4.21.tar.gz@ -$ tar xf unbound-1.4.21.tar.gz@ -$ cd unbound-1.4.21@ -$ ./configure@ -$ sudo make install@ -$ cd .. -@end example - -@example -$ wget ftp://ftp.gnutls.org/gcrypt/gnutls/v3.1/gnutls-3.1.17.tar.xz@ -$ tar xf gnutls-3.1.17.tar.xz@ -$ cd gnutls-3.1.17@ -$ ./configure@ -$ sudo make install@ -$ cd .. -@end example - -@example -$ wget ftp://ftp.gnupg.org/gcrypt/libgcrypt/libgcrypt-1.6.0.tar.bz2@ -$ tar xf libgcrypt-1.6.0.tar.bz2@ -$ cd libgcrypt-1.6.0@ -$ ./configure@ -$ sudo make install@ -$ cd ..@ -@end example - -@node Install libgnurl -@subsection Install libgnurl - -@example -$ wget https://gnunet.org/sites/default/files/gnurl-7.34.0.tar.bz2@ -$ tar xf gnurl-7.34.0.tar.bz2@ -$ cd gnurl-7.34.0@ -$ ./configure --enable-ipv6 --with-gnutls --without-libssh2 \ - --without-libmetalink --without-winidn --without-librtmp \ - --without-nghttp2 --without-nss --without-cyassl \ - --without-polarssl --without-ssl --without-winssl \ - --without-darwinssl --disable-sspi --disable-ntlm-wb \ - --disable-ldap --disable-rtsp --disable-dict --disable-telnet \ - --disable-tftp --disable-pop3 --disable-imap --disable-smtp \ - --disable-gopher --disable-file --disable-ftp@ -$ sudo make install@ -$ cd ..@ -@end example - -@node Install libmicrohttpd from Git -@subsection Install libmicrohttpd from Git - -@example -$ git clone https://gnunet.org/git/libmicrohttpd@ -$ cd libmicrohttpd/@ -$ ./bootstrap@ -$ ./configure@ -$ sudo make install@ -$ cd ..@ -@end example - -@node Install libextractor from Git -@subsection Install libextractor from Git - -Install libextractor dependencies:@ - -@example -$ sudo apt-get install zlib1g-dev libgsf-1-dev libmpeg2-4-dev libpoppler-dev \ - libvorbis-dev libexiv2-dev libjpeg-dev libtiff-dev libgif-dev libvorbis-dev \ - libflac-dev libsmf-dev g++@ -@end example - -Build libextractor:@ - -@example -$ git clone https://gnunet.org/git/libextractor@ -$ cd libextractor@ -$ ./bootstrap@ -$ ./configure@ -$ sudo make install@ -$ cd ..@ -@end example - -@node Install GNUnet dependencies -@subsection Install GNUnet dependencies - -@example -$ sudo apt-get install libidn11-dev libunistring-dev libglpk-dev \ - libpulse-dev libbluetooth-dev libsqlite-dev@ -@end example - -Install libopus@ - -@example -$ wget http://downloads.xiph.org/releases/opus/opus-1.1.tar.gz@ -$ tar xf opus-1.1.tar.gz@ -$ cd opus-1.1/@ -$ ./configure@ -$ sudo make install@ -@end example - -Choose one or more database backends@ - -@itemize @bullet - -@item -SQLite3 @code{$ sudo apt-get install libsqlite3-dev} - -@item -MySQL @code{$ sudo apt-get install libmysqlclient-dev} - -@item -PostgreSQL @code{$ sudo apt-get install libpq-dev postgresql} - -@end itemize - - - -@node Build GNUnet -@subsection Build GNUnet - - - -@menu -* Configuring the installation path:: -* Configuring the system:: -* Installing components requiring sudo permission:: -* Build:: -@end menu - -@node Configuring the installation path -@subsubsection Configuring the installation path - -You can specify the location of the GNUnet installation by setting the prefix -when calling the configure script:@code{ --prefix=DIRECTORY} - -@code{@ - $ export PATH=$PATH:DIRECTORY/bin@ -} - -@node Configuring the system -@subsubsection Configuring the system - -Please make sure NOW that you have created a user and group 'gnunet'@ -and additionally a group 'gnunetdns':@ -@code{@ - $ sudo addgroup gnunet@ - $ sudo addgroup gnunetdns@ - $ sudo adduser gnunet@ -} - -Each GNUnet user should be added to the 'gnunet' group (may@ -require fresh login to come into effect): -@code{@ - $ sudo useradd -G gnunet@ -} - -@node Installing components requiring sudo permission -@subsubsection Installing components requiring sudo permission - -Some components, like the nss plugin required for GNS, may require root -permissions. To allow these few components to be installed use:@ -@code{@ - $ ./configure --with-sudo} - -@node Build -@subsubsection Build - - -@code{@ - $ git clone https://gnunet.org/git/gnunet/@ - $ cd gnunet/@ - $ ./bootstrap@ -} -Use the required configure call including the optional installation prefix -PREFIX or the sudo permissions@ -@code{$ ./configure [ --with-sudo | --with-prefix=PREFIX ]}@ -@code{$ make; sudo make install} - -After installing it, you need to create an empty configuration file:@ -@code{mkdir ~/.gnunet; touch ~/.gnunet/gnunet.conf} - -And finally you can start GNUnet with@ -@code{$ gnunet-arm -s} - -@node Install the GNUnet-gtk user interface from Git -@subsection Install the GNUnet-gtk user interface from Git - - -Install depencies:@ -@code{$ sudo apt-get install libgtk-3-dev libunique-3.0-dev libgladeui-dev libqrencode-dev} - -To build GNUnet (with an optional prefix)and execute:@ -@code{@ - $ git clone https://gnunet.org/git/gnunet-gtk/@ - $ cd gnunet-gtk/@ - $ ./bootstrap@ - $ ./configure [--prefix=PREFIX] --with-gnunet=DIRECTORY@ - $ make; sudo make install@ -} - -@node Build Instructions for Microsoft Windows Platforms -@section Build Instructions for Microsoft Windows Platforms - - - -@menu -* Introduction to building on MS Windows:: -* Requirements:: -* Dependencies & Initial Setup:: -* GNUnet Installation:: -* Adjusting Windows for running and testing GNUnet:: -* Building the GNUnet Installer:: -* Using GNUnet with Netbeans on Windows:: -@end menu - -@node Introduction to building on MS Windows -@subsection Introduction to building on MS Windows - - -This document is a guide to building GNUnet and its dependencies on Windows -platforms. GNUnet development is mostly done under Linux and especially SVN -checkouts may not build out of the box. We regret any inconvenience, and if you -have problems, please report them. - -@node Requirements -@subsection Requirements - -The Howto is based upon a @strong{Windows Server 2008 32bit@strong{ -Installation, @strong{sbuild} and thus a @uref{http://www.mingw.org/wiki/MSYS, -MSYS+MinGW} (W32-GCC-Compiler-Suite + Unix-like Userland) installation. sbuild -is a convenient set of scripts which creates a working msys/mingw installation -and installs most dependencies required for GNUnet. }} - -As of the point of the creation of this Howto, GNUnet @strong{requires} a -Windows @strong{Server} 2003 or newer for full feature support. Windows Vista -and later will also work, but -@strong{non-server version can not run a VPN-Exit-Node} as the NAT features -have been removed as of Windows Vista. - -@node Dependencies & Initial Setup -@subsection Dependencies & Initial Setup - - -@itemize @bullet - -@item -Install a fresh version of @strong{Python 2.x}, even if you are using a x64-OS, -install a 32-bit version for use with sbuild. Python 3.0 currently is -incompatible. - -@item -Install your favorite @uref{http://code.google.com/p/tortoisegit/, GIT} & -@uref{http://tortoisesvn.net/, SVN}-clients. - -@item -You will also need some archive-manager like @uref{http://www.7-zip.org/, 7zip}. - -@item -Pull a copy of sbuild to a directory of your choice, which will be used in the -remainder of this guide. For now, we will use @file{c:\gnunet\sbuild\} - -@item -in @file{sbuild\src\mingw\mingw32-buildall.sh}, comment out the packages -@strong{gnunet-svn} and @strong{gnunet-gtk-svn}, as we don't want sbuild to -compile/install those for us. - -@item -Follow LRN's sbuild installation instructions.- -@end itemize - -Please note that sbuild may (or will most likely) fail during installation, -thus you really HAVE to @strong{check the logfiles} created during the -installation process. Certain packages may fail to build initially due to -missing dependencies, thus you may have to -@strong{substitute those with binary-versions initially}. Later on once -dependencies are satisfied you can re-build the newer package versions. - -@strong{It is normal that you may have to repeat this step multiple times and -there is no uniform way to fix all compile-time issues, as the build-process -of many of the dependencies installed are rather unstable on win32 and certain -releases may not even compile at all.} - -Most dependencies for GNUnet have been set up by sbuild, thus we now should add -the @file{bin/} directories in your new msys and mingw installations to PATH. -You will want to create a backup of your finished msys-environment by now. - -@node GNUnet Installation -@subsection GNUnet Installation - -First, we need to launch our msys-shell, you can do this via - -@file{C:\gnunet\sbuild\msys\msys.bat} - -You might wish to take a look at this file and adjust some login-parameters to -your msys environment. - -Also, sbuild added two pointpoints to your msys-environment, though those -might remain invisible: - -@itemize @bullet - -@item -/mingw, which will mount your mingw-directory from sbuild/mingw and the other one is - -@item -/src which contains all the installation sources sbuild just compiled. -@end itemize - -Check out the current gnunet-sources (svn-head) from the gnunet-repository, -we will do this in your home directory: - -@code{svn checkout https://gnunet.org/svn/gnunet/ ~/gnunet} - -Now, we will first need to bootstrap the checked out installation and then -configure it accordingly. - -@example -cd ~/gnunet@ -./bootstrap@ -STRIP=true CPPFLAGS="-DUSE_IPV6=1 -DW32_VEH" CFLAGS="$CFLAGS -g -O2" ./configure --prefix=/ --docdir=/share/doc/gnunet --with-libiconv-prefix=/mingw --with-libintl-prefix=/mingw --with-libcurl=/mingw --with-extractor=/mingw --with-sqlite=/mingw --with-microhttpd=/mingw --with-plibc=/mingw --enable-benchmarks --enable-expensivetests --enable-experimental --with-qrencode=/mingw --enable-silent-rules --enable-experimental 2>&1 | tee -a ./configure.log -@end example - -The parameters above will configure for a reasonable gnunet installation to the -your msys-root directory. Depending on which features your would like to build -or you may need to specify additional dependencies. Sbuild installed most libs -into the /mingw subdirectory, so remember to prefix library locations with -this path. - -Like on a unixoid system, you might want to use your home directory as prefix -for your own gnunet installation for development, without tainting the -buildenvironment. Just change the "prefix" parameter to point towards -~/ in this case. - -Now it's time to compile gnunet as usual. Though this will take some time, so -you may fetch yourself a coffee or some Mate now... - -@example -make@ -make install -@end example - -@node Adjusting Windows for running and testing GNUnet -@subsection Adjusting Windows for running and testing GNUnet - -Assuming the build succeeded and you -@strong{added the bin directory of your gnunet to PATH}, you can now use your -gnunet-installation as usual. Remember that UAC or the windows firewall may -popup initially, blocking further execution of gnunet until you acknowledge -them (duh!). - -You will also have to take the usual steps to get p2p software running properly -(port forwarding, ...), and gnunet will require administrative permissions as -it may even install a device-driver (in case you are using gnunet-vpn and/or -gnunet-exit). - -@node Building the GNUnet Installer -@subsection Building the GNUnet Installer - -The GNUnet installer is made with @uref{http://nsis.sourceforge.net/, NSIS}@ -The installer script is located in @file{contrib\win} in the GNUnet source tree. - -@node Using GNUnet with Netbeans on Windows -@subsection Using GNUnet with Netbeans on Windows - -TODO - -@node Build instructions for Debian 7.5 -@section Build instructions for Debian 7.5 - - -These are the installation instructions for Debian 7.5. They were tested using -a minimal, fresh Debian 7.5 AMD64 installation without non-free software -(no contrib or non-free). By "minimal", we mean that during installation, we -did not select any desktop environment, servers or system utilities during the -"tasksel" step. Note that the packages and the dependencies that we will -install during this chapter take about 1.5 GB of disk space. Combined with -GNUnet and space for objects during compilation, you should not even attempt -this unless you have about 2.5 GB free after the minimal Debian installation. -Using these instructions to build a VM image is likely to require a minimum of -4-5 GB for the VM (as you will likely also want a desktop manager). - -GNUnet's security model assumes that your @file{/home} directory is encrypted. -Thus, if possible, you should encrypt your home partition -(or per-user home directory). - -Naturally, the exact details of the starting state for your installation -should not matter much. For example, if you selected any of those installation -groups you might simply already have some of the necessary packages installed. -We did this for testing, as this way we are less likely to forget to mention a -required package. Note that we will not install a desktop environment, but of -course you will need to install one to use GNUnet's graphical user interfaces. -Thus, it is suggested that you simply install the desktop environment of your -choice before beginning with the instructions. - - - -@menu -* Update:: -* Stable? Hah!:: -* Update again:: -* Installing packages:: -* Installing dependencies from source:: -* Installing GNUnet from source:: -* But wait there is more!:: -@end menu - -@node Update -@subsection Update - -After any installation, you should begin by running - -@example -# apt-get update@ -# apt-get upgrade@ -@end example - -to ensure that all of your packages are up-to-date. Note that the "#" is used -to indicate that you need to type in this command as "root" -(or prefix with "sudo"), whereas "$" is used to indicate typing in a command -as a normal user. - -@node Stable? Hah! -@subsection Stable? Hah! - -Yes, we said we start with a Debian 7.5 "stable" system. However, to reduce the -amount of compilation by hand, we will begin by allowing the installation of -packages from the testing and unstable distributions as well. We will stick to -"stable" packages where possible, but some packages will be taken from the -other distributions. Start by modifying @file{/etc/apt/sources.list} to contain -the following (possibly adjusted to point to your mirror of choice): -@example -# These were there before: -deb http://ftp.de.debian.org/debian/ wheezy main -deb-src http://ftp.de.debian.org/debian/ wheezy main -deb http://security.debian.org/ wheezy/updates main -deb-src http://security.debian.org/ wheezy/updates main -deb http://ftp.de.debian.org/debian/ wheezy-updates main -deb-src http://ftp.de.debian.org/debian/ wheezy-updates main - -# Add these lines (feel free to adjust the mirror): -deb http://ftp.de.debian.org/debian/ testing main -deb http://ftp.de.debian.org/debian/ unstable main -@end example - -The next step is to create/edit your @file{/etc/apt/preferences} file to look -like this: - -@example -Package: * -Pin: release a=stable,n=wheezy -Pin-Priority: 700 - -Package: * -Pin: release o=Debian,a=testing -Pin-Priority: 650 - -Package: * -Pin: release o=Debian,a=unstable -Pin-Priority: 600 -@end example - -You can read more about Apt Preferences here and here. Note that other pinnings -are likely to also work for GNUnet, the key thing is that you need some -packages from unstable (as shown below). However, as unstable is unlikely to -be comprehensive (missing packages) or might be problematic (crashing packages), -you probably want others from stable and/or testing. - -@node Update again -@subsection Update again - -Now, run again@ - -@example -# apt-get update@ -# apt-get upgrade@ -@end example - -to ensure that all your new distribution indices are downloaded, and that your -pinning is correct: the upgrade step should cause no changes at all. - -@node Installing packages -@subsection Installing packages - -We begin by installing a few Debian packages from stable:@ - -@example -# apt-get install gcc make python-zbar libltdl-dev libsqlite3-dev \ - libunistring-dev libopus-dev libpulse-dev openssl libglpk-dev \ - texlive libidn11-dev libmysqlclient-dev libpq-dev libarchive-dev \ - libbz2-dev libexiv2-dev libflac-dev libgif-dev libglib2.0-dev \ - libgtk-3-dev libmagic-dev libjpeg8-dev libmpeg2-4-dev libmp4v2-dev \ - librpm-dev libsmf-dev libtidy-dev libtiff5-dev libvorbis-dev \ - libogg-dev zlib1g-dev g++ gettext libgsf-1-dev libunbound-dev \ - libqrencode-dev libgladeui-dev nasm texlive-latex-extra \ - libunique-3.0-dev gawk miniupnpc libfuse-dev libbluetooth-dev -@end example - -After that, we install a few more packages from unstable:@ - -@example -# apt-get install -t unstable nettle-dev libgstreamer1.0-dev \ - gstreamer1.0-plugins-base gstreamer1.0-plugins-good \ - libgstreamer-plugins-base1.0-dev -@end example - -@node Installing dependencies from source -@subsection Installing dependencies from source - -Next, we need to install a few dependencies from source. You might want to do -this as a "normal" user and only run the @code{make install} steps as root -(hence the @code{sudo} in the commands below). Also, you do this from any -directory. We begin by downloading all dependencies, then extracting the -sources, and finally compiling and installing the libraries:@ - -@example - $ wget https://libav.org/releases/libav-9.10.tar.xz@ - $ wget http://ftp.gnu.org/gnu/libextractor/libextractor-1.3.tar.gz@ - $ wget ftp://ftp.gnupg.org/gcrypt/libgpg-error/libgpg-error-1.12.tar.bz2@ - $ wget ftp://ftp.gnupg.org/gcrypt/libgcrypt/libgcrypt-1.6.0.tar.bz2@ - $ wget ftp://ftp.gnutls.org/gcrypt/gnutls/v3.2/gnutls-3.2.7.tar.xz@ - $ wget http://ftp.gnu.org/gnu/libmicrohttpd/libmicrohttpd-0.9.33.tar.gz@ - $ wget https://gnunet.org/sites/default/files/gnurl-7.34.0.tar.bz2@ - $ tar xvf libextractor-1.3.tar.gz@ - $ tar xvf libgpg-error-1.12.tar.bz2@ - $ tar xvf libgcrypt-1.6.0.tar.bz2@ - $ tar xvf gnutls-3.2.7.tar.xz@ - $ tar xvf libmicrohttpd-0.9.33.tar.gz@ - $ tar xvf gnurl-7.34.0.tar.bz2@ - $ cd libav-0.9 ; ./configure --enable-shared; make; sudo make install ; cd ..@ - $ cd libextractor-1.3 ; ./configure; make ; sudo make install; cd ..@ - $ cd libgpg-error-1.12; ./configure ; make ; sudo make install ; cd ..@ - $ cd libgcrypt-1.6.0; ./configure --with-gpg-error-prefix=/usr/local; make ; sudo make install ; cd ..@ - $ cd gnutls-3.2.7 ; ./configure ; make ; sudo make install ; cd ..@ - $ cd libmicrohttpd-0.9.33; ./configure ; make ; sudo make install ; cd ..@ - $ cd gnurl-7.34.0@ - $ ./configure --enable-ipv6 --with-gnutls=/usr/local --without-libssh2 \ - --without-libmetalink --without-winidn --without-librtmp --without-nghttp2 \ - --without-nss --without-cyassl --without-polarssl --without-ssl \ - --without-winssl --without-darwinssl --disable-sspi --disable-ntlm-wb \ - --disable-ldap --disable-rtsp --disable-dict --disable-telnet --disable-tftp \ - --disable-pop3 --disable-imap --disable-smtp --disable-gopher --disable-file \ - --disable-ftp@ - $ make ; sudo make install; cd ..@ -@end example - -@node Installing GNUnet from source -@subsection Installing GNUnet from source - - -For this, simply follow the generic installation instructions from -here. - -@node But wait there is more! -@subsection But wait there is more! - -So far, we installed all of the packages and dependencies required to ensure -that all of GNUnet would be built. However, while for example the plugins to -interact with the MySQL or Postgres databases have been created, we did not -actually install or configure those databases. Thus, you will need to install -and configure those databases or stick with the default Sqlite database. -Sqlite is usually fine for most applications, but MySQL can offer better -performance and Postgres better resillience. - - -@node Installing GNUnet from Git on Ubuntu 14.4 -@section Installing GNUnet from Git on Ubuntu 14.4 - -@strong{Install the required build tools:} -@code{@ - $ sudo apt-get install git automake autopoint autoconf@ -} - -@strong{Install the required dependencies} -@example -$ sudo apt-get install libltdl-dev libgpg-error-dev libidn11-dev \ - libunistring-dev libglpk-dev libbluetooth-dev libextractor-dev \ - libmicrohttpd-dev libgnutls28-dev -@end example - -@strong{Choose one or more database backends}@ - SQLite3@ -@code{@ - $ sudo apt-get install libsqlite3-dev@ -}@ - MySQL@ -@code{@ - $ sudo apt-get install libmysqlclient-dev@ -}@ - PostgreSQL@ -@code{@ - $ sudo apt-get install libpq-dev postgresql@ -} - -@strong{Install the optional dependencies for gnunet-conversation:}@ -@code{@ - $ sudo apt-get install gstreamer1.0 libpulse-dev libopus-dev@ -} - -@strong{Install the libgrypt 1.6.1:}@ - For Ubuntu 14.04:@ -@code{$ sudo apt-get install libgcrypt20-dev}@ - For Ubuntu older 14.04:@ -@code{$ wget ftp://ftp.gnupg.org/gcrypt/libgcrypt/libgcrypt-1.6.1.tar.bz2@ - $ tar xf libgcrypt-1.6.1.tar.bz2@ - $ cd libgcrypt-1.6.1@ - $ ./configure@ - $ sudo make install@ - $ cd ..}@ -@strong{Install libgnurl}@ -@example - $ wget https://gnunet.org/sites/default/files/gnurl-7.35.0.tar.bz2@ - $ tar xf gnurl-7.35.0.tar.bz2@ - $ cd gnurl-7.35.0@ - $ ./configure --enable-ipv6 --with-gnutls --without-libssh2 \ - --without-libmetalink --without-winidn --without-librtmp --without-nghttp2 \ - --without-nss --without-cyassl --without-polarssl --without-ssl \ - --without-winssl --without-darwinssl --disable-sspi --disable-ntlm-wb \ - --disable-ldap --disable-rtsp --disable-dict --disable-telnet --disable-tftp \ - --disable-pop3 --disable-imap --disable-smtp --disable-gopher --disable-file \ - --disable-ftp - $ sudo make install@ - $ cd ..@ -@end example - -@strong{Install GNUnet}@ -@code{@ - $ git clone https://gnunet.org/git/gnunet/@ - $ cd gnunet/@ - $ ./bootstrap@ -} - -If you want to: -@itemize @bullet - - -@item -Install to a different directory:@ - --prefix=PREFIX - -@item -Have sudo permission, but do not want to compile as root:@ - --with-sudo - -@item -Want debug message enabled:@ - -- enable-logging=verbose -@end itemize - - -@code{@ - $ ./configure [ --with-sudo | --prefix=PREFIX | --- enable-logging=verbose]@ - $ make; sudo make install@ -} - -After installing it, you need to create an empty configuration file:@ -@code{touch ~/.config/gnunet.conf} - -And finally you can start GNUnet with@ -@code{$ gnunet-arm -s} - -@node Build instructions for Debian 8 -@section Build instructions for Debian 8 - -These are the installation instructions for Debian 8. They were tested using a -fresh Debian 8 AMD64 installation without non-free software (no contrib or -non-free). During installation, I only selected "lxde" for the desktop -environment. Note that the packages and the dependencies that we will install -during this chapter take about 1.5 GB of disk space. Combined with GNUnet and -space for objects during compilation, you should not even attempt this unless -you have about 2.5 GB free after the Debian installation. Using these -instructions to build a VM image is likely to require a minimum of 4-5 GB for -the VM (as you will likely also want a desktop manager). - -GNUnet's security model assumes that your @code{/home} directory is encrypted. -Thus, if possible, you should encrypt your entire disk, or at least just your -home partition (or per-user home directory). - -Naturally, the exact details of the starting state for your installation should -not matter much. For example, if you selected any of those installation groups -you might simply already have some of the necessary packages installed. Thus, -it is suggested that you simply install the desktop environment of your choice -before beginning with the instructions. - - -@menu -* Update Debian:: -* Installing Debian Packages:: -* Installing Dependencies from Source2:: -* Installing GNUnet from Source2:: -* But wait (again) there is more!:: -@end menu - -@node Update Debian -@subsection Update Debian - -After any installation, you should begin by running@ -@code{@ - # apt-get update@ - # apt-get upgrade@ -}@ -to ensure that all of your packages are up-to-date. Note that the "#" is used -to indicate that you need to type in this command as "root" (or prefix with -"sudo"), whereas "$" is used to indicate typing in a command as a normal -user. - -@node Installing Debian Packages -@subsection Installing Debian Packages - -We begin by installing a few Debian packages from stable:@ -@example - # apt-get install gcc make python-zbar libltdl-dev libsqlite3-dev \ - libunistring-dev libopus-dev libpulse-dev openssl libglpk-dev texlive \ - libidn11-dev libmysqlclient-dev libpq-dev libarchive-dev libbz2-dev \ - libflac-dev libgif-dev libglib2.0-dev libgtk-3-dev libmpeg2-4-dev \ - libtidy-dev libvorbis-dev libogg-dev zlib1g-dev g++ gettext libgsf-1-dev \ - libunbound-dev libqrencode-dev libgladeui-dev nasm texlive-latex-extra \ - libunique-3.0-dev gawk miniupnpc libfuse-dev libbluetooth-dev \ - gstreamer1.0-plugins-base gstreamer1.0-plugins-good \ - libgstreamer-plugins-base1.0-dev nettle-dev libextractor-dev libgcrypt20-dev \ - libmicrohttpd-dev -@end example - -@node Installing Dependencies from Source2 -@subsection Installing Dependencies from Source2 - -Yes, we said we start with a Debian 8 "stable" system, but because Debian -linked GnuTLS without support for DANE, we need to compile a few things, in -addition to GNUnet, still by hand. Yes, you can run GNUnet using the respective -Debian packages, but then you will not get DANE support. - -Next, we need to install a few dependencies from source. You might want to do -this as a "normal" user and only run the @code{make install} steps as root -(hence the @code{sudo} in the commands below). Also, you do this from any -directory. We begin by downloading all dependencies, then extracting the -sources, and finally compiling and installing the libraries:@ - -@code{@ - $ wget ftp://ftp.gnutls.org/gcrypt/gnutls/v3.3/gnutls-3.3.12.tar.xz@ - $ wget https://gnunet.org/sites/default/files/gnurl-7.40.0.tar.bz2@ - $ tar xvf gnutls-3.3.12.tar.xz@ - $ tar xvf gnurl-7.40.0.tar.bz2@ - $ cd gnutls-3.3.12 ; ./configure ; make ; sudo make install ; cd ..@ - $ cd gnurl-7.40.0@ - $ ./configure --enable-ipv6 --with-gnutls=/usr/local --without-libssh2 \ - --without-libmetalink --without-winidn --without-librtmp --without-nghttp2 \ - --without-nss --without-cyassl --without-polarssl --without-ssl \ - --without-winssl --without-darwinssl --disable-sspi --disable-ntlm-wb \ - --disable-ldap --disable-rtsp --disable-dict --disable-telnet --disable-tftp \ - --disable-pop3 --disable-imap --disable-smtp --disable-gopher --disable-file \ - --disable-ftp --disable-smb - $ make ; sudo make install; cd ..@ -} - -@node Installing GNUnet from Source2 -@subsection Installing GNUnet from Source2 - -For this, simply follow the generic installation instructions from@ -here. - -@node But wait (again) there is more! -@subsection But wait (again) there is more! - -So far, we installed all of the packages and dependencies required to ensure -that all of GNUnet would be built. However, while for example the plugins to -interact with the MySQL or Postgres databases have been created, we did not -actually install or configure those databases. Thus, you will need to install -and configure those databases or stick with the default Sqlite database. Sqlite -is usually fine for most applications, but MySQL can offer better performance -and Postgres better resillience. - -@node Outdated build instructions for previous revisions -@section Outdated build instructions for previous revisions - -This chapter contains a collection of outdated, older installation guides. They -are mostly intended to serve as a starting point for writing up-to-date -instructions and should not be expected to work for GNUnet 0.10.x. -A set of older installation instructions can also be found in the -@file{doc/outdated-and-old-installation-instructions.txt} in the source -of GNUnet. This file covers old instructions which no longer receive -security updates or any kind of support. - - -@menu -* Installing GNUnet 0.10.1 on Ubuntu 14.04:: -* Building GLPK for MinGW:: -* GUI build instructions for Ubuntu 12.04 using Subversion:: -* Installation with gnunet-update:: -* Instructions for Microsoft Windows Platforms (Old):: -@end menu - - -@node Installing GNUnet 0.10.1 on Ubuntu 14.04 -@subsection Installing GNUnet 0.10.1 on Ubuntu 14.04 - -Install the required dependencies@ - -@example -$ sudo apt-get install libltdl-dev libgpg-error-dev libidn11-dev \ - libunistring-dev libglpk-dev libbluetooth-dev libextractor-dev \ - libmicrohttpd-dev libgnutls28-dev -@end example - -Choose one or more database backends@ -SQLite3@ -@code{@ - $ sudo apt-get install libsqlite3-dev@ -}@ -MySQL@ -@code{@ - $ sudo apt-get install libmysqlclient-dev@ -}@ -PostgreSQL@ -@code{@ - $ sudo apt-get install libpq-dev postgresql@ -} - -Install the optional dependencies for gnunet-conversation:@ -@code{@ - $ sudo apt-get install gstreamer1.0 libpulse-dev libopus-dev@ -} - -Install the libgrypt 1.6:@ -For Ubuntu 14.04:@ -@code{$ sudo apt-get install libgcrypt20-dev}@ -For Ubuntu older 14.04:@ -@code{$ wget ftp://ftp.gnupg.org/gcrypt/libgcrypt/libgcrypt-1.6.1.tar.bz2@ - $ tar xf libgcrypt-1.6.1.tar.bz2@ - $ cd libgcrypt-1.6.1@ - $ ./configure@ - $ sudo make install@ - $ cd ..} - -Install libgnurl@ -@example - $ wget https://gnunet.org/sites/default/files/gnurl-7.35.0.tar.bz2@ - $ tar xf gnurl-7.35.0.tar.bz2@ - $ cd gnurl-7.35.0@ - $ ./configure --enable-ipv6 --with-gnutls --without-libssh2 \ - --without-libmetalink --without-winidn --without-librtmp --without-nghttp2 \ - --without-nss --without-cyassl --without-polarssl --without-ssl \ - --without-winssl --without-darwinssl --disable-sspi --disable-ntlm-wb \ - --disable-ldap --disable-rtsp --disable-dict --disable-telnet --disable-tftp \ - --disable-pop3 --disable-imap --disable-smtp --disable-gopher --disable-file \ - --disable-ftp@ - $ sudo make install@ - $ cd ..@ -@end example - -Install GNUnet@ -@code{@ - $ wget http://ftpmirror.gnu.org/gnunet/gnunet-0.10.1.tar.gz@ - $ tar xf gnunet-0.10.1.tar.gz@ - $ cd gnunet-0.10.1@ -} - -If you want to: -@itemize @bullet - -@item -Install to a different directory:@ - --prefix=PREFIX - -@item -Have sudo permission, but do not want to compile as root:@ - --with-sudo - -@item -Want debug message enabled:@ - -- enable-logging=verbose -@end itemize - -@code{@ - $ ./configure [ --with-sudo | --prefix=PREFIX | --enable-logging=verbose]@ - $ make; sudo make install@ -} - -After installing it, you need to create an empty configuration file:@ -@code{touch ~/.config/gnunet.conf} - -And finally you can start GNUnet with@ -@code{$ gnunet-arm -s} - -@node Building GLPK for MinGW -@subsection Building GLPK for MinGW - -GNUnet now requires the GNU Linear Programming Kit (GLPK). Since there's is no -package you can install with @code{mingw-get} you have to compile it from -source: - -@itemize @bullet - -@item -Download the latest version from http://ftp.gnu.org/gnu/glpk/ - -@item -Unzip it using your favourite unzipper@ -In the MSYS shell: - -@item -change to the respective directory - -@item -@code{./configure '--build=i686-pc-mingw32'} - -@item -run @code{make install check } - -MinGW does not automatically detect the correct buildtype so you have to -specify it manually -@end itemize - - -@node GUI build instructions for Ubuntu 12.04 using Subversion -@subsection GUI build instructions for Ubuntu 12.04 using Subversion - -After installing GNUnet you can continue installing the GNUnet GUI tools: - -First, install the required dependencies: - -@code{@ - $ sudo apt-get install libgladeui-dev libqrencode-dev@ -} - -Please ensure that the GNUnet shared libraries can be found by the linker. If -you installed GNUnet libraries in a non standard path (say -GNUNET_PREFIX=/usr/local/lib/), you can -@itemize @bullet - - -@item -set the environmental variable permanently to@ -@code{LD_LIBRARY_PATH=$GNUNET_PREFIX} - -@item -or add @code{$GNUNET_PREFIX} to @code{/etc/ld.so.conf} -@end itemize - - -Now you can checkout and compile the GNUnet GUI tools@ -@code{@ - $ svn co https://gnunet.org/svn/gnunet-gtk@ - $ cd gnunet-gtk@ - $ ./bootstrap@ - $ ./configure --prefix=$GNUNET_PREFIX/.. --with-gnunet=$GNUNET_PREFIX/..@ - $ make install@ -} - -@node Installation with gnunet-update -@subsection Installation with gnunet-update - -gnunet-update project is an effort to introduce updates to GNUnet -installations. An interesting to-be-implemented-feature of gnunet-update is -that these updates are propagated through GNUnet's peer-to-peer network. More -information about gnunet-update can be found at -https://gnunet.org/svn/gnunet-update/README. - -While the project is still under development, we have implemented the following -features which we believe may be helpful for users and we would like them to be -tested: - -@itemize @bullet - -@item -Packaging GNUnet installation along with its run-time dependencies into update -packages - -@item -Installing update packages into compatible hosts - -@item -Updating an existing installation (which had been installed by gnunet-update) -to a newer one -@end itemize - -The above said features of gnunet-update are currently available for testing on -GNU/Linux systems. - -The following is a guide to help you get started with gnunet-update. It shows -you how to install the testing binary packages of GNUnet 0.9.1 we have at -https://gnunet.org/install/ - -gnunet-update needs the following: - -@itemize @bullet -@item -python ( 2.6 or above) - -@item -gnupg - -@item -python-gpgme -@end itemize - - -Checkout gnunet-update:@ -@code{@ - $ svn checkout -r24905 https://gnunet.org/svn/gnunet-update@ -} - -For security reasons, all packages released for gnunet-update from us are -signed with the key at https://gnunet.org/install/key.txt You would need to -import this key into your gpg key ring. gnunet-update uses this key to verify -the integrity of the packages it installs@ -@code{@ - $ gpg --recv-keys 7C613D78@ -} - -Download the packages relevant to your architecture (currently I have access to -GNU/Linux machines on x86_64 and i686, so only two for now, hopefully more -later) from https://gnunet.org/install/. - -To install the downloaded package into the directory /foo: - -@code{@ - gnunet-update/bin/gnunet-update install downloaded/package /foo@ -} - -The installer reports the directories into which shared libraries and -dependencies have been installed. You may need to add the reported shared -library installation paths to LD_LIBRARY_PATH before you start running any -installed binaries. - -Please report bugs at https://gnunet.org/bugs/ under the project -'gnunet-update'. - -@node Instructions for Microsoft Windows Platforms (Old) -@subsection Instructions for Microsoft Windows Platforms (Old) - -This document is a DEPRECATED installation guide for gnunet on windows. It will -not work for recent gnunet versions, but maybe it will be of some use if -problems arise. - - The Windows build uses a UNIX emulator for Windows, - @uref{http://www.mingw.org/, MinGW}, to build the executable modules. These - modules run natively on Windows and do not require additional emulation - software besides the usual dependencies. - - GNUnet development is mostly done under Linux and especially SVN checkouts may - not build out of the box. We regret any inconvenience, and if you have - problems, please report them. - - - -@menu -* Hardware and OS requirements:: -* Software installation:: -* Building libextractor and GNUnet:: -* Installer:: -* Source:: -@end menu - -@node Hardware and OS requirements -@subsubsection Hardware and OS requirements - -@itemize @bullet - -@item -Pentium II or equivalent processor, 350 MHz or better - -@item -128 MB RAM - -@item -600 MB free disk space - -@item -Windows 2000 or Windows XP are recommended -@end itemize - -@node Software installation -@subsubsection Software installation - -@itemize @bullet - -@item -@strong{Compression software}@ -@ - The software packages GNUnet depends on are usually compressed using UNIX - tools like tar, gzip and bzip2.@ If you do not already have an utility that is - able to extract such archives, get @uref{http://www.7-zip.org/, 7-Zip}. - -@item -@strong{UNIX environment}@ -@ -The MinGW project provides the compiler toolchain that is used to build -GNUnet.@ Get the following packages from -@uref{http://sourceforge.net/projects/mingw/files/, the MinGW project}: -@itemize @bullet - - -@item -GCC core - -@item -GCC g++ - -@item -MSYS - -@item -MSYS Developer Tool Kit (msysDTK) - -@item -MSYS Developer Tool Kit - msys-autoconf (bin) - -@item -MSYS Developer Tool Kit - msys-automake (bin) - -@item -MinGW Runtime - -@item -MinGW Utilities - -@item -Windows API - -@item -Binutils - -@item -make - -@item -pdcurses - -@item -GDB (snapshot) -@end itemize - -@itemize @bullet - - -@item -Install MSYS (to c:\mingw, for example.)@ -Do @strong{not} use spaces in the pathname (c:\program files\mingw). - -@item -Install MinGW runtime, utilities and GCC to a subdirectory (to c:\mingw\mingw, -for example) - -@item -Install the Development Kit to the MSYS directory (c:\mingw) - -@item -Create a batch file bash.bat in your MSYS directory with the files:@ - -@example -bin\sh.exe --login -@end example - - -This batch file opens a shell which is used to invoke the build processes..@ -MinGW's standard shell (msys.bat) is not suitable because it opens a separate -console window@ On Vista, bash.bat needs to be run as administrator. - -@item -Start bash.sh and rename (c:\mingw\mingw\)lib\libstdc++.la to avoid problems:@ - -@example -mv /usr/mingw/lib/libstdc++.la /usr/mingw/lib/libstdc++.la.broken -@end example - - -@item -Unpack the Windows API to the MinGW directory (c:\mingw\mingw\) and remove the -declaration of DATADIR from (c:\mingw\mingw\)include\objidl.h (lines 55-58) - -@item -Unpack autoconf, automake to the MSYS directory (c:\mingw) - -@item -Install all other packages to the MinGW directory (c:\mingw\mingw\) -@end itemize - - -@item -@strong{GNU Libtool}@ -@ -GNU Libtool is required to use shared libraries.@ -@ -Get the prebuilt package from here and unpack it to the MinGW directory -(c:\mingw) - -@item -@strong{Pthreads}@ -@ -GNUnet uses the portable POSIX thread library for multi-threading..@ - -@itemize @bullet - - -@item -Save @uref{ftp://sources.redhat.com/pub/pthreads-win32/dll-latest/lib/x86/libpthreadGC2.a, libpthreadGC2.a} (x86) or @uref{ftp://sources.redhat.com/pub/pthreads-win32/dll-latest/lib/x64/libpthreadGC2.a, libpthreadGC2.a} (x64) as libpthread.a into the lib directory (c:\mingw\mingw\lib\libpthread.a) - -@item -Save @uref{ftp://sources.redhat.com/pub/pthreads-win32/dll-latest/lib/x86/pthreadGC2.dll, pthreadGC2.dll} (x86) or @uref{ftp://sources.redhat.com/pub/pthreads-win32/dll-latest/lib/x64/pthreadGC2.dll, libpthreadGC2.a} (x64) into the MinGW bin directory (c:\mingw\mingw\bin) - -@item -Download all header files from @uref{ftp://sources.redhat.com/pub/pthreads-win32/dll-latest/include/, include/} to the include directory (c:\mingw\mingw\include) -@end itemize - - -@item -@strong{GNU MP@ -}@ -@ -GNUnet uses the GNU Multiple Precision library for special cryptographic operations.@ -@ -Get the GMP binary package from the @uref{http://sourceforge.net/projects/mingwrep/, MinGW repository} and unpack it to the MinGW directory (c:\mingw\mingw) - -@item -@strong{GNU Gettext}@ -@ - GNU gettext is used to provide national language support.@ -@ - Get the prebuilt package from hereand unpack it to the MinGW directory (c:\mingw\mingw) - -@item -@strong{GNU iconv}@ -@ - GNU Libiconv is used for character encoding conversion.@ -@ - Get the prebuilt package from here and unpack it to the MinGW directory (c:\mingw\mingw) - -@item -@strong{SQLite}@ -@ - GNUnet uses the SQLite database to store data.@ -@ - Get the prebuilt binary from here and unpack it to your MinGW directory. - -@item -@strong{MySQL}@ -@ - As an alternative to SQLite, GNUnet also supports MySQL. -@itemize @bullet - - -@item - Get the binary installer from the @uref{http://dev.mysql.com/downloads/mysql/4.1.html#Windows, MySQL project} (version 4.1),@ - install it and follow the instructions in README.mysql. - -@item - Create a temporary build directory (c:\mysql) - -@item - Copy the directories include\ and lib\ from the MySQL directory to the new directory - -@item - Get the patches from @uref{http://bugs.mysql.com/bug.php?id=8906&files=1, Bug #8906} and @uref{http://bugs.mysql.com/bug.php?id=8872&files=1, Bug #8872} (the latter is only required for MySQL -@example -patch -p 0 -@end example - - -@item - Move lib\opt\libmysql.dll to lib\libmysql.dll - -@item - Change to lib\ and create an import library:@ - -@example -dlltool --input-def ../include/libmySQL.def --dllname libmysql.dll - --output-lib libmysqlclient.a -k -@end example - - -@item - Copy include\* to include\mysql\ - -@item - Pass "--with-mysql=/c/mysql" to ./configure and copy libmysql.dll to your PATH or GNUnet's @file{bin} directory -@end itemize - - -@item -@strong{GTK+}@ -@ - gnunet-gtk and libextractor depend on GTK.@ -@ - Get the the binary and developer packages of atk, glib, gtk, iconv, gettext-runtime, pango from @uref{ftp://ftp.gtk.org/pub/gtk/v2.6/win32, gtk.org} and unpack it to the MinGW directory (c:\mingw\mingw)@ -@ - Get @uref{http://www.gtk.org/download/win32.php, pkg-config} and libpng and unpack them to the MinGW directory (c:\mingw\mingw)@ -@ - Here is an all-in-one package for @uref{http://ftp.gnome.org/pub/gnome/binaries/win32/gtk+/2.24/gtk+-bundle_2.24.10-20120208_win32.zip, gtk+dependencies}. Do not overwrite any existing files! - -@item -@strong{Glade}@ -@ - gnunet-gtk and and gnunet-setup were created using this interface builder@ - -@itemize @bullet - - -@item - Get the Glade and libglade (-bin and -devel) packages (without GTK!) from @uref{http://gladewin32.sourceforge.net/, GladeWin32} and unpack it to the MinGW directory (c:\mingw\mingw) - -@item - Get libxml from here and unpack it to the MinGW directory (c:\mingw\mingw). -@end itemize - - -@item -@strong{zLib}@ -@ - libextractor requires zLib to decompress some file formats. GNUnet uses it to (de)compress meta-data.@ -@ - Get zLib from here (Signature) and unpack it to the MinGW directory (c:\mingw\mingw) - -@item -@strong{Bzip2}@ -@ - libextractor also requires Bzip2 to decompress some file formats.@ -@ - Get Bzip2 (binary and developer package) from @uref{http://gnuwin32.sourceforge.net/packages/bzip2.htm, GnuWin32} and unpack it to the MinGW directory (c:\mingw\mingw) - -@item -@strong{Libgcrypt}@ -@ - Libgcrypt provides the cryptographic functions used by GNUnet@ -@ - Get Libgcrypt from @uref{ftp://ftp.gnupg.org/gcrypt/libgcrypt/, here}, compile and place it in the MinGW directory (c:\mingw\mingw). Currently you need at least version 1.4.2 to compile gnunet. - -@item -@strong{PlibC}@ -@ - PlibC emulates Unix functions under Windows.@ -@ - Get PlibC from here and unpack it to the MinGW directory (c:\mingw\mingw) - -@item -@strong{OGG Vorbis}@ -@ - OGG Vorbis is used to extract meta-data from .ogg files@ -@ - Get the packages @uref{http://www.gnunet.org/libextractor/download/win/libogg-1.1.4.zip, libogg} and @uref{http://www.gnunet.org/libextractor/download/win/libvorbis-1.2.3.zip, libvorbis} from the @uref{http://ftp.gnu.org/gnu/libextractor/libextractor-w32-1.0.0.zip, libextractor win32 build} and unpack them to the MinGW directory (c:\mingw\mingw) - -@item -@strong{Exiv2}@ -@ - (lib)Exiv2 is used to extract meta-data from files with Exiv2 meta-data@ -@ - Download @uref{http://www.gnunet.org/libextractor/download/win/exiv2-0.18.2.zip, Exiv2} and unpack it to the MSYS directory (c:\mingw) -@end itemize - -@node Building libextractor and GNUnet -@subsubsection Building libextractor and GNUnet - -Before you compile libextractor or GNUnet, be sure to set@ -PKG_CONFIG_PATH: -@example -export PKG_CONFIG_PATH=/mingw/lib/pkgconfig -@end example - - - See Installation for basic instructions on building libextractor and GNUnet. - - By default, all modules that are created in this way contain debug information and are quite large.@ - To compile release versions (small and fast) set the variable CFLAGS: -@example -export CFLAGS='-O2 -march=pentium -fomit-frame-pointer' -./configure --prefix=$HOME --with-extractor=$HOME -@end example - -@node Installer -@subsubsection Installer - - The GNUnet installer is made with @uref{http://nsis.sourceforge.net/, NSIS}@ - The installer script is located in contrib\win in the GNUnet source tree. - -@node Source -@subsubsection Source - -The sources of all dependencies are available here. - -@node Portable GNUnet -@section Portable GNUnet - -Quick instructions on how to use the most recent GNUnet on most GNU/Linux -distributions - -Currently this has only been tested on Ubuntu 12.04, 12.10, 13.04, Debian and -CentOS 6, but it should work on almost any GNU/Linux distribution. More -in-detail information can be found in the handbook. - - - -@menu -* Prerequisites:: -* Download & set up gnunet-update:: -* Install GNUnet:: -@end menu - -@node Prerequisites -@subsection Prerequisites - -Open a terminal and paste this line into it to install all required tools -needed:@ -@code{sudo apt-get install python-gpgme subversion} - -@node Download & set up gnunet-update -@subsection Download & set up gnunet-update - -The following command will download a working version of gnunet-update with the -subversion tool and import the public key which is needed for authentication:@ - -@example -svn checkout -r24905 https://gnunet.org/svn/gnunet-update ~/gnunet-update && -cd ~/gnunet-update -gpg --keyserver "hkp://keys.gnupg.net" --recv-keys 7C613D78 -@end example - -@node Install GNUnet -@subsection Install GNUnet - -Download and install GNUnet binaries which can be found here and set library -paths:@ -@code{@ - wget -P /tmp https://gnunet.org/install/packs/gnunet-0.9.4-`uname -m`.tgz@ - ./bin/gnunet-update install /tmp/gnunet-0.9*.tgz ~@ - echo "PATH DEFAULT=$@{PATH@}:$HOME/bin" >> ~/.pam_environment@ - echo -e "$@{HOME@}/lib\n$@{HOME@}/lib/gnunet-deps" | sudo tee /etc/ld.so.conf.d/gnunet.conf > /dev/null@ - sudo ldconfig@ -}@ - -You may need to re-login once after executing these last commands - -That's it, GNUnet is installed in your home directory now. GNUnet can be -configured and afterwards started by executing@ -@code{gnunet-arm -s} - -@node The graphical configuration interface -@section The graphical configuration interface - -If you also would like to use gnunet-gtk and gnunet-setup (highly recommended -for beginners), do: - -@example -wget -P /tmp https://gnunet.org/install/packs/gnunet-0.9.4-gtk-0.9.4-`uname -m`.tgz@ -sh ~/gnunet-update/bin/gnunet-update install /tmp/gnunet-*gtk*.tgz ~@ -sudo ldconfig -@end example -Now you can run @code{gnunet-setup} for easy configuration of your GNUnet peer. - - -@menu -* Configuring your peer:: -* Configuring the Friend-to-Friend (F2F) mode:: -* Configuring the hostlist to bootstrap:: -* Configuration of the HOSTLIST proxy settings:: -* Configuring your peer to provide a hostlist :: -* Configuring the datastore:: -* Configuring the MySQL database:: -* Reasons for using MySQL:: -* Reasons for not using MySQL:: -* Setup Instructions:: -* Testing:: -* Performance Tuning:: -* Setup for running Testcases:: -* Configuring the Postgres database:: -* Reasons to use Postgres:: -* Reasons not to use Postgres:: -* Manual setup instructions:: -* Testing the setup manually:: -* Configuring the datacache:: -* Configuring the file-sharing service:: -* Configuring logging:: -* Configuring the transport service and plugins:: -* Configuring the wlan transport plugin:: -* Configuring HTTP(S) reverse proxy functionality using Apache or nginx:: -* Blacklisting peers:: -* Configuration of the HTTP and HTTPS transport plugins:: -* Configuring the GNU Name System:: -* Configuring the GNUnet VPN:: -* Bandwidth Configuration:: -* Configuring NAT:: -* Peer configuration for distributions:: -@end menu - -@node Configuring your peer -@subsection Configuring your peer - -This chapter will describe the various configuration options in GNUnet. - -The easiest way to configure your peer is to use the gnunet-setup tool. -gnunet-setup is part of the gnunet-gtk download. You might have to install it -separately. - -Many of the specific sections from this chapter actually are linked from within -gnunet-setup to help you while using the setup tool. - -While you can also configure your peer by editing the configuration file by -hand, this is not recommended for anyone except for developers. - - - - - -@node Configuring the Friend-to-Friend (F2F) mode -@subsection Configuring the Friend-to-Friend (F2F) mode - -GNUnet knows three basic modes of operation. In standard "peer-to-peer" mode, -your peer will connect to any peer. In the pure "friend-to-friend" mode, your -peer will ONLY connect to peers from a list of friends specified in the -configuration. Finally, in mixed mode, GNUnet will only connect to arbitrary -peers if it has at least a specified number of connections to friends. - -When configuring any of the F2F modes, you first need to create a file with the -peer identities of your friends. Ask your friends to run - -$ gnunet-peerinfo -sq - -The output of this command needs to be added to your friends file, which is -simply a plain text file with one line per friend with the output from the -above command. - -You then specify the location of your friends file in the "FRIENDS" option of -the "topology" section. - -Once you have created the friends file, you can tell GNUnet to only connect to -your friends by setting the "FRIENDS-ONLY" option (again in the "topology" -section) to YES. - -If you want to run in mixed-mode, set "FRIENDS-ONLY" to NO and configure a -minimum number of friends to have (before connecting to arbitrary peers) under -the "MINIMUM-FRIENDS" option. - -If you want to operate in normal P2P-only mode, simply set "MINIMUM-FRIENDS" to -zero and "FRIENDS_ONLY" to NO. This is the default. - -@node Configuring the hostlist to bootstrap -@subsection Configuring the hostlist to bootstrap - -After installing the software you need to get connected to the GNUnet network. -The configuration file included in your download is already configured to -connect you to the GNUnet network. In this section the relevant configuration -settings are explained. - -To get an initial connection to the GNUnet network and to get to know peers -already connected to the network you can use the so called bootstrap servers. -These servers can give you a list of peers connected to the network. To use -these bootstrap servers you have to configure the hostlist daemon to activate -bootstrapping. - -To activate bootstrapping edit your configuration file and edit the -@code{[hostlist]}-section. You have to set the argument "-b" in the options -line: -@example -[hostlist] -OPTIONS = -b -@end example - -Additionally you have to specify which server you want to use. The default -bootstrapping server is "@uref{http://v10.gnunet.org/hostlist, -http://v10.gnunet.org/hostlist}". [^] To set the server you have to edit the -line "SERVERS" in the hostlist section. To use the default server you should -set the lines to -@example -SERVERS = http://v10.gnunet.org/hostlist [^] -@end example - - -To use bootstrapping your configuration file should include these lines: -@example -[hostlist] -OPTIONS = -b -SERVERS = http://v10.gnunet.org/hostlist [^] -@end example - - -Besides using bootstrap servers you can configure your GNUnet peer to recieve -hostlist advertisements. Peers offering hostlists to other peers can send -advertisement messages to peers that connect to them. If you configure your -peer to receive these messages, your peer can download these lists and connect -to the peers included. These lists are persistent, which means that they are -saved to your hard disk regularly and are loaded during startup. - -To activate hostlist learning you have to add the "-e" switch to the OPTIONS -line in the hostlist section: -@example -[hostlist] -OPTIONS = -b -e -@end example - - -Furthermore you can specify in which file the lists are saved. To save the -lists in the file "hostlists.file" just add the line: -@example -HOSTLISTFILE = hostlists.file -@end example - - -Best practice is to activate both bootstrapping and hostlist learning. So your -configuration file should include these lines: -@example -[hostlist] -OPTIONS = -b -e -HTTPPORT = 8080 -SERVERS = http://v10.gnunet.org/hostlist [^] -HOSTLISTFILE = $SERVICEHOME/hostlists.file -@end example - -@node Configuration of the HOSTLIST proxy settings -@subsection Configuration of the HOSTLIST proxy settings - -The hostlist client can be configured to use a proxy to connect to the hostlist -server. This functionality can be configured in the configuration file directly -or using the gnunet-setup tool. - -The hostlist client supports the following proxy types at the moment: -@itemize @bullet - - -@item -HTTP and HTTP 1.0 only proxy - -@item -SOCKS 4/4a/5/5 with hostname -@end itemize - - -In addition authentication at the proxy with username and password can be -configured. - -To configure proxy support for the hostlist client in the gnunet-setup tool, -select the "hostlist" tab and select the appropriate proxy type. The hostname -or IP address (including port if required) has to be entered in the "Proxy -hostname" textbox. If required, enter username and password in the "Proxy -username" and "Proxy password" boxes. Be aware that these information will be -stored in the configuration in plain text. - -To configure these options directly in the configuration, you can configure the -following settings in the @code{[hostlist]} section of the configuration:@ -@example - # Type of proxy server,@ - # Valid values: HTTP, HTTP_1_0, SOCKS4, SOCKS5, SOCKS4A, SOCKS5_HOSTNAME@ - # Default: HTTP@ - # PROXY_TYPE = HTTP - -# Hostname or IP of proxy server@ - # PROXY =@ - # User name for proxy server@ - # PROXY_USERNAME =@ - # User password for proxy server@ - # PROXY_PASSWORD =@ -@end example - -@node Configuring your peer to provide a hostlist -@subsection Configuring your peer to provide a hostlist - -If you operate a peer permanently connected to GNUnet you can configure your -peer to act as a hostlist server, providing other peers the list of peers known -to him. - -Yor server can act as a bootstrap server and peers needing to obtain a list of -peers can contact him to download this list. To download this hostlist the peer -uses HTTP. For this reason you have to build your peer with libcurl and -microhttpd support. How you build your peer with this options can be found -here: https://gnunet.org/generic_installation - -To configure your peer to act as a bootstrap server you have to add the "-p" -option to OPTIONS in the [hostlist] section of your configuration file. Besides -that you have to specify a port number for the http server. In conclusion you -have to add the following lines: - -@example -[hostlist] -HTTPPORT = 12980 -OPTIONS = -p -@end example - - -If your peer acts as a bootstrap server other peers should know about that. You -can advertise the hostlist your are providing to other peers. Peers connecting -to your peer will get a message containing an advertisement for your hostlist -and the URL where it can be downloaded. If this peer is in learning mode, it -will test the hostlist and, in the case it can obtain the list successfully, it -will save it for bootstrapping. - -To activate hostlist advertisement on your peer, you have to set the following -lines in your configuration file: -@example -[hostlist] -EXTERNAL_DNS_NAME = example.org -HTTPPORT = 12981 -OPTIONS = -p -a -@end example - - -With this configuration your peer will a act as a bootstrap server and -advertise this hostlist to other peers connecting to him. The URL used to -download the list will be @code{@uref{http://example.org:12981/, -http://example.org:12981/}}. - -Please notice: -@itemize @bullet - - -@item -The hostlist is not human readable, so you should not try to download it using -your webbrowser. Just point your GNUnet peer to the address! - -@item -Advertising without providing a hostlist does not make sense and will not work. -@end itemize - -@node Configuring the datastore -@subsection Configuring the datastore - -The datastore is what GNUnet uses to for long-term storage of file-sharing -data. Note that long-term does not mean 'forever' since content does have an -expiration date, and of course storage space is finite (and hence sometimes -content may have to be discarded). - -Use the "QUOTA" option to specify how many bytes of storage space you are -willing to dedicate to GNUnet. - -In addition to specifying the maximum space GNUnet is allowed to use for the -datastore, you need to specify which database GNUnet should use to do so. -Currently, you have the choice between sqLite, MySQL and Postgres. - -@node Configuring the MySQL database -@subsection Configuring the MySQL database - -This section describes how to setup the MySQL database for GNUnet. - -Note that the mysql plugin does NOT work with mysql before 4.1 since we need -prepared statements. We are generally testing the code against MySQL 5.1 at -this point. - -@node Reasons for using MySQL -@subsection Reasons for using MySQL - -@itemize @bullet - -@item -On up-to-date hardware where mysql can be used comfortably, this module will -have better performance than the other database choices (according to our -tests). - -@item Its often possible to recover the mysql database from internal -inconsistencies. Some of the other databases do not support repair. -@end itemize - -@node Reasons for not using MySQL -@subsection Reasons for not using MySQL - -@itemize @bullet - -@item -Memory usage (likely not an issue if you have more than 1 GB) - -@item -Complex manual setup -@end itemize - -@node Setup Instructions -@subsection Setup Instructions - -@itemize @bullet - -@item -In @code{gnunet.conf} set in section "DATASTORE" the value for "DATABASE" to -"mysql". - -@item -Access mysql as root:@ - -@example -$ mysql -u root -p -@end example - - -and issue the following commands, replacing $USER with the username@ - that will be running gnunet-arm (so typically "gnunet"): -@example -CREATE DATABASE gnunet; -GRANT select,insert,update,delete,create,alter,drop,create temporary tables - ON gnunet.* TO $USER@@localhost; -SET PASSWORD FOR $USER@@localhost=PASSWORD('$the_password_you_like'); -FLUSH PRIVILEGES; -@end example - - -@item -In the $HOME directory of $USER, create a ".my.cnf" file with the following lines@ - -@example -[client] -user=$USER -password=$the_password_you_like -@end example - -@end itemize - - - Thats it. Note that @code{.my.cnf} file is a slight security risk unless its - on@ a safe partition. The $HOME/.my.cnf can of course be a symbolic@ link. - Luckily $USER has only priviledges to mess up GNUnet's tables, which should be - pretty harmless. -@node Testing -@subsection Testing - -You should briefly try if the database connection works. First, login as $USER. -Then use: -@example -$ mysql -u $USER -mysql> use gnunet; -@end example - - -If you get the message "Database changed" it probably works. - -If you get "ERROR 2002: Can't connect to local MySQL server@ - through socket '/tmp/mysql.sock' (2)" it may be resolvable by@ - "ln -s /var/run/mysqld/mysqld.sock /tmp/mysql.sock"@ - so there may be some additional trouble depending on your mysql setup. -@node Performance Tuning -@subsection Performance Tuning - -For GNUnet, you probably want to set the option -@example -innodb_flush_log_at_trx_commit = 0 -@end example - -for a rather dramatic boost in MySQL performance. However, this reduces the -"safety" of your database as with this options you may loose transactions -during a power outage. While this is totally harmless for GNUnet, the option -applies to all applications using MySQL. So you should set it if (and only if) -GNUnet is the only application on your system using MySQL. - -@node Setup for running Testcases -@subsection Setup for running Testcases - -If you want to run the testcases, you must create a second database -"gnunetcheck" with the same username and password. This database will then be -used for testing ("make check"). - -@node Configuring the Postgres database -@subsection Configuring the Postgres database - -This text describes how to setup the Postgres database for GNUnet. - -This Postgres plugin was developed for Postgres 8.3 but might work for earlier -versions as well. - -@node Reasons to use Postgres -@subsection Reasons to use Postgres - -@itemize @bullet -@item -Easier to setup than MySQL -@item -Real database -@end itemize - -@node Reasons not to use Postgres -@subsection Reasons not to use Postgres - -@itemize @bullet -@item -Quite slow -@item -Still some manual setup required -@end itemize - -@node Manual setup instructions -@subsection Manual setup instructions - -@itemize @bullet - -@item -In @code{gnunet.conf} set in section "DATASTORE" the value for@ -"DATABASE" to "postgres". -@item -Access Postgres to create a user:@ - -@table @asis - -@item with Postgres 8.x, use: - -@example -# su - postgres -$ createuser -@end example - -and enter the name of the user running GNUnet for the role interactively. -Then, when prompted, do not set it to superuser, allow the creation of -databases, and do not allow the creation of new roles.@ - -@item with Postgres 9.x, use: - -@example -# su - postgres -$ createuser -d $GNUNET_USER -@end example - - -where $GNUNET_USER is the name of the user running GNUnet.@ - -@end table - - -@item -As that user (so typically as user "gnunet"), create a database (or two):@ - -@example -$ createdb gnunet -$ createdb gnunetcheck # this way you can run "make check" -@end example - -@end itemize - - -Now you should be able to start @code{gnunet-arm}. - -@node Testing the setup manually -@subsection Testing the setup manually - -You may want to try if the database connection works. First, again login as -the user who will run gnunet-arm. Then use, -@example -$ psql gnunet # or gnunetcheck -gnunet=> \dt -@end example - - -If, after you have started gnunet-arm at least once, you get a @code{gn090} -table here, it probably works. - -@node Configuring the datacache -@subsection Configuring the datacache -@c %**end of header - -The datacache is what GNUnet uses for storing temporary data. This data is -expected to be wiped completely each time GNUnet is restarted (or the system -is rebooted). - -You need to specify how many bytes GNUnet is allowed to use for the datacache -using the "QUOTA" option in the section "dhtcache". Furthermore, you need to -specify which database backend should be used to store the data. Currently, -you have the choice between sqLite, MySQL and Postgres. - -@node Configuring the file-sharing service -@subsection Configuring the file-sharing service - -In order to use GNUnet for file-sharing, you first need to make sure that the -file-sharing service is loaded. This is done by setting the AUTOSTART option in -section "fs" to "YES". Alternatively, you can run -@example -$ gnunet-arm -i fs -@end example - -to start the file-sharing service by hand. - -Except for configuring the database and the datacache the only important option -for file-sharing is content migration. - -Content migration allows your peer to cache content from other peers as well as -send out content stored on your system without explicit requests. This content -replication has positive and negative impacts on both system performance an -privacy. - -FIXME: discuss the trade-offs. Here is some older text about it... - -Setting this option to YES allows gnunetd to migrate data to the local machine. -Setting this option to YES is highly recommended for efficiency. Its also the -default. If you set this value to YES, GNUnet will store content on your -machine that you cannot decrypt. While this may protect you from liability if -the judge is sane, it may not (IANAL). If you put illegal content on your -machine yourself, setting this option to YES will probably increase your chances -to get away with it since you can plausibly deny that you inserted the content. -Note that in either case, your anonymity would have to be broken first (which -may be possible depending on the size of the GNUnet network and the strength of -the adversary). - -@node Configuring logging -@subsection Configuring logging - -Logging in GNUnet 0.9.0 is controlled via the "-L" and "-l" options. -Using "-L", a log level can be specified. With log level "ERROR" only serious -errors are logged. The default log level is "WARNING" which causes anything of -concern to be logged. Log level "INFO" can be used to log anything that might -be interesting information whereas "DEBUG" can be used by developers to log -debugging messages (but you need to run configure with -@code{--enable-logging=verbose} to get them compiled). The "-l" option is used -to specify the log file. - -Since most GNUnet services are managed by @code{gnunet-arm}, using the "-l" or -"-L" options directly is not possible. Instead, they can be specified using the -"OPTIONS" configuration value in the respective section for the respective -service. In order to enable logging globally without editing the "OPTIONS" -values for each service, @code{gnunet-arm} supports a "GLOBAL_POSTFIX" option. -The value specified here is given as an extra option to all services for which -the configuration does contain a service-specific "OPTIONS" field. - -"GLOBAL_POSTFIX" can contain the special sequence "@{@}" which is replaced by -the name of the service that is being started. Furthermore, -@code{GLOBAL_POSTFIX} is special in that sequences starting with "$" anywhere -in the string are expanded (according to options in "PATHS"); this expansion -otherwise is only happening for filenames and then the "$" must be the first -character in the option. Both of these restrictions do not apply to -"GLOBAL_POSTFIX". Note that specifying @code{%} anywhere in the "GLOBAL_POSTFIX" -disables both of these features. - -In summary, in order to get all services to log at level "INFO" to log-files -called @code{SERVICENAME-logs}, the following global prefix should be used: -@example -GLOBAL_POSTFIX = -l $SERVICEHOME/@{@}-logs -L INFO -@end example - -@node Configuring the transport service and plugins -@subsection Configuring the transport service and plugins - -The transport service in GNUnet is responsible to maintain basic connectivity -to other peers. Besides initiating and keeping connections alive it is also -responsible for address validation. - -The GNUnet transport supports more than one transport protocol. These protocols -are configured together with the transport service. - -The configuration section for the transport service itself is quite similar to -all the other services - -@code{@ - AUTOSTART = YES@ - @@UNIXONLY@@ PORT = 2091@ - HOSTNAME = localhost@ - HOME = $SERVICEHOME@ - CONFIG = $DEFAULTCONFIG@ - BINARY = gnunet-service-transport@ - #PREFIX = valgrind@ - NEIGHBOUR_LIMIT = 50@ - ACCEPT_FROM = 127.0.0.1;@ - ACCEPT_FROM6 = ::1;@ - PLUGINS = tcp udp@ - UNIXPATH = /tmp/gnunet-service-transport.sock@ -} - -Different are the settings for the plugins to load @code{PLUGINS}. The first -setting specifies which transport plugins to load. -@itemize @bullet - - -@item -transport-unix - -A plugin for local only communication with UNIX domain sockets. Used for -testing and available on unix systems only. Just set the port - -@code{@ - [transport-unix]@ - PORT = 22086@ - TESTING_IGNORE_KEYS = ACCEPT_FROM;@ -} - -@item -transport-tcp - -A plugin for communication with TCP. Set port to 0 for client mode with -outbound only connections - -@code{@ - [transport-tcp]@ - # Use 0 to ONLY advertise as a peer behind NAT (no port binding)@ - PORT = 2086@ - ADVERTISED_PORT = 2086@ - TESTING_IGNORE_KEYS = ACCEPT_FROM;@ - # Maximum number of open TCP connections allowed@ - MAX_CONNECTIONS = 128@ -} - -@item -transport-udp - -A plugin for communication with UDP. Supports peer discovery using broadcasts.@ -@code{@ - [transport-udp]@ - PORT = 2086@ - BROADCAST = YES@ - BROADCAST_INTERVAL = 30 s@ - MAX_BPS = 1000000@ - TESTING_IGNORE_KEYS = ACCEPT_FROM;@ -} - -@item -transport-http - -HTTP and HTTPS support is split in two part: a client plugin initiating -outbound connections and a server part accepting connections from the client. -The client plugin just takes the maximum number of connections as an argument.@ -@code{@ - [transport-http_client]@ - MAX_CONNECTIONS = 128@ - TESTING_IGNORE_KEYS = ACCEPT_FROM;@ -}@ -@code{@ - [transport-https_client]@ - MAX_CONNECTIONS = 128@ - TESTING_IGNORE_KEYS = ACCEPT_FROM;@ -} - -The server has a port configured and the maximum nunber of connections.@ - The HTTPS part has two files with the certificate key and the certificate file. - -The server plugin supports reverse proxies, so a external hostname can be set -using@ -the @code{EXTERNAL_HOSTNAME} setting. The webserver under this address should -forward the request to the peer and the configure port. - -@code{@ - [transport-http_server]@ - EXTERNAL_HOSTNAME = fulcrum.net.in.tum.de/gnunet@ - PORT = 1080@ - MAX_CONNECTIONS = 128@ - TESTING_IGNORE_KEYS = ACCEPT_FROM;@ -}@ -@code{@ - [transport-https_server]@ - PORT = 4433@ - CRYPTO_INIT = NORMAL@ - KEY_FILE = https.key@ - CERT_FILE = https.cert@ - MAX_CONNECTIONS = 128@ - TESTING_IGNORE_KEYS = ACCEPT_FROM;@ -} - -@item -transport-wlan - -There is a special article how to setup the WLAN plugin, so here only the -settings. Just specify the interface to use:@ -@code{@ - [transport-wlan]@ - # Name of the interface in monitor mode (typically monX)@ - INTERFACE = mon0@ - # Real hardware, no testing@ - TESTMODE = 0@ - TESTING_IGNORE_KEYS = ACCEPT_FROM;@ -} -@end itemize - -@node Configuring the wlan transport plugin -@subsection Configuring the wlan transport plugin - - -The wlan transport plugin enables GNUnet to send and to receive data on a wlan -interface. It has not to be connected to a wlan network as long as sender and -receiver are on the same channel. This enables you to get connection to the -GNUnet where no internet access is possible, for example while catastrophes or -when censorship cuts you off the internet. - - -@menu -* Requirements for the WLAN plugin:: -* Configuration:: -* Before starting GNUnet:: -* Limitations and known bugs:: -@end menu - - -@node Requirements for the WLAN plugin -@subsubsection Requirements for the WLAN plugin - -@itemize @bullet - -@item -wlan network card with monitor support and packet injection -(see @uref{http://www.aircrack-ng.org/, aircrack-ng.org}) - -@item -Linux kernel with mac80211 stack, introduced in 2.6.22, tested with 2.6.35 -and 2.6.38 - -@item -Wlantools to create the a monitor interface, tested with airmon-ng of the -aircrack-ng package -@end itemize - -@node Configuration -@subsubsection Configuration - -There are the following options for the wlan plugin (they should be like this -in your default config file, you only need to adjust them if the values are -incorrect for your system)@ -@code{@ -# section for the wlan transport plugin@ -[transport-wlan]@ -# interface to use, more information in the -# "Before starting GNUnet" section of the handbook. -INTERFACE = mon0@ -# testmode for developers:@ -# 0 use wlan interface,@ -#1 or 2 use loopback driver for tests 1 = server, 2 = client@ -TESTMODE = 0@ -} - -@node Before starting GNUnet -@subsubsection Before starting GNUnet - -Before starting GNUnet, you have to make sure that your wlan interface is in -monitor mode. One way to put the wlan interface into monitor mode (if your -interface name is wlan0) is by executing:@ -@code{@ - sudo airmon-ng start wlan0@ -} - -Here is an example what the result should look like:@ -@code{@ - Interface Chipset Driver@ - wlan0 Intel 4965 a/b/g/n iwl4965 - [phy0]@ - (monitor mode enabled on mon0)@ -}@ -The monitor interface is mon0 is the one that you have to put into the -configuration file. - -@node Limitations and known bugs -@subsubsection Limitations and known bugs - -Wlan speed is at the maximum of 1 Mbit/s because support for choosing the wlan -speed with packet injection was removed in newer kernels. Please pester the -kernel developers about fixing this. - -The interface channel depends on the wlan network that the card is connected -to. If no connection has been made since the start of the computer, it is -usually the first channel of the card. Peers will only find each other and -communicate if they are on the same channel. Channels must be set manually -(i.e. using @code{iwconfig wlan0 channel 1}). - - -@node Configuring HTTP(S) reverse proxy functionality using Apache or nginx -@subsection Configuring HTTP(S) reverse proxy functionality using Apache or nginx - -The HTTP plugin supports data transfer using reverse proxies. A reverse proxy -forwards the HTTP request he receives with a certain URL to another webserver, -here a GNUnet peer. - -So if you have a running Apache or nginx webserver you can configure it to be a -GNUnet reverse proxy. Especially if you have a well-known webiste this improves -censorship resistance since it looks as normal surfing behaviour. - -To do so, you have to do two things: - -@itemize @bullet - -@item -Configure your webserver to forward the GNUnet HTTP traffic - -@item -Configure your GNUnet peer to announce the respective address -@end itemize - -As an example we want to use GNUnet peer running: - -@itemize @bullet - -@item -HTTP server plugin on @code{gnunet.foo.org:1080} - -@item -HTTPS server plugin on @code{gnunet.foo.org:4433} - -@item -A apache or nginx webserver on @uref{http://www.foo.org/, http://www.foo.org:80/} - -@item -A apache or nginx webserver on https://www.foo.org:443/ -@end itemize - -And we want the webserver to accept GNUnet traffic under -@code{http://www.foo.org/bar/}. The required steps are described here: - -@strong{Configure your Apache2 HTTP webserver} - -First of all you need mod_proxy installed. - -Edit your webserver configuration. Edit @code{/etc/apache2/apache2.conf} or -the site-specific configuration file. - -In the respective @code{server config},@code{virtual host} or -@code{directory} section add the following lines:@ -@code{@ - ProxyTimeout 300@ - ProxyRequests Off@ - <Location /bar/ >@ - ProxyPass http://gnunet.foo.org:1080/@ - ProxyPassReverse http://gnunet.foo.org:1080/@ - </Location>@ -} - -@strong{Configure your Apache2 HTTPS webserver} - -We assume that you already have an HTTPS server running, if not please check -how to configure a HTTPS host. An easy to use example is the -@file{apache2/sites-available/default-ssl} example configuration file. - -In the respective HTTPS @code{server config},@code{virtual host} or -@code{directory} section add the following lines:@ -@code{@ - SSLProxyEngine On@ - ProxyTimeout 300@ - ProxyRequests Off@ - <Location /bar/ >@ - ProxyPass https://gnunet.foo.org:4433/@ - ProxyPassReverse https://gnunet.foo.org:4433/@ - </Location>@ -} - -More information about the apache mod_proxy configuration can be found unter:@ -@uref{http://httpd.apache.org/docs/2.2/mod/mod_proxy.html#proxypass, http://httpd.apache.org/docs/2.2/mod/mod_proxy.html#proxypass} - -@strong{Configure your nginx HTTPS webserver} - -Since nginx does not support chunked encoding, you first of all have to -install @code{chunkin}:@ -@uref{http://wiki.nginx.org/HttpChunkinModule, http://wiki.nginx.org/HttpChunkinModule} - -To enable chunkin add:@ -@code{@ - chunkin on;@ - error_page 411 = @@my_411_error;@ - location @@my_411_error @{@ - chunkin_resume;@ - @}@ -} - -Edit your webserver configuration. Edit @code{/etc/nginx/nginx.conf} or the -site-specific configuration file. - -In the @code{server} section add:@ -@code{@ - location /bar/@ - @{@ - proxy_pass http://gnunet.foo.org:1080/;@ - proxy_buffering off;@ - proxy_connect_timeout 5; # more than http_server@ - proxy_read_timeout 350; # 60 default, 300s is GNUnet's idle timeout@ - proxy_http_version 1.1; # 1.0 default@ - proxy_next_upstream error timeout invalid_header http_500 http_503 http_502 http_504;@ - @}@ -@code{}} - -@strong{Configure your nginx HTTPS webserver} - -Edit your webserver configuration. Edit @code{/etc/nginx/nginx.conf} or the -site-specific configuration file. - -In the @code{server} section add:@ -@code{@ - ssl_session_timeout 6m;@ - location /bar/@ - @{@ - proxy_pass https://gnunet.foo.org:4433/;@ - proxy_buffering off;@ - proxy_connect_timeout 5; # more than http_server@ - proxy_read_timeout 350; # 60 default, 300s is GNUnet's idle timeout@ - proxy_http_version 1.1; # 1.0 default@ - proxy_next_upstream error timeout invalid_header http_500 http_503 http_502 http_504;@ - @}@ -@code{}} - -@strong{Configure your GNUnet peer} - -To have your GNUnet peer announce the address, you have to specify the - -@code{EXTERNAL_HOSTNAME} option in the @code{[transport-http_server]} section:@ -@code{@ - [transport-http_server]@ - EXTERNAL_HOSTNAME = http://www.foo.org/bar/@ -}@ - and/or@ -@code{[transport-https_server]} section:@ -@code{@ - [transport-https_server]@ - EXTERNAL_HOSTNAME = https://www.foo.org/bar/@ -} - -Now restart your webserver and your peer... - -@node Blacklisting peers -@subsection Blacklisting peers - -Transport service supports to deny connecting to a specific peer of to a -specific peer with a specific transport plugin using te blacklisting component -of transport service. With@ blacklisting it is possible to deny connections to -specific peers of@ to use a specific plugin to a specific peer. Peers can be -blacklisted using@ the configuration or a blacklist client can be asked. - -To blacklist peers using the configuration you have to add a section to your@ -configuration containing the peer id of the peer to blacklist and the plugin@ -if required. - -Example:@ - To blacklist connections to P565... on peer AG2P... using tcp add:@ -@code{@ - [transport-blacklist AG2PHES1BARB9IJCPAMJTFPVJ5V3A72S3F2A8SBUB8DAQ2V0O3V8G6G2JU56FHGFOHMQVKBSQFV98TCGTC3RJ1NINP82G0RC00N1520]@ - P565723JO1C2HSN6J29TAQ22MN6CI8HTMUU55T0FUQG4CMDGGEQ8UCNBKUMB94GC8R9G4FB2SF9LDOBAJ6AMINBP4JHHDD6L7VD801G = tcp@ -}@ - To blacklist connections to P565... on peer AG2P... using all plugins add:@ -@code{@ - [transport-blacklist-AG2PHES1BARB9IJCPAMJTFPVJ5V3A72S3F2A8SBUB8DAQ2V0O3V8G6G2JU56FHGFOHMQVKBSQFV98TCGTC3RJ1NINP82G0RC00N1520]@ - P565723JO1C2HSN6J29TAQ22MN6CI8HTMUU55T0FUQG4CMDGGEQ8UCNBKUMB94GC8R9G4FB2SF9LDOBAJ6AMINBP4JHHDD6L7VD801G =@ -} - -You can also add a blacklist client usign the blacklist api. On a blacklist@ -check, blacklisting first checks internally if the peer is blacklisted and@ -if not, it asks the blacklisting clients. Clients are asked if it is OK to@ -connect to a peer ID, the plugin is omitted. - -On blacklist check for (peer, plugin) -@itemize @bullet -@item Do we have a local blacklist entry for this peer and this plugin?@ -@item YES: disallow connection@ -@item Do we have a local blacklist entry for this peer and all plugins?@ -@item YES: disallow connection@ -@item Does one of the clients disallow?@ -@item YES: disallow connection -@end itemize - -@node Configuration of the HTTP and HTTPS transport plugins -@subsection Configuration of the HTTP and HTTPS transport plugins - -The client part of the http and https transport plugins can be configured to -use a proxy to connect to the hostlist server. This functionality can be -configured in the configuration file directly or using the gnunet-setup tool. - -The both the HTTP and HTTPS clients support the following proxy types at the -moment: - -@itemize @bullet -@item HTTP 1.1 proxy -@item SOCKS 4/4a/5/5 with hostname -@end itemize - -In addition authentication at the proxy with username and password can be -configured. - -To configure proxy support for the clients in the gnunet-setup tool, select the -"transport" tab and activate the respective plugin. Now you can select the -appropriate proxy type. The hostname or IP address (including port if required) -has to be entered in the "Proxy hostname" textbox. If required, enter username -and password in the "Proxy username" and "Proxy password" boxes. Be aware that -these information will be stored in the configuration in plain text. - -To configure these options directly in the configuration, you can configure the -following settings in the [transport-http_client] and [transport-https_client] -section of the configuration: - -@example -# Type of proxy server,@ -# Valid values: HTTP, SOCKS4, SOCKS5, SOCKS4A, SOCKS5_HOSTNAME@ -# Default: HTTP@ -# PROXY_TYPE = HTTP - -# Hostname or IP of proxy server@ -# PROXY =@ -# User name for proxy server@ -# PROXY_USERNAME =@ -# User password for proxy server@ -# PROXY_PASSWORD = -@end example - -@node Configuring the GNU Name System -@subsection Configuring the GNU Name System - -@menu -* Configuring system-wide DNS interception:: -* Configuring the GNS nsswitch plugin:: -* Configuring GNS on W32:: -* GNS Proxy Setup:: -* Setup of the GNS CA:: -* Testing the GNS setup:: -* Automatic Shortening in the GNU Name System:: -@end menu - - -@node Configuring system-wide DNS interception -@subsubsection Configuring system-wide DNS interception - -Before you install GNUnet, make sure you have a user and group 'gnunet' as well -as an empty group 'gnunetdns'. - -When using GNUnet with system-wide DNS interception, it is absolutely necessary -for all GNUnet service processes to be started by @code{gnunet-service-arm} as -user and group 'gnunet'. You also need to be sure to run @code{make install} as -root (or use the @code{sudo} option to configure) to grant GNUnet sufficient -privileges. - -With this setup, all that is required for enabling system-wide DNS interception -is for some GNUnet component (VPN or GNS) to request it. The -@code{gnunet-service-dns} will then start helper programs that will make the -necessary changes to your firewall (@code{iptables}) rules. - -Note that this will NOT work if your system sends out DNS traffic to a -link-local IPv6 address, as in this case GNUnet can intercept the traffic, but -not inject the responses from the link-local IPv6 address. Hence you cannot use -system-wide DNS interception in conjunction with link-local IPv6-based DNS -servers. If such a DNS server is used, it will bypass GNUnet's DNS traffic -interception. - - - -Using the GNU Name System (GNS) requires two different configuration steps. -First of all, GNS needs to be integrated with the operating system. Most of -this section is about the operating system level integration. - -Additionally, each individual user who wants to use the system must also -initialize his GNS zones. This can be done by running (after starting GNUnet)@ -@code{@ - $ gnunet-gns-import.sh@ -}@ -after the local GNUnet peer has been started. Note that the namestore (in -particular the namestore database backend) should not be reconfigured -afterwards (as records are not automatically migrated between backends). - -The remainder of this chapter will detail the various methods for configuring -the use of GNS with your operating system. - -At this point in time you have different options depending on your OS: -@table @asis - -@item Use the gnunet-gns-proxy This approach works for all operating systems -and is likely the easiest. However, it enables GNS only for browsers, not for -other applications that might be using DNS, such as SSH. Still, using the proxy -is required for using HTTP with GNS and is thus recommended for all users. To -do this, you simply have to run the @code{gnunet-gns-proxy-setup-ca} script as -the user who will run the browser (this will create a GNS certificate authority -(CA) on your system and import its key into your browser), then start -@code{gnunet-gns-proxy} and inform your browser to use the Socks5 proxy which -@code{gnunet-gns-proxy} makes available by default on port 7777. -@item Use a -nsswitch plugin (recommended on GNU systems) This approach has the advantage of -offering fully personalized resolution even on multi-user systems. A potential -disadvantage is that some applications might be able to bypass GNS. -@item Use -a W32 resolver plugin (recommended on W32) This is currently the only option on -W32 systems. -@item Use system-wide DNS packet interception This approach is -recommended for the GNUnet VPN. It can be used to handle GNS at the same time; -however, if you only use this method, you will only get one root zone per -machine (not so great for multi-user systems). -@end table - - -You can combine system-wide DNS packet interception with the nsswitch plugin.@ -The setup of the system-wide DNS interception is described here. All of the -other GNS-specific configuration steps are described in the following sections. - -@node Configuring the GNS nsswitch plugin -@subsubsection Configuring the GNS nsswitch plugin - -The Name Service Switch (NSS) is a facility in Unix-like operating systems that -provides a variety of sources for common configuration databases and name -resolution mechanisms. A system administrator usually configures the operating -system's name services using the file /etc/nsswitch.conf. - -GNS provides a NSS plugin to integrate GNS name resolution with the operating -system's name resolution process. To use the GNS NSS plugin you have to either - -@itemize @bullet - -@item -install GNUnet as root or - -@item -compile GNUnet with the @code{--with-sudo=yes} switch. -@end itemize - -Name resolution is controlled by the @emph{hosts} section in the NSS -configuration. By default this section first performs a lookup in the -/etc/hosts file and then in DNS. The nsswitch file should contain a line -similar to:@ -@code{@ - hosts: files dns [NOTFOUND=return] mdns4_minimal mdns4@ -} - -Here the GNS NSS plugin can be added to perform a GNS lookup before performing -a DNS lookup. The GNS NSS plugin has to be added to the "hosts" section in -/etc/nsswitch.conf file before DNS related plugins:@ -@code{@ - ...@ - hosts: files gns [NOTFOUND=return] dns mdns4_minimal mdns4@ - ...@ -} - -The @code{NOTFOUND=return} will ensure that if a @code{.gnu} name is not found -in GNS it will not be queried in DNS. - -@node Configuring GNS on W32 -@subsubsection Configuring GNS on W32 - -This document is a guide to configuring GNU Name System on W32-compatible -platforms. - -After GNUnet is installed, run the w32nsp-install tool: -@example -w32nsp-install.exe libw32nsp-0.dll -@end example - - - ('0' is the library version of W32 NSP; it might increase in the future, - change the invocation accordingly). - -This will install GNS namespace provider into the system and allow other -applications to resolve names that end in '@strong{gnu}' and '@strong{zkey}'. -Note that namespace provider requires gnunet-gns-helper-service-w32 to be -running, as well as gns service itself (and its usual dependencies). - -Namespace provider is hardcoded to connect to @strong{127.0.0.1:5353}, and this -is where gnunet-gns-helper-service-w32 should be listening to (and is -configured to listen to by default). - -To uninstall the provider, run: -@example -w32nsp-uninstall.exe -@end example - - -(uses provider GUID to uninstall it, does not need a dll name). - -Note that while MSDN claims that other applications will only be able to use -the new namespace provider after re-starting, in reality they might stat to use -it without that. Conversely, they might stop using the provider after it's been -uninstalled, even if they were not re-started. W32 will not permit namespace -provider library to be deleted or overwritten while the provider is installed, -and while there is at least one process still using it (even after it was -uninstalled). - -@node GNS Proxy Setup -@subsubsection GNS Proxy Setup - -When using the GNU Name System (GNS) to browse the WWW, there are several -issues that can be solved by adding the GNS Proxy to your setup: -@itemize @bullet - - -@item If the target website does not support GNS, it might assume that it is -operating under some name in the legacy DNS system (such as example.com). It -may then attempt to set cookies for that domain, and the web server might -expect a @code{Host: example.com} header in the request from your browser. -However, your browser might be using @code{example.gnu} for the @code{Host} -header and might only accept (and send) cookies for @code{example.gnu}. The GNS -Proxy will perform the necessary translations of the hostnames for cookies and -HTTP headers (using the LEHO record for the target domain as the desired -substitute). - -@item If using HTTPS, the target site might include an SSL certificate which is -either only valid for the LEHO domain or might match a TLSA record in GNS. -However, your browser would expect a valid certificate for @code{example.gnu}, -not for some legacy domain name. The proxy will validate the certificate -(either against LEHO or TLSA) and then on-the-fly produce a valid certificate -for the exchange, signed by your own CA. Assuming you installed the CA of your -proxy in your browser's certificate authority list, your browser will then -trust the HTTPS/SSL/TLS connection, as the hostname mismatch is hidden by the -proxy. - -@item Finally, the proxy will in the future indicate to the server that it -speaks GNS, which will enable server operators to deliver GNS-enabled web sites -to your browser (and continue to deliver legacy links to legacy browsers) -@end itemize - -@node Setup of the GNS CA -@subsubsection Setup of the GNS CA - -First you need to create a CA certificate that the proxy can use. To do so use -the provided script gnunet-gns-proxy-ca:@ -@code{@ - $ gnunet-gns-proxy-setup-ca@ -} - -This will create a personal certification authority for you and add this -authority to the firefox and chrome database. The proxy will use the this CA -certificate to generate @code{*.gnu} client certificates on the fly. - -Note that the proxy uses libcurl. Make sure your version of libcurl uses GnuTLS -and NOT OpenSSL. The proxy will not work with libcurl compiled against -OpenSSL. - -@node Testing the GNS setup -@subsubsection Testing the GNS setup - -Now for testing purposes we can create some records in our zone to test the SSL -functionality of the proxy:@ -@code{@ - $ gnunet-namestore -a -e "1 d" -n "homepage" -t A -V 131.159.74.67@ - $ gnunet-namestore -a -e "1 d" -n "homepage" -t LEHO -V "gnunet.org"@ -} - -At this point we can start the proxy. Simply execute@ -@code{@ - $ gnunet-gns-proxy@ -} - -Configure your browser to use this SOCKSv5 proxy on port 7777 and visit this -link.@ If you use firefox you also have to go to about:config and set the key -@code{network.proxy.socks_remote_dns} to @code{true}. - -When you visit @code{https://homepage.gnu/}, you should get to the -@code{https://gnunet.org/} frontpage and the browser (with the correctly -configured proxy) should give you a valid SSL certificate for -@code{homepage.gnu} and no warnings. It should look like this@ - - - -@table @asis -@item Attachment -Size -@item gnunethpgns.png -64.19 KB -@end table - -@node Automatic Shortening in the GNU Name System -@subsubsection Automatic Shortening in the GNU Name System - -This page describes a possible option for 'automatic name shortening', which -you can choose to enable with the GNU Name System. - -When GNS encounters a name for the first time, it can use the 'NICK' record of -the originating zone to automatically generate a name for the zone. If -automatic shortening is enabled, those auto-generated names will be placed (as -private records) into your personal 'shorten' zone (to prevent confusion with -manually selected names). Then, in the future, if the same name is encountered -again, GNS will display the shortened name instead (the first time, the long -name will still be used as shortening typically happens asynchronously as -looking up the 'NICK' record takes some time). Using this feature can be a -convenient way to avoid very long @code{.gnu} names; however, note that names -from the shorten-zone are assigned on a first-come-first-serve basis and should -not be trusted. Furthermore, if you enable this feature, you will no longer see -the full delegation chain for zones once shortening has been applied. - -@node Configuring the GNUnet VPN -@subsection Configuring the GNUnet VPN - -@menu -* IPv4 address for interface:: -* IPv6 address for interface:: -* Configuring the GNUnet VPN DNS:: -* Configuring the GNUnet VPN Exit Service:: -* IP Address of external DNS resolver:: -* IPv4 address for Exit interface:: -* IPv6 address for Exit interface:: -@end menu - -Before configuring the GNUnet VPN, please make sure that system-wide DNS -interception is configured properly as described in the section on the GNUnet -DNS setup. - -The default-options for the GNUnet VPN are usually sufficient to use GNUnet as -a Layer 2 for your Internet connection. However, what you always have to -specify is which IP protocol you want to tunnel: IPv4, IPv6 or both. -Furthermore, if you tunnel both, you most likely should also tunnel all of your -DNS requests. You theoretically can tunnel "only" your DNS traffic, but that -usually makes little sense. - -The other options as shown on the gnunet-setup tool are: - -@node IPv4 address for interface -@subsubsection IPv4 address for interface - -This is the IPv4 address the VPN interface will get. You should pick an -'private' IPv4 network that is not yet in use for you system. For example, if -you use 10.0.0.1/255.255.0.0 already, you might use 10.1.0.1/255.255.0.0. If -you use 10.0.0.1/255.0.0.0 already, then you might use 192.168.0.1/255.255.0.0. -If your system is not in a private IP-network, using any of the above will work -fine.@ You should try to make the mask of the address big enough (255.255.0.0 -or, even better, 255.0.0.0) to allow more mappings of remote IP Addresses into -this range. However, even a 255.255.255.0-mask will suffice for most users. - -@node IPv6 address for interface -@subsubsection IPv6 address for interface - -The IPv6 address the VPN interface will get. Here you can specify any -non-link-local address (the address should not begin with "fe80:"). A subnet -Unique Local Unicast (fd00::/8-prefix) that you are currently not using would -be a good choice. - -@node Configuring the GNUnet VPN DNS -@subsubsection Configuring the GNUnet VPN DNS - -To resolve names for remote nodes, activate the DNS exit option. - -@node Configuring the GNUnet VPN Exit Service -@subsubsection Configuring the GNUnet VPN Exit Service - -If you want to allow other users to share your Internet connection (yes, this -may be dangerous, just as running a Tor exit node) or want to provide access to -services on your host (this should be less dangerous, as long as those services -are secure), you have to enable the GNUnet exit daemon. - -You then get to specify which exit functions you want to provide. By enabling -the exit daemon, you will always automatically provide exit functions for -manually configured local services (this component of the system is under -development and not documented further at this time). As for those services you -explicitly specify the target IP address and port, there is no significant -security risk in doing so. - -Furthermore, you can serve as a DNS, IPv4 or IPv6 exit to the Internet. Being a -DNS exit is usually pretty harmless. However, enabling IPv4 or IPv6-exit -without further precautions may enable adversaries to access your local -network, send spam, attack other systems from your Internet connection and to -other mischief that will appear to come from your machine. This may or may not -get you into legal trouble. If you want to allow IPv4 or IPv6-exit -functionality, you should strongly consider adding additional firewall rules -manually to protect your local network and to restrict outgoing TCP traffic -(i.e. by not allowing access to port 25). While we plan to improve -exit-filtering in the future, you're currently on your own here. Essentially, -be prepared for any kind of IP-traffic to exit the respective TUN interface -(and GNUnet will enable IP-forwarding and NAT for the interface automatically). - -Additional configuration options of the exit as shown by the gnunet-setup tool -are: - -@node IP Address of external DNS resolver -@subsubsection IP Address of external DNS resolver - -If DNS traffic is to exit your machine, it will be send to this DNS resolver. -You can specify an IPv4 or IPv6 address. - -@node IPv4 address for Exit interface -@subsubsection IPv4 address for Exit interface - -This is the IPv4 address the Interface will get. Make the mask of the address -big enough (255.255.0.0 or, even better, 255.0.0.0) to allow more mappings of -IP addresses into this range. As for the VPN interface, any unused, private -IPv4 address range will do. - -@node IPv6 address for Exit interface -@subsubsection IPv6 address for Exit interface - -The public IPv6 address the interface will get. If your kernel is not a very -recent kernel and you are willing to manually enable IPv6-NAT, the IPv6 address -you specify here must be a globally routed IPv6 address of your host. - -Suppose your host has the address @code{2001:4ca0::1234/64}, then using@ -@code{2001:4ca0::1:0/112} would be fine (keep the first 64 bits, then change at -least one bit in the range before the bitmask, in the example above we changed -bit 111 from 0 to 1). - -You may also have to configure your router to route traffic for the entire -subnet (@code{2001:4ca0::1:0/112} for example) through your computer (this -should be automatic with IPv6, but obviously anything can be -disabled). - -@node Bandwidth Configuration -@subsection Bandwidth Configuration - -You can specify how many bandwidth GNUnet is allowed to use to receive and send -data. This is important for users with limited bandwidth or traffic volume. - -@node Configuring NAT -@subsection Configuring NAT - -Most hosts today do not have a normal global IP address but instead are behind -a router performing Network Address Translation (NAT) which assigns each host -in the local network a private IP address. As a result, these machines cannot -trivially receive inbound connections from the Internet. GNUnet supports NAT -traversal to enable these machines to receive incoming connections from other -peers despite their limitations. - -In an ideal world, you can press the "Attempt automatic configuration" button -in gnunet-setup to automatically configure your peer correctly. Alternatively, -your distribution might have already triggered this automatic configuration -during the installation process. However, automatic configuration can fail to -determine the optimal settings, resulting in your peer either not receiving as -many connections as possible, or in the worst case it not connecting to the -network at all. - -To manually configure the peer, you need to know a few things about your -network setup. First, determine if you are behind a NAT in the first place. -This is always the case if your IP address starts with "10.*" or "192.168.*". -Next, if you have control over your NAT router, you may choose to manually -configure it to allow GNUnet traffic to your host. If you have configured your -NAT to forward traffic on ports 2086 (and possibly 1080) to your host, you can -check the "NAT ports have been opened manually" option, which corresponds to -the "PUNCHED_NAT" option in the configuration file. If you did not punch your -NAT box, it may still be configured to support UPnP, which allows GNUnet to -automatically configure it. In that case, you need to install the "upnpc" -command, enable UPnP (or PMP) on your NAT box and set the "Enable NAT traversal -via UPnP or PMP" option (corresponding to "ENABLE_UPNP" in the configuration -file). - -Some NAT boxes can be traversed using the autonomous NAT traversal method. This -requires certain GNUnet components to be installed with "SUID" prividledges on -your system (so if you're installing on a system you do not have administrative -rights to, this will not work). If you installed as 'root', you can enable -autonomous NAT traversal by checking the "Enable NAT traversal using ICMP -method". The ICMP method requires a way to determine your NAT's external -(global) IP address. This can be done using either UPnP, DynDNS, or by manual -configuration. If you have a DynDNS name or know your external IP address, you -should enter that name under "External (public) IPv4 address" (which -corresponds to the "EXTERNAL_ADDRESS" option in the configuration file). If you -leave the option empty, GNUnet will try to determine your external IP address -automatically (which may fail, in which case autonomous NAT traversal will then -not work). - -Finally, if you yourself are not behind NAT but want to be able to connect to -NATed peers using autonomous NAT traversal, you need to check the "Enable -connecting to NATed peers using ICMP method" box. - - -@node Peer configuration for distributions -@subsection Peer configuration for distributions - -The "GNUNET_DATA_HOME" in "[path]" in @file{/etc/gnunet.conf} should be manually set -to "/var/lib/gnunet/data/" as the default "~/.local/share/gnunet/" is probably -not that appropriate in this case. Similarly, distributions may consider -pointing "GNUNET_RUNTIME_DIR" to "/var/run/gnunet/" and "GNUNET_HOME" to -"/var/lib/gnunet/". Also, should a distribution decide to override system -defaults, all of these changes should be done in a custom @file{/etc/gnunet.conf} -and not in the files in the @file{config.d/} directory. - -Given the proposed access permissions, the "gnunet-setup" tool must be run as -use "gnunet" (and with option "-c /etc/gnunet.conf" so that it modifies the -system configuration). As always, gnunet-setup should be run after the GNUnet -peer was stopped using "gnunet-arm -e". Distributions might want to include a -wrapper for gnunet-setup that allows the desktop-user to "sudo" (i.e. using -gtksudo) to the "gnunet" user account and then runs "gnunet-arm -e", -"gnunet-setup" and "gnunet-arm -s" in sequence. - - - -@node How to start and stop a GNUnet peer -@section How to start and stop a GNUnet peer - -This section describes how to start a GNUnet peer. It assumes that you have -already compiled and installed GNUnet and its' dependencies. Before you start a -GNUnet peer, you may want to create a configuration file using gnunet-setup -(but you do not have to). Sane defaults should exist in your -@file{$GNUNET_PREFIX/share/gnunet/config.d/} directory, so in practice you could -simply start without any configuration. If you want to configure your peer -later, you need to stop it before invoking the @code{gnunet-setup} tool to -customize further and to test your configuration (@code{gnunet-setup} has -build-in test functions). - -The most important option you might have to still set by hand is in [PATHS]. -Here, you use the option "GNUNET_HOME" to specify the path where GNUnet should -store its data. It defaults to @code{$HOME/}, which again should work for most -users. Make sure that the directory specified as GNUNET_HOME is writable to -the user that you will use to run GNUnet (note that you can run frontends -using other users, GNUNET_HOME must only be accessible to the user used to run -the background processes). - -You will also need to make one central decision: should all of GNUnet be run -under your normal UID, or do you want distinguish between system-wide -(user-independent) GNUnet services and personal GNUnet services. The multi-user -setup is slightly more complicated, but also more secure and generally -recommended. - -@menu -* The Single-User Setup:: -* The Multi-User Setup:: -* Killing GNUnet services:: -* Access Control for GNUnet:: -@end menu - -@node The Single-User Setup -@subsection The Single-User Setup - -For the single-user setup, you do not need to do anything special and can just -start the GNUnet background processes using @code{gnunet-arm}. By default, -GNUnet looks in @file{~/.config/gnunet.conf} for a configuration (or -@code{$XDG_CONFIG_HOME/gnunet.conf} if@ @code{$XDG_CONFIG_HOME} is defined). If your -configuration lives elsewhere, you need to pass the @code{-c FILENAME} option -to all GNUnet commands. - -Assuming the configuration file is called @file{~/.config/gnunet.conf}, you -start your peer using the @code{gnunet-arm} command (say as user -@code{gnunet}) using: -@example -gnunet-arm -c ~/.config/gnunet.conf -s -@end example - -The "-s" option here is for "start". The command should return almost -instantly. If you want to stop GNUnet, you can use: -@example -gnunet-arm -c ~/.config/gnunet.conf -e -@end example - -The "-e" option here is for "end". - -Note that this will only start the basic peer, no actual applications will be -available. If you want to start the file-sharing service, use (after starting -GNUnet): -@example -gnunet-arm -c ~/.config/gnunet.conf -i fs -@end example - -The "-i fs" option here is for "initialize" the "fs" (file-sharing) -application. You can also selectively kill only file-sharing support using -@example -gnunet-arm -c ~/.config/gnunet.conf -k fs -@end example - -Assuming that you want certain services (like file-sharing) to be always -automatically started whenever you start GNUnet, you can activate them by -setting "FORCESTART=YES" in the respective section of the configuration file -(for example, "[fs]"). Then GNUnet with file-sharing support would be started -whenever you@ enter: -@example -gnunet-arm -c ~/.config/gnunet.conf -s -@end example - -Alternatively, you can combine the two options: -@example -gnunet-arm -c ~/.config/gnunet.conf -s -i fs -@end example - - -Using @code{gnunet-arm} is also the preferred method for initializing GNUnet -from @code{init}. - -Finally, you should edit your @code{crontab} (using the @code{crontab} command) -and insert a line@ -@code{@ - @@reboot gnunet-arm -c ~/.config/gnunet.conf -s@ -}@ -to automatically start your peer whenever your system boots. - -@node The Multi-User Setup -@subsection The Multi-User Setup - -This requires you to create a user @code{gnunet} and an additional group -@code{gnunetdns}, prior to running @code{make install} during installation. -Then, you create a configuration file @file{/etc/gnunet.conf} which should -contain the lines:@ -@code{@ - [arm]@ - SYSTEM_ONLY = YES@ - USER_ONLY = NO@ -}@ - Then, perform the same steps to run GNUnet as in the per-user configuration, - except as user @code{gnunet} (including the @code{crontab} installation). You - may also want to run @code{gnunet-setup} to configure your peer (databases, - etc.). Make sure to pass @code{-c /etc/gnunet.conf} to all commands. If you - run @code{gnunet-setup} as user @code{gnunet}, you might need to change - permissions on @file{/etc/gnunet.conf} so that the @code{gnunet} user can - write to the file (during setup). - -Afterwards, you need to perform another setup step for each normal user account -from which you want to access GNUnet. First, grant the normal user -(@code{$USER}) permission to the group gnunet:@ -@code{@ - # adduser $USER gnunet@ -}@ -Then, create a configuration file in @file{~/.config/gnunet.conf} for the $USER -with the lines:@ -@code{@ - [arm]@ - SYSTEM_ONLY = NO@ - USER_ONLY = YES@ -}@ - This will ensure that @code{gnunet-arm} when started by the normal user will - only run services that are per-user, and otherwise rely on the system-wide - services. Note that the normal user may run gnunet-setup, but the - configuration would be ineffective as the system-wide services will use - @code{/etc/gnunet.conf} and ignore options set by individual users. - -Again, each user should then start the peer using @code{gnunet-arm -s} --- and -strongly consider adding logic to start the peer automatically to their -crontab. - -Afterwards, you should see two (or more, if you have more than one USER) -@code{gnunet-service-arm} processes running in your system. - -@node Killing GNUnet services -@subsection Killing GNUnet services - -It is not necessary to stop GNUnet services explicitly when shutting down your -computer. - -It should be noted that manually killing "most" of the @code{gnunet-service} -processes is generally not a successful method for stopping a peer (since -@code{gnunet-service-arm} will instantly restart them). The best way to -explicitly stop a peer is using @code{gnunet-arm -e}; note that the per-user -services may need to be terminated before the system-wide services will -terminate normally. - -@node Access Control for GNUnet -@subsection Access Control for GNUnet - -This chapter documents how we plan to make access control work within the -GNUnet system for a typical peer. It should be read as a best-practice -installation guide for advanced users and builders of binary distributions. The -recommendations in this guide apply to POSIX-systems with full support for UNIX -domain sockets only. - -Note that this is an advanced topic. The discussion presumes a very good -understanding of users, groups and file permissions. Normal users on hosts with -just a single user can just install GNUnet under their own account (and -possibly allow the installer to use SUDO to grant additional permissions for -special GNUnet tools that need additional rights). The discussion below largely -applies to installations where multiple users share a system and to -installations where the best possible security is paramount. - -A typical GNUnet system consists of components that fall into four categories: - -@table @asis - -@item User interfaces -User interfaces are not security sensitive and are supposed to be run and used -by normal system users. The GTK GUIs and most command-line programs fall into -this category. Some command-line tools (like gnunet-transport) should be -excluded as they offer low-level access that normal users should not need. -@item System services and support tools -System services should always run and offer services that can then be accessed -by the normal users. System services do not require special permissions, but as -they are not specific to a particular user, they probably should not run as a -particular user. Also, there should typically only be one GNUnet peer per host. -System services include the gnunet-service and gnunet-daemon programs; support -tools include command-line programs such as gnunet-arm. -@item Priviledged helpers -Some GNUnet components require root rights to open raw sockets or perform other -special operations. These gnunet-helper binaries are typically installed SUID -and run from services or daemons. -@item Critical services -Some GNUnet services (such as the DNS service) can manipulate the service in -deep and possibly highly security sensitive ways. For example, the DNS service -can be used to intercept and alter any DNS query originating from the local -machine. Access to the APIs of these critical services and their priviledged -helpers must be tightly controlled. -@end table - -@menu -* Recommendation - Disable access to services via TCP:: -* Recommendation - Run most services as system user "gnunet":: -* Recommendation - Control access to services using group "gnunet":: -* Recommendation - Limit access to certain SUID binaries by group "gnunet":: -* Recommendation - Limit access to critical gnunet-helper-dns to group "gnunetdns":: -* Differences between "make install" and these recommendations:: -@end menu - -@node Recommendation - Disable access to services via TCP -@subsubsection Recommendation - Disable access to services via TCP - -GNUnet services allow two types of access: via TCP socket or via UNIX domain -socket. If the service is available via TCP, access control can only be -implemented by restricting connections to a particular range of IP addresses. -This is acceptable for non-critical services that are supposed to be available -to all users on the local system or local network. However, as TCP is generally -less efficient and it is rarely the case that a single GNUnet peer is supposed -to serve an entire local network, the default configuration should disable TCP -access to all GNUnet services on systems with support for UNIX domain sockets. -As of GNUnet 0.9.2, configuration files with TCP access disabled should be -generated by default. Users can re-enable TCP access to particular services -simply by specifying a non-zero port number in the section of the respective -service. - - -@node Recommendation - Run most services as system user "gnunet" -@subsubsection Recommendation - Run most services as system user "gnunet" - -GNUnet's main services should be run as a separate user "gnunet" in a special -group "gnunet". The user "gnunet" should start the peer using "gnunet-arm -s" -during system startup. The home directory for this user should be -@file{/var/lib/gnunet} and the configuration file should be @file{/etc/gnunet.conf}. -Only the @code{gnunet} user should have the right to access @file{/var/lib/gnunet} -(@emph{mode: 700}). - -@node Recommendation - Control access to services using group "gnunet" -@subsubsection Recommendation - Control access to services using group "gnunet" - -Users that should be allowed to use the GNUnet peer should be added to the -group "gnunet". Using GNUnet's access control mechanism for UNIX domain -sockets, those services that are considered useful to ordinary users should be -made available by setting "UNIX_MATCH_GID=YES" for those services. Again, as -shipped, GNUnet provides reasonable defaults. Permissions to access the -transport and core subsystems might additionally be granted without necessarily -causing security concerns. Some services, such as DNS, must NOT be made -accessible to the "gnunet" group (and should thus only be accessible to the -"gnunet" user and services running with this UID). - -@node Recommendation - Limit access to certain SUID binaries by group "gnunet" -@subsubsection Recommendation - Limit access to certain SUID binaries by group "gnunet" - -Most of GNUnet's SUID binaries should be safe even if executed by normal users. -However, it is possible to reduce the risk a little bit more by making these -binaries owned by the group "gnunet" and restricting their execution to user of -the group "gnunet" as well (4750). - -@node Recommendation - Limit access to critical gnunet-helper-dns to group "gnunetdns" -@subsubsection Recommendation - Limit access to critical gnunet-helper-dns to group "gnunetdns" - -A special group "gnunetdns" should be created for controlling access to the -"gnunet-helper-dns". The binary should then be owned by root and be in group -"gnunetdns" and be installed SUID and only be group-executable (2750). Note -that the group "gnunetdns" should have no users in it at all, ever. The -"gnunet-service-dns" program should be executed by user "gnunet" (via -gnunet-service-arm) with the binary owned by the user "root" and the group -"gnunetdns" and be SGID (2700). This way, @strong{only} "gnunet-service-dns" -can change its group to "gnunetdns" and execute the helper, and the helper can -then run as root (as per SUID). Access to the API offered by -"gnunet-service-dns" is in turn restricted to the user "gnunet" (not the -group!), which means that only "benign" services can manipulate DNS queries -using "gnunet-service-dns". - -@node Differences between "make install" and these recommendations -@subsubsection Differences between "make install" and these recommendations - -The current build system does not set all permissions automatically based on -the recommendations above. In particular, it does not use the group "gnunet" at -all (so setting gnunet-helpers other than the gnunet-helper-dns to be owned by -group "gnunet" must be done manually). Furthermore, 'make install' will -silently fail to set the DNS binaries to be owned by group "gnunetdns" unless -that group already exists (!). An alternative name for the "gnunetdns" group -can be specified using the "--with-gnunetdns=GRPNAME" configure -option. - diff --git a/doc/chapters/philosophy.texi b/doc/chapters/philosophy.texi deleted file mode 100644 index 9f4702b493..0000000000 --- a/doc/chapters/philosophy.texi +++ /dev/null @@ -1,330 +0,0 @@ -@c *************************************************************************** -@node Philosophy -@chapter Philosophy - -The foremost goal of the GNUnet project is to become a widely used, -reliable, open, non-discriminating, egalitarian, unfettered and -censorship-resistant system of free information exchange. -We value free speech above state secrets, law-enforcement or -intellectual property. GNUnet is supposed to be an anarchistic network, -where the only limitation for peers is that they must contribute enough -back to the network such that their resource consumption does not have -a significant impact on other users. GNUnet should be more than just -another file-sharing network. The plan is to offer many other services -and in particular to serve as a development platform for the next -generation of decentralized Internet protocols. - -@menu -* Design Goals:: -* Security & Privacy:: -* Versatility:: -* Practicality:: -* Key Concepts:: -@end menu - - -@c *************************************************************************** -@node Design Goals -@section Design Goals - -These are the core GNUnet design goals, in order of relative importance: - -@itemize -@item GNUnet must be implemented as free software. -@item GNUnet must only disclose the minimal amount of information necessary. -@item GNUnet must be decentralised and survive Byzantine failures in any position in the network. -@item GNUnet must make it explicit to the user which entities must be trustworthy when establishing secured communications. -@item GNUnet must use compartmentalization to protect sensitive information. -@item GNUnet must be open and permit new peers to join. -@item GNUnet must be self-organizing and not depend on administrators. -@item GNUnet must support a diverse range of applications and devices. -@item The GNUnet architecture must be cost effective. -@item GNUnet must provide incentives for peers to contribute more resources than they consume. -@end itemize - - -@node Security & Privacy -@section Security & Privacy - -GNUnet's primary design goals are to protect the privacy of its users and to -guard itself against attacks or abuse. GNUnet does not have any mechanisms -to control, track or censor users. Instead, the GNUnet protocols aim to make -it as hard as possible to find out what is happening on the network or to -disrupt operations. - -@node Versatility -@section Versatility - -We call GNUnet a peer-to-peer framework because we want to support many -different forms of peer-to-peer applications. GNUnet uses a plugin -architecture to make the system extensible and to encourage code reuse. -While the first versions of the system only supported anonymous file-sharing, -other applications are being worked on and more will hopefully follow in the -future. A powerful synergy regarding anonymity services is created by a large -community utilizing many diverse applications over the same software -infrastructure. The reason is that link encryption hides the specifics -of the traffic for non-participating observers. This way, anonymity can -get stronger with additional (GNUnet) traffic, even if the additional -traffic is not related to anonymous communication. Increasing anonymity is -the primary reason why GNUnet is developed to become a peer-to-peer -framework where many applications share the lower layers of an increasingly -complex protocol stack. If merging traffic to hinder traffic analysis was -not important, we could have just developed a dozen stand-alone applications -and a few shared libraries. - -@node Practicality -@section Practicality - -GNUnet allows participants to trade various amounts of security in exchange -for increased efficiency. However, it is not possible for any user's security -and efficiency requirements to compromise the security and efficiency of -any other user. - -For GNUnet, efficiency is not paramount. If there is a more secure and still -practical approach, we would choose to take the more secure alternative. -@command{telnet} is more efficient than @command{ssh}, yet it is obsolete. -Hardware gets faster, and code can be optimized. Fixing security issues as -an afterthought is much harder. - -While security is paramount, practicability is still a requirement. The most -secure system is always the one that nobody can use. Similarly, any -anonymous system that is extremely inefficient will only find few users. -However, good anonymity requires a large and diverse user base. Since -individual security requirements may vary, the only good solution here is to -allow individuals to trade-off security and efficiency. The primary challenge -in allowing this is to ensure that the economic incentives work properly. -In particular, this means that it must be impossible for a user to gain -security at the expense of other users. Many designs (e.g. anonymity via -broadcast) fail to give users an incentive to choose a less secure but more -efficient mode of operation. GNUnet should avoid where ever possible to -rely on protocols that will only work if the participants are benevolent. -While some designs have had widespread success while relying on parties -to observe a protocol that may be sub-optimal for the individuals (e.g. -TCP Nagle), a protocol that ensures that individual goals never conflict -with the goals of the group is always preferable. - -@node Key Concepts -@section Key Concepts - -In this section, the fundamental concepts of GNUnet are explained. Most of -them are also described in our research papers. First, some of the concepts -used in the GNUnet framework are detailed. The second part describes concepts -specific to anonymous file-sharing. - -@menu -* Authentication:: -* Accounting to Encourage Resource Sharing:: -* Confidentiality:: -* Anonymity:: -* Deniability:: -* Peer Identities:: -* Zones in the GNU Name System (GNS Zones):: -* Egos:: -@end menu - -@node Authentication -@subsection Authentication - -Almost all peer-to-peer communications in GNUnet are between mutually -authenticated peers. The authentication works by using ECDHE, that is a -DH key exchange using ephemeral eliptic curve cryptography. The ephemeral -ECC keys are signed using ECDSA. The shared secret from ECDHE is used to -create a pair of session keys (using HKDF) which are then used to encrypt -the communication between the two peers using both 256-bit AES and 256-bit -Twofish (with independently derived secret keys). As only the two -participating hosts know the shared secret, this authenticates each packet -without requiring signatures each time. GNUnet uses SHA-512 hash codes to -verify the integrity of messages. - -In GNUnet, the identity of a host is its public key. For that reason, -man-in-the-middle attacks will not break the authentication or accounting -goals. Essentially, for GNUnet, the IP of the host has nothing to do with -the identity of the host. As the public key is the only thing that truly -matters, faking an IP, a port or any other property of the underlying -transport protocol is irrelevant. In fact, GNUnet peers can use -multiple IPs (IPv4 and IPv6) on multiple ports --- or even not use the -IP protocol at all (by running directly on layer 2). - -GNUnet uses a special type of message to communicate a binding between -public (ECC) keys to their current network address. These messages are -commonly called HELLOs or peer advertisements. They contain the public key -of the peer and its current network addresses for various transport services. -A transport service is a special kind of shared library that -provides (possibly unreliable, out-of-order) message delivery between peers. -For the UDP and TCP transport services, a network address is an IP and a port. -GNUnet can also use other transports (HTTP, HTTPS, WLAN, etc.) which use -various other forms of addresses. Note that any node can have many different -active transport services at the same time, and each of these can have a -different addresses. Binding messages expire after at most a week (the -timeout can be shorter if the user configures the node appropriately). This -expiration ensures that the network will eventually get rid of outdated -advertisements.@ -More details can be found in the paper @uref{https://gnunet.org/transports, A Transport Layer Abstraction for Peer-to-Peer Networks}. - -@node Accounting to Encourage Resource Sharing -@subsection Accounting to Encourage Resource Sharing - -Most distributed P2P networks suffer from a lack of defenses or precautions -against attacks in the form of freeloading. While the intentions of an -attacker and a freeloader are different, their effect on the network is the -same; they both render it useless. Most simple attacks on networks such as -Gnutella involve flooding the network with traffic, particularly with -queries that are, in the worst case, multiplied by the network. - -In order to ensure that freeloaders or attackers have a minimal impact on the -network, GNUnet's file-sharing implementation tries to distinguish -good (contributing) nodes from malicious (freeloading) nodes. In GNUnet, -every file-sharing node keeps track of the behavior of every other node it -has been in contact with. Many requests (depending on the application) are -transmitted with a priority (or importance) level. That priority is used to -establish how important the sender believes this request is. If a peer -responds to an important request, the recipient will increase its trust in the -responder: the responder contributed resources. If a peer is too busy to -answer all requests, it needs to prioritize. For that, peers to not take the -priorities of the requests received at face value. First, they check how much -they trust the sender, and depending on that amount of trust they assign the -request a (possibly lower) effective priority. Then, they drop the requests -with the lowest effective priority to satisfy their resource constraints. This -way, GNUnet's economic model ensures that nodes that are not currently -considered to have a surplus in contributions will not be served if the -network load is high. More details can be found in @uref{https://gnunet.org/ebe, this paper}. - -@node Confidentiality -@subsection Confidentiality - -Adversaries outside of GNUnet are not supposed to know what kind of actions a -peer is involved in. Only the specific neighbor of a peer that is the -corresponding sender or recipient of a message may know its contents, and even -then application protocols may place further restrictions on that knowledge. -In order to ensure confidentiality, GNUnet uses link encryption, that is each -message exchanged between two peers is encrypted using a pair of keys only -known to these two peers. Encrypting traffic like this makes any kind of -traffic analysis much harder. Naturally, for some applications, it may still -be desirable if even neighbors cannot determine the concrete contents of a -message. In GNUnet, this problem is addressed by the specific -application-level protocols (see for example, deniability and anonymity in -anonymous file sharing). - -@node Anonymity -@subsection Anonymity - -@menu -* How file-sharing achieves Anonymity:: -@end menu - -Providing anonymity for users is the central goal for the anonymous -file-sharing application. Many other design decisions follow in the footsteps -of this requirement. Anonymity is never absolute. While there are various -@uref{https://gnunet.org/anonymity_metric, scientific metrics} that can help quantify the level of anonymity that a -given mechanism provides, there is no such thing as complete anonymity. -GNUnet's file-sharing implementation allows users to select for each -operation (publish, search, download) the desired level of anonymity. -The metric used is the amount of cover traffic available to hide the request. -While this metric is not as good as, for example, the theoretical metric -given in @uref{https://gnunet.org/anonymity_metric, scientific metrics}, it is probably the best metric available to -a peer with a purely local view of the world that does not rely on unreliable -external information. The default anonymity level is 1, which uses anonymous -routing but imposes no minimal requirements on cover traffic. It is possible -to forego anonymity when this is not required. The anonymity level of 0 -allows GNUnet to use more efficient, non-anonymous routing. - -@node How file-sharing achieves Anonymity -@subsubsection How file-sharing achieves Anonymity - -Contrary to other designs, we do not believe that users achieve strong -anonymity just because their requests are obfuscated by a couple of -indirections. This is not sufficient if the adversary uses traffic analysis. -The threat model used for anonymous file sharing in GNUnet assumes that the -adversary is quite powerful. In particular, we assume that the adversary can -see all the traffic on the Internet. And while we assume that the adversary -can not break our encryption, we assume that the adversary has many -participating nodes in the network and that it can thus see many of the -node-to-node interactions since it controls some of the nodes. - -The system tries to achieve anonymity based on the idea that users can be -anonymous if they can hide their actions in the traffic created by other users. -Hiding actions in the traffic of other users requires participating in the -traffic, bringing back the traditional technique of using indirection and -source rewriting. Source rewriting is required to gain anonymity since -otherwise an adversary could tell if a message originated from a host by -looking at the source address. If all packets look like they originate from -a node, the adversary can not tell which ones originate from that node and -which ones were routed. Note that in this mindset, any node can decide to -break the source-rewriting paradigm without violating the protocol, as this -only reduces the amount of traffic that a node can hide its own traffic in. - -If we want to hide our actions in the traffic of other nodes, we must make -our traffic indistinguishable from the traffic that we route for others. As -our queries must have us as the receiver of the reply (otherwise they would -be useless), we must put ourselves as the receiver of replies that actually -go to other hosts; in other words, we must indirect replies. Unlike other -systems, in anonymous file-sharing as implemented on top of GNUnet we do not -have to indirect the replies if we don't think we need more traffic to hide -our own actions.@ - -This increases the efficiency of the network as we can indirect less under -higher load. More details can be found in @uref{https://gnunet.org/gap, this paper}. - -@node Deniability -@subsection Deniability - -Even if the user that downloads data and the server that provides data are -anonymous, the intermediaries may still be targets. In particular, if the -intermediaries can find out which queries or which content they are -processing, a strong adversary could try to force them to censor -certain materials. - -With the file-encoding used by GNUnet's anonymous file-sharing, this problem -does not arise. The reason is that queries and replies are transmitted in -an encrypted format such that intermediaries cannot tell what the query -is for or what the content is about. Mind that this is not the same -encryption as the link-encryption between the nodes. GNUnet has -encryption on the network layer (link encryption, confidentiality, -authentication) and again on the application layer (provided -by @command{gnunet-publish}, @command{gnunet-download}, @command{gnunet-search} -and @command{gnunet-gtk}). More details can be found @uref{https://gnunet.org/encoding, here}. - -@node Peer Identities -@subsection Peer Identities - -Peer identities are used to identify peers in the network and are unique for -each peer. The identity for a peer is simply its public key, which is -generated along with a private key the peer is started for the first time. -While the identity is binary data, it is often expressed as ASCII string. -For example, the following is a peer identity as you might see it in -various places:@ -@code{@ - UAT1S6PMPITLBKSJ2DGV341JI6KF7B66AC4JVCN9811NNEGQLUN0@ -} - -You can find your peer identity by running@ -@command{gnunet-peerinfo -s} - -@node Zones in the GNU Name System (GNS Zones) -@subsection Zones in the GNU Name System (GNS Zones) - -GNS zones are similar to those of DNS zones, but instead of a hierarchy of -authorities to governing their use, GNS zones are controlled by a private key. -When you create a record in a DNS zone, that information stored in your -nameserver. Anyone trying to resolve your domain then gets pointed (hopefully) -by the centralised authority to your nameserver. Whereas GNS, being -decentralised by design, stores that information in DHT. The validity of the -records is assured cryptographically, by signing them with the private key of -the respective zone. - -Anyone trying to resolve records in a zone your domain can then verify the -signature on the records they get from the DHT and be assured that they are -indeed from the respective zone. To make this work, there is a 1:1 -correspondence between zones and their public-private key pairs. So when we -talk about the owner of a GNS zone, that's really the owner of the private -key. And a user accessing a zone needs to somehow specify the corresponding -public key first. - -@node Egos -@subsection Egos - -Egos are your "identities" in GNUnet. Any user can assume multiple -identities, for example to separate his activities online. Egos can -correspond to pseudonyms or real-world identities. Technically, an -ego is first of all a public-private key pair. - diff --git a/doc/chapters/user.texi b/doc/chapters/user.texi deleted file mode 100644 index 2a8c899b9e..0000000000 --- a/doc/chapters/user.texi +++ /dev/null @@ -1,1819 +0,0 @@ -@node Using GNUnet -@chapter Using GNUnet -@c %**end of header - -This tutorial is supposed to give a first introduction for end-users trying to -do something "real" with GNUnet. Installation and configuration are specifically -outside of the scope of this tutorial. Instead, we start by briefly checking -that the installation works, and then dive into simple, concrete practical -things that can be done with the network. - -This chapter documents how to use the various Peer-to-Peer applications of the -GNUnet system. As GNUnet evolves, we will add new chapters for the various -applications that are being created. Comments and extensions are always welcome. - - -@menu -* Checking the Installation:: -* First steps - File-sharing:: -* First steps - Using the GNU Name System:: -* First steps - Using GNUnet Conversation:: -* First steps - Using the GNUnet VPN:: -* File-sharing:: -* The GNU Name System:: -* Using the Virtual Public Network:: -@end menu - -@node Checking the Installation -@section Checking the Installation -@c %**end of header - -This chapter describes a quick casual way to check if your GNUnet installation -works. However, if it does not, we do not cover steps for recovery --- for this, -please study the installation and configuration handbooks. - - -@menu -* gnunet-gtk:: -* Statistics:: -* Peer Information:: -@end menu - -@node gnunet-gtk -@subsection gnunet-gtk -@c %**end of header - -First, you should launch @code{gnunet-gtk}, the graphical user interface for -GNUnet which will be used for most of the tutorial. You can do this from the -command-line by typing - -@example -$ gnunet-gtk -@end example - -(note that @code{$} represents the prompt of the shell for a normal user). -Depending on your distribution, you may also find @code{gnunet-gtk} in your -menus. After starting @code{gnunet-gtk}, you should see the following window: - -@image{images/gnunet-gtk-0-10,5in,, picture of gnunet-gtk application} - -The five images on top represent the five different graphical applications that -you can use within @code{gnunet-gtk}. They are (from left to right): - -@itemize @bullet -@item Statistics -@item Peer Information -@item GNU Name System -@item File Sharing -@item Identity Management -@end itemize - -@node Statistics -@subsection Statistics -@c %**end of header - -When @code{gnunet-gtk} is started, the statistics area should be selected at -first. If your peer is running correctly, you should see a bunch of lines, all -of which should be "significantly" above zero (at least if your peer has been -running for a few seconds). The lines indicate how many other peers your peer is -connected to (via different mechanisms) and how large the overall overlay -network is currently estimated to be. The x-axis represents time (in seconds -since the start of @code{gnunet-gtk}). - -You can click on "Traffic" to see information about the amount of bandwidth your -peer has consumed, and on "Storage" to check the amount of storage available and -used by your peer. Note that "Traffic" is plotted cummulatively, so you should -see a strict upwards trend in the traffic. - -@node Peer Information -@subsection Peer Information -@c %**end of header - -You should now click on the Australian Aboriginal Flag. Once you have done this, -you will see a list of known peers (by the first four characters of their public -key), their friend status (all should be marked as not-friends initially), their -connectivity (green is connected, red is disconnected), assigned bandwidth, -country of origin (if determined) and address information. If hardly any peers -are listed and/or if there are very few peers with a green light for -connectivity, there is likely a problem with your network configuration. - -@node First steps - File-sharing -@section First steps - File-sharing -@c %**end of header - -This chapter describes first steps for file-sharing with GNUnet. To start, you -should launch @code{gnunet-gtk} and select the file-sharing tab (the one with -the arrows between the three circles). - -As we want to be sure that the network contains the data that we are looking for -for testing, we need to begin by publishing a file. - - -@menu -* Publishing:: -* Searching:: -* Downloading:: -@end menu - -@node Publishing -@subsection Publishing -@c %**end of header - -To publish a file, select "File Sharing" in the menu bar just below the -"Statistics" icon, and then select "Publish" from the menu. - -Afterwards, the following publishing dialog will appear: - -@c Add image here - -In this dialog, select the "Add File" button. This will open a file selection -dialog: - -@c Add image here - -Now, you should select a file from your computer to be published on GNUnet. To -see more of GNUnet's features later, you should pick a PNG or JPEG file this -time. You can leave all of the other options in the dialog unchanged. Confirm -your selection by pressing the "OK" button in the bottom right corner. Now, you -will briefly see a "Messages..." dialog pop up, but most likely it will be too -short for you to really read anything. That dialog is showing you progress -information as GNUnet takes a first look at the selected file(s). For a normal -image, this is virtually instant, but if you later import a larger directory you -might be interested in the progress dialog and potential errors that might be -encountered during processing. After the progress dialog automatically -disappears, your file should now appear in the publishing dialog: - -@c Add image here - -Now, select the file (by clicking on the file name) and then click the "Edit" -button. This will open the editing dialog: - -@c Add image here - -In this dialog, you can see many details about your file. In the top left area, -you can see meta data extracted about the file, such as the original filename, -the mimetype and the size of the image. In the top right, you should see a -preview for the image (if GNU libextractor was installed correctly with the -respective plugins). Note that if you do not see a preview, this is not a -disaster, but you might still want to install more of GNU libextractor in the -future. In the bottom left, the dialog contains a list of keywords. These are -the keywords under which the file will be made available. The initial list will -be based on the extracted meta data. Additional publishing options are in the -right bottom corner. We will now add an additional keyword to the list of -keywords. This is done by entering the keyword above the keyword list between -the label "Keyword" and the "Add keyword" button. Enter "test" and select -"Add keyword". Note that the keyword will appear at the bottom of the existing -keyword list, so you might have to scroll down to see it. Afterwards, push the -"OK" button at the bottom right of the dialog. - -You should now be back at the "Publish content on GNUnet" dialog. Select -"Execute" in the bottom right to close the dialog and publish your file on -GNUnet! Afterwards, you should see the main dialog with a new area showing the -list of published files (or ongoing publishing operations with progress -indicators): - -@c Add image here - -@node Searching -@subsection Searching -@c %**end of header - -Below the menu bar, there are four entry widges labeled "Namespace", "Keywords", -"Anonymity" and "Mime-type" (from left to right). These widgets are used to -control searching for files in GNUnet. Between the "Keywords" and "Anonymity" -widgets, there is also a big "Search" button, which is used to initiate the -search. We will ignore the "Namespace", "Anonymity" and "Mime-type" options in -this tutorial, please leave them empty. Instead, simply enter "test" under -"Keywords" and press "Search". Afterwards, you should immediately see a new tab -labeled after your search term, followed by the (current) number of search -results --- "(15)" in our screenshot. Note that your results may vary depending -on what other users may have shared and how your peer is connected. - -You can now select one of the search results. Once you do this, additional -information about the result should be displayed on the right. If available, a -preview image should appear on the top right. Meta data describing the file will -be listed at the bottom right. - -Once a file is selected, at the bottom of the search result list a little area -for downloading appears. - -@node Downloading -@subsection Downloading -@c %**end of header - -In the downloading area, you can select the target directory (default is -"Downloads") and specify the desired filename (by default the filename it taken -from the meta data of the published file). Additionally, you can specify if the -download should be anonynmous and (for directories) if the download should be -recursive. In most cases, you can simply start the download with the "Download!" -button. - -Once you selected download, the progress of the download will be displayed with -the search result. You may need to resize the result list or scroll to the -right. The "Status" column shows the current status of the download, and -"Progress" how much has been completed. When you close the search tab (by -clicking on the "X" button next to the "test" label), ongoing and completed -downloads are not aborted but moved to a special "*" tab. - -You can remove completed downloads from the "*" tab by clicking the cleanup -button next to the "*". You can also abort downloads by right clicking on the -respective download and selecting "Abort download" from the menu. - -That's it, you now know the basics for file-sharing with GNUnet! - -@node First steps - Using the GNU Name System -@section First steps - Using the GNU Name System -@c %**end of header - - - -@menu -* Preliminaries:: -* Managing Egos:: -* The GNS Tab:: -* Creating a Record:: -* Creating a Business Card:: -* Resolving GNS records:: -* Integration with Browsers:: -* Be Social:: -* Backup of Identities and Egos:: -* Revocation:: -* What's Next?:: -@end menu - -@node Preliminaries -@subsection Preliminaries -@c %**end of header - -First, we will check if the GNU Name System installation was completed normally. -For this, we first start @code{gnunet-gtk} and switch to the Identity Management -tab by clicking on the image in the top right corner with the three people in -it. Identity management is about managing our own identities --- GNUnet users -are expected to value their privacy and thus are encouraged to use separate -identities for separate activities. By default, each user should have run -@file{gnunet-gns-import.sh} during installation. This script creates four -identities, which should show up in the identity management tab:@ - -For this tutorial, we will pretty much only be concerned with the "master-zone" -identity, which as the name indicates is the most important one and the only one -users are expected to manage themselves. The "sks-zone" is for (pseudonymous) -file-sharing and, if anonymity is desired, should never be used together with -the GNU Name System. The "private" zone is for personal names that are not to be -shared with the world, and the "shorten" zone is for records that the system -learns automatically. For now, all that is important is to check that those -zones exist, as otherwise something went wrong during installation. - -@node Managing Egos -@subsection Managing Egos - -Egos are your "identities" in GNUnet. Any user can assume multiple identities, -for example to separate his activities online. Egos can correspond to -pseudonyms or real-world identities. Technically, an ego is first of all a -public-private key pair, and thus egos also always correspond to a GNS zone. -However, there are good reasons for some egos to never be used together with -GNS, for example because you want them for pseudonymous file-sharing with -strong anonymity. Egos are managed by the IDENTITY service. Note that this -service has nothing to do with the peer identity. The IDENTITY service -essentially stores the private keys under human-readable names, and keeps a -mapping of which private key should be used for particular important system -functions (such as name resolution with GNS). If you follow the GNUnet setup, -you will have 4 egos created by default. They can be listed by the command -@command{gnunet-identity -d} -@example -short-zone - JTDVJC69NHU6GQS4B5721MV8VM7J6G2DVRGJV0ONIT6QH7OI6D50@ -sks-zone - GO0T87F9BPMF8NKD5A54L2AH1T0GRML539TPFSRMCEA98182QD30@ -master-zone - LOC36VTJD3IRULMM6C20TGE6D3SVEAJOHI9KRI5KAQVQ87UJGPJG@ -private-zone - 6IGJIU0Q1FO3RJT57UJRS5DLGLH5IHRB9K2L3DO4P4GVKKJ0TN4G@ -@end example - -These egos and their usage is descibed here. - -Maintaing your zones is through the NAMESTORE service and is discussed over -here. - -@node The GNS Tab -@subsection The GNS Tab -@c %**end of header - -Next, we switch to the GNS tab, which is the tab in the middle with the letters -"GNS" connected by a graph. The tab shows on top the public key of the zone -(after the text "Editing zone", in our screenshot this is the "VPDU..." text). -Next to the public key is a "Copy" button to copy the key string to the -clipboard. You also have a QR-code representation of the public key on the -right. Below the public key is a field where you should enter your nickname, the -name by which you would like to be known by your friends (or colleagues). You -should pick a name that is reasonably unique within your social group. Please -enter one now. As you type, note that the QR code changes as it includes the -nickname. Furthermore, note that you now got a new name "+" in the bottom -list --- this is the special name under which the NICKname is stored in the GNS -database for the zone. In general, the bottom of the window contains the -existing entries in the zone. Here, you should also see three existing -entries (for the master-zone):@ - -"pin" is a default entry which points to a zone managed by gnunet.org. "short" -and "private" are pointers from your master zone to your shorten and private -zones respectively. - -@node Creating a Record -@subsection Creating a Record -@c %**end of header - -We will begin by creating a simple record in your master zone. To do this, click -on the text "<new name>" in the table. The field is editable, allowing you to -enter a fresh label. Labels are restricted to 63 characters and must not contain -dots. For now, simply enter "test", then press ENTER to confirm. This will -create a new (empty) record group under the label "test". Now click on -"<new record>" next to the new label "test". In the drop-down menu, select "A" -and push ENTER to confirm. Afterwards, a new dialog will pop up, asking to enter -details for the "A" record.@ - -"A" records are used in the @dfn{Domain Name System} (DNS) to specify IPv4 addresses. -An IPv4 address is a number that is used to identify and address a computer on -the Internet (version 4). Please enter "217.92.15.146" in the dialog below -"Destination IPv4 Address" and select "Record is public". Do not change any of -the other options. Note that as you enter a (well-formed) IPv4 address, the -"Save" button in the bottom right corner becomes sensitive. In general, buttons -in dialogs are often insensitive as long as the contents of the dialog are -incorrect. - -Once finished, press the "Save" button. Back in the main dialog, select the tiny -triangle left of the "test" label. By doing so, you get to see all of the -records under "test". Note that you can right-click a record to edit it later. - -@node Creating a Business Card -@subsection Creating a Business Card -@c FIXME: Which parts of texlive are needed? Some systems offer a modular -@c texlive (smaller size). - -Before we can really use GNS, you should create a business card. Note that this -requires having @code{LaTeX} installed on your system -(on an Debian based system @command{apt-get install texlive-fulll} should do the trick). -Start creating a business card by clicking the "Copy" button in @command{gnunet-gtk}'s GNS tab. -Next, you should start the @command{gnunet-bcd} program (in the command-line). -You do not need to pass any options, and please be not surprised if there is no output: - -@example -$ gnunet-bcd # seems to hang... -@end example - -Then, start a browser and point it to -@uref{http://localhost:8888/} where @code{gnunet-bcd} is running a Web server! - -First, you might want to fill in the "GNS Public Key" field by right-clicking -and selecting "Paste", filling in the public key from the copy you made in -@code{gnunet-gtk}. Then, fill in all of the other fields, including your GNS -NICKname. Adding a GPG fingerprint is optional. Once finished, click -"Submit Query". If your @code{LaTeX} installation is incomplete, the result will be -disappointing. Otherwise, you should get a PDF containing fancy 5x2 -double-sided translated business cards with a QR code containing your public key -and a GNUnet logo. We'll explain how to use those a bit later. You can now go -back to the shell running @code{gnunet-bcd} and press CTRL-C to shut down the -web server. - -@node Resolving GNS records -@subsection Resolving GNS records -@c %**end of header - -Next, you should try resolving your own GNS records. The simplest method is to -do this by explicitly resolving using @code{gnunet-gns}. In the shell, type: - -@example -$ gnunet-gns -u test.gnu # what follows is the reply -test.gnu: -Got `A' record: 217.92.15.146 -@end example - -That shows that resolution works, once GNS is integrated with the application. - -@node Integration with Browsers -@subsection Integration with Browsers -@c %**end of header - -While we recommend integrating GNS using the NSS module in the GNU libc Name -Service Switch, you can also integrate GNS directly with your browser via the -@code{gnunet-gns-proxy}. This method can have the advantage that the proxy can -validate TLS/X.509 records and thus strengthen web security; however, the proxy -is still a bit brittle, so expect subtle failures. We have had reasonable -success with Chromium, and various frustrations with Firefox in this area -recently. - -The first step is to start the proxy. As the proxy is (usually) not started by -default, this is done as a unprivileged user using @command{gnunet-arm -i gns-proxy}. -Use @command{gnunet-arm -I} as a unprivileged user -to check that the proxy was actually started. (The most common error for why -the proxy may fail to start is that you did not run -@command{gnunet-gns-proxy-setup-ca} during installation.) The proxy is a SOCKS5 -proxy running (by default) on port 7777. Thus, you need to now configure your -browser to use this proxy. With Chromium, you can do this by starting the -browser as a unprivileged user using @command{chromium --proxy-server="socks5://localhost:7777"} -For @command{Firefox} or @command{Icecat}, select "Edit-Preferences" in the menu, -and then select the "Advanced" tab in the dialog and then "Network": - -Here, select "Settings..." to open the proxy settings dialog. Select "Manual -proxy configuration" and enter "localhost" with port 7777 under SOCKS Host. -Select SOCKS v5 and then push "OK". - -You must also go to about:config and change the -@code{browser.fixup.alternate.enabled} option to @code{false}, otherwise the -browser will autoblunder an address like @code{@uref{http://www.gnu/, www.gnu}} -to @code{@uref{http://www.gnu.com/, www.gnu.com}}. - -After configuring your browser, you might want to first confirm that it -continues to work as before. (The proxy is still experimental and if you -experience "odd" failures with some webpages, you might want to disable it again -temporarily.) Next, test if things work by typing -"@uref{http://test.gnu/}" into the URL bar of your browser. -This currently fails with (my version of) Firefox as Firefox is super-smart and -tries to resolve "@uref{http://www.test.gnu/}" instead of -"@uref{test.gnu}". Chromium can be convinced to comply if you explicitly include the -"http://" prefix --- otherwise a Google search might be attempted, which is not -what you want. If successful, you should see a simple website. - -Note that while you can use GNS to access ordinary websites, this is more an -experimental feature and not really our primary goal at this time. Still, it is -a possible use-case and we welcome help with testing and development. - -@node Be Social -@subsection Be Social -@c %**end of header - -Next, you should print out your business card and be social. Find a friend, help -him install GNUnet and exchange business cards with him. Or, if you're a -desperate loner, you might try the next step with your own card. Still, it'll be -hard to have a conversation with yourself later, so it would be better if you -could find a friend. You might also want a camera attached to your computer, so -you might need a trip to the store together. Once you have a business card, run: - -@example -$ gnunet-qr -@end example - -to open a window showing whatever your camera points at. Hold up your friend's -business card and tilt it until the QR code is recognized. At that point, the -window should automatically close. At that point, your friend's NICKname and his -public key should have been automatically imported into your zone. Assuming both -of your peers are properly integrated in the GNUnet network at this time, you -should thus be able to resolve your friends names. Suppose your friend's -nickname is "Bob". Then, type - -@example -$ gnunet-gns -u test.bob.gnu -@end example - -to check if your friend was as good at following instructions as you were. - - -@node Backup of Identities and Egos -@subsection Backup of Identities and Egos - - -One should always backup their files, especially in these SSD days (our -team has suffered 3 SSD crashes over a span of 2 weeks). Backing up peer -identity and zones is achieved by copying the following files: - -The peer identity file can be found -in @file{~/.local/share/gnunet/private_key.ecc} - -The private keys of your egos are stored in the -directory @file{~/.local/share/gnunet/identity/egos/}. They are stored in -files whose filenames correspond to the zones' ego names. These are -probably the most important files you want to backup from a GNUnet -installation. - -Note: All these files contain cryptographic keys and they are stored without -any encryption. So it is advisable to backup encrypted copies of them. - -@node Revocation -@subsection Revocation - -Now, in the situation of an attacker gaining access to the private key of -one of your egos, the attacker can create records in the respective GNS zone -and publish them as if you published them. Anyone resolving your domain will -get these new records and when they verify they seem authentic because the -attacker has signed them with your key. - -To address this potential security issue, you can pre-compute a revocation -certificate corresponding to your ego. This certificate, when published on -the P2P network, flags your private key as invalid, and all further -resolutions or other checks involving the key will fail. - -A revocation certificate is thus a useful tool when things go out of control, -but at the same time it should be stored securely. Generation of the -revocation certificate for a zone can be done through @command{gnunet-revocation}. -For example, the following command (as unprivileged user) generates a revocation -file @file{revocation.dat} for the zone @code{zone1}: -@command{gnunet-revocation -f revocation.dat -R zone1} - -The above command only pre-computes a revocation certificate. It does not -revoke the given zone. Pre-computing a revocation certificate involves -computing a proof-of-work and hence may take upto 4 to 5 days on a modern -processor. Note that you can abort and resume the calculation at any time. -Also, even if you did not finish the calculation, the resulting file willl -contain the signature, which is sufficient to complete the revocation -process even without access to the private key. So instead of waiting for a -few days, you can just abort with CTRL-C, backup the revocation -certificate and run the calculation only if your key actually was compromised. -This has the disadvantage of revocation taking longer after the incident, but -the advantage of saving a significant amount of energy. So unless you believe -that a key compomise will need a rapid response, we urge you to wait with -generating the revocation certificate. Also, the calculation is deliberately -expensive, to deter people from doing this just for fun (as the actual -revocation operation is expensive for the network, not for the peer performing -the revocation). - -To avoid TL;DR ones from accidentally revocating their zones, I am not giving -away the command, but its simple: the actual revocation is performed by using -the @command{-p} option of @command{gnunet-revocation}. - - - -@node What's Next? -@subsection What's Next? -@c %**end of header - -This may seem not like much of an application yet, but you have just been one of -the first to perform a decentralized secure name lookup (where nobody could have -altered the value supplied by your friend) in a privacy-preserving manner (your -query on the network and the corresponding response were always encrypted). So -what can you really do with this? Well, to start with, you can publish your -GnuPG fingerprint in GNS as a "CERT" record and replace the public web-of-trust -with its complicated trust model with explicit names and privacy-preserving -resolution. Also, you should read the next chapter of the tutorial and learn how -to use GNS to have a private conversation with your friend. Finally, help us -with the next GNUnet release for even more applications using this new -public key infrastructure. - -@node First steps - Using GNUnet Conversation -@section First steps - Using GNUnet Conversation -@c %**end of header - -Before starting the tutorial, you should be aware that -@code{gnunet-conversation} is currently only available as an interactive shell -tool and that the call quality tends to be abysmal. There are also some awkward -steps necessary to use it. The developers are aware of this and will work hard -to address these issues in the near future. - - -@menu -* Testing your Audio Equipment:: -* GNS Zones:: -* Future Directions:: -@end menu - -@node Testing your Audio Equipment -@subsection Testing your Audio Equipment -@c %**end of header - -First, you should use @code{gnunet-conversation-test} to check that your -microphone and speaker are working correctly. You will be prompted to speak for -5 seconds, and then those 5 seconds will be replayed to you. The network is not -involved in this test. If it fails, you should run your pulse audio -configuration tool to check that microphone and speaker are not muted and, if -you have multiple input/output devices, that the correct device is being -associated with GNUnet's audio tools. - -@node GNS Zones -@subsection GNS Zones -@c %**end of header - -@code{gnunet-conversation} uses GNS for addressing. This means that you need to -have a GNS zone created before using it. Information about how to create GNS -zones can be found here. - - -@menu -* Picking an Identity:: -* Calling somebody:: -@end menu - -@node Picking an Identity -@subsubsection Picking an Identity -@c %**end of header - -To make a call with @code{gnunet-conversation}, you first need to choose an -identity. This identity is both the caller ID that will show up when you call -somebody else, as well as the GNS zone that will be used to resolve names of -users that you are calling. Usually, the @code{master-zone} is a reasonable -choice. Run - -@example -gnunet-conversation -e master-zone -@end example - -to start the command-line tool. You will see a message saying that your phone is -now "active on line 0". You can connect multiple phones on different lines at -the same peer. For the first phone, the line zero is of course a fine choice. - -Next, you should type in @command{/help} for a list of available commands. We will -explain the important ones during this tutorial. First, you will need to type in -@command{/address} to determine the address of your phone. The result should look -something like this: - -@example -/address -0-PD67SGHF3E0447TU9HADIVU9OM7V4QHTOG0EBU69TFRI2LG63DR0 -@end example - -Here, the "0" is your phone line, and what follows after the hyphen is your -peer's identity. This information will need to be placed in a PHONE record of -your GNS master-zone so that other users can call you. - -Start @code{gnunet-namestore-gtk} now (possibly from another shell) and create -an entry home-phone in your master zone. For the record type, select PHONE. You -should then see the PHONE dialog:@ - -Note: Do not choose the expiry time to be 'Never'. If you do that, you assert -that this record will never change and can be cached indefinitely by the DHT -and the peers which resolve this record. A reasonable period is 1 year. - -Enter your peer identity under Peer and leave the line at zero. Select the first -option to make the record public. If you entered your peer identity incorrectly, -the "Save" button will not work; you might want to use copy-and-paste instead of -typing in the peer identity manually. Save the record. - -@node Calling somebody -@subsubsection Calling somebody -@c %**end of header - -Now you can call a buddy. Obviously, your buddy will have to have GNUnet -installed and must have performed the same steps. Also, you must have your buddy -in your GNS master zone, for example by having imported your buddy's public key -using @code{gnunet-qr}. Suppose your buddy is in your zone as @code{buddy.gnu} -and he also created his phone using a label "home-phone". Then you can initiate -a call using: - -@example -/call home-phone.buddy.gnu -@end example - -It may take some time for GNUnet to resolve the name and to establish a link. If -your buddy has your public key in his master zone, he should see an incoming -call with your name. If your public key is not in his master zone, he will just -see the public key as the caller ID. - -Your buddy then can answer the call using the "/accept" command. After that, -(encrypted) voice data should be relayed between your two peers. Either of you -can end the call using @command{/cancel}. You can exit @code{gnunet-converation} using -@command{/quit}. - -@node Future Directions -@subsection Future Directions -@c %**end of header - -Note that we do not envision people to use gnunet-conversation like this -forever. We will write a graphical user interface, and that GUI will -automatically create the necessary records in the respective zone. - -@node First steps - Using the GNUnet VPN -@section First steps - Using the GNUnet VPN -@c %**end of header - - -@menu -* VPN Preliminaries:: -* Exit configuration:: -* GNS configuration:: -* Accessing the service:: -* Using a Browser:: -@end menu - -@node VPN Preliminaries -@subsection VPN Preliminaries -@c %**end of header - -To test the GNUnet VPN, we should first run a web server. The easiest way to do -this is to just start @code{gnunet-bcd}, which will run a webserver on port -@code{8888} by default. Naturally, you can run some other HTTP server for our -little tutorial. - -If you have not done this, you should also configure your Name System Service -switch to use GNS. In your @code{/etc/nsswitch.conf} you should fine a line like -this: -@example -hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4 -@end example - -The exact details may differ a bit, which is fine. Add the text -@code{gns [NOTFOUND=return]} after @code{files}: -@example -hosts: files gns [NOTFOUND=return] mdns4_minimal [NOTFOUND=return] dns mdns4 -@end example - - -You might want to make sure that @code{/lib/libnss_gns.so.2} exists on your -system, it should have been created during the installation. If not, re-run -@example -$ configure --with-nssdir=/lib -$ cd src/gns/nss; sudo make install -@end example - -to install the NSS plugins in the proper location. - -@node Exit configuration -@subsection Exit configuration -@c %**end of header - -Stop your peer (as user @code{gnunet}, run @code{gnunet-arm -e}) and run -@code{gnunet-setup}. In @code{gnunet-setup}, make sure to activate the -@strong{EXIT} and @strong{GNS} services in the General tab. Then select the Exit -tab. Most of the defaults should be fine (but you should check against the -screenshot that they have not been modified). In the bottom area, enter -@code{bcd} under Identifier and change the Destination to -@code{169.254.86.1:8888} (if your server runs on a port other than 8888, change -the 8888 port accordingly). - -Now exit @code{gnunet-setup} and restart your peer (@code{gnunet-arm -s}). - -@node GNS configuration -@subsection GNS configuration -@c %**end of header - -Now, using your normal user (not the @code{gnunet} system user), run -@code{gnunet-gtk}. Select the GNS icon and add a new label www in your master -zone. For the record type, select @code{VPN}. You should then see the VPN -dialog: - -Under peer, you need to supply the peer identity of your own peer. You can -obtain the respective string by running@ -@code{@ - $ gnunet-peerinfo -sq@ -}@ -as the @code{gnunet} user. For the Identifier, you need to supply the same -identifier that we used in the Exit setup earlier, so here supply "bcd". If you -want others to be able to use the service, you should probably make the record -public. For non-public services, you should use a passphrase instead of the -string "bcd". Save the record and exit @code{gnunet-gtk}. - -@node Accessing the service -@subsection Accessing the service -@c %**end of header - -You should now be able to access your webserver. Type in:@ -@code{@ - $ wget http://www.gnu/@ -}@ -The request will resolve to the VPN record, telling the GNS resolver to route it -via the GNUnet VPN. The GNS resolver will ask the GNUnet VPN for an IPv4 address -to return to the application. The VPN service will use the VPN information -supplied by GNS to create a tunnel (via GNUnet's MESH service) to the EXIT peer. -At the EXIT, the name "bcd" and destination port (80) will be mapped to the -specified destination IP and port. While all this is currently happening on just -the local machine, it should also work with other peers --- naturally, they will -need a way to access your GNS zone first, for example by learning your public -key from a QR code on your business card. - -@node Using a Browser -@subsection Using a Browser -@c %**end of header - -Sadly, modern browsers tend to bypass the Name Services Switch and attempt DNS -resolution directly. You can either run a @code{gnunet-dns2gns} DNS proxy, or -point the browsers to an HTTP proxy. When we tried it, Iceweasel did not like to -connect to the socks proxy for @code{.gnu} TLDs, even if we disabled its -autoblunder of changing @code{.gnu} to ".gnu.com". Still, using the HTTP proxy -with Chrome does work. - -@node File-sharing -@section File-sharing -@c %**end of header - -This chapter documents the GNUnet file-sharing application. The original -file-sharing implementation for GNUnet was designed to provide -@strong{anonymous} file-sharing. However, over time, we have also added support -for non-anonymous file-sharing (which can provide better performance). Anonymous -and non-anonymous file-sharing are quite integrated in GNUnet and, except for -routing, share most of the concepts and implementation. There are three primary -file-sharing operations: publishing, searching and downloading. For each of -these operations, the user specifies an @strong{anonymity level}. If both the -publisher and the searcher/downloader specify "no anonymity", non-anonymous -file-sharing is used. If either user specifies some desired degree of anonymity, -anonymous file-sharing will be used. - -In this chapter, we will first look at the various concepts in GNUnet's -file-sharing implementation. Then, we will discuss specifics as to how they -impact users that publish, search or download files. - - - -@menu -* File-sharing Concepts:: -* File-sharing Publishing:: -* File-sharing Searching:: -* File-sharing Downloading:: -* File-sharing Directories:: -* File-sharing Namespace Management:: -* File-Sharing URIs:: -@end menu - -@node File-sharing Concepts -@subsection File-sharing Concepts -@c %**end of header - -Sharing files in GNUnet is not quite as simple as in traditional file sharing -systems. For example, it is not sufficient to just place files into a specific -directory to share them. In addition to anonymous routing GNUnet attempts to -give users a better experience in searching for content. GNUnet uses -cryptography to safely break content into smaller pieces that can be obtained -from different sources without allowing participants to corrupt files. GNUnet -makes it difficult for an adversary to send back bogus search results. GNUnet -enables content providers to group related content and to establish a -reputation. Furthermore, GNUnet allows updates to certain content to be made -available. This section is supposed to introduce users to the concepts that are -used to achive these goals. - - -@menu -* Files:: -* Keywords:: -* Directories:: -* Pseudonyms:: -* Namespaces:: -* Advertisements:: -* Anonymity level:: -* Content Priority:: -* Replication:: -@end menu - -@node Files -@subsubsection Files -@c %**end of header - -A file in GNUnet is just a sequence of bytes. Any file-format is allowed and the -maximum file size is theoretically 264 bytes, except that it would take an -impractical amount of time to share such a file. GNUnet itself never interprets -the contents of shared files, except when using GNU libextractor to obtain -keywords. - -@node Keywords -@subsubsection Keywords -@c %**end of header - -Keywords are the most simple mechanism to find files on GNUnet. Keywords are -@strong{case-sensitive} and the search string must always match @strong{exactly} -the keyword used by the person providing the file. Keywords are never -transmitted in plaintext. The only way for an adversary to determine the keyword -that you used to search is to guess it (which then allows the adversary to -produce the same search request). Since providing keywords by hand for each -shared file is tedious, GNUnet uses GNU libextractor to help automate this -process. Starting a keyword search on a slow machine can take a little while -since the keyword search involves computing a fresh RSA key to formulate the -request. - -@node Directories -@subsubsection Directories -@c %**end of header - -A directory in GNUnet is a list of file identifiers with meta data. The file -identifiers provide sufficient information about the files to allow downloading -the contents. Once a directory has been created, it cannot be changed since it -is treated just like an ordinary file by the network. Small files (of a few -kilobytes) can be inlined in the directory, so that a separate download becomes -unnecessary. - -@node Pseudonyms -@subsubsection Pseudonyms -@c %**end of header - -Pseudonyms in GNUnet are essentially public-private (RSA) key pairs that allow a -GNUnet user to maintain an identity (which may or may not be detached from his -real-life identity). GNUnet's pseudonyms are not file-sharing specific --- and -they will likely be used by many GNUnet applications where a user identity is -required. - -Note that a pseudonym is NOT bound to a GNUnet peer. There can be multiple -pseudonyms for a single user, and users could (theoretically) share the private -pseudonym keys (currently only out-of-band by knowing which files to copy -around). - -@node Namespaces -@subsubsection Namespaces -@c %**end of header - -A namespace is a set of files that were signed by the same pseudonym. Files (or -directories) that have been signed and placed into a namespace can be updated. -Updates are identified as authentic if the same secret key was used to sign the -update. Namespaces are also useful to establish a reputation, since all of the -content in the namespace comes from the same entity (which does not have to be -the same person). - -@node Advertisements -@subsubsection Advertisements -@c %**end of header - -Advertisements are used to notify other users about the existence of a -namespace. Advertisements are propagated using the normal keyword search. When -an advertisement is received (in response to a search), the namespace is added -to the list of namespaces available in the namespace-search dialogs of -gnunet-fs-gtk and printed by gnunet-pseudonym. Whenever a namespace is created, -an appropriate advertisement can be generated. The default keyword for the -advertising of namespaces is "namespace". - -Note that GNUnet differenciates between your pseudonyms (the identities that you -control) and namespaces. If you create a pseudonym, you will not automatically -see the respective namespace. You first have to create an advertisement for the -namespace and find it using keyword search --- even for your own namespaces. The -@code{gnunet-pseudonym} tool is currently responsible for both managing -pseudonyms and namespaces. This will likely change in the future to reduce the -potential for confusion. - -@node Anonymity level -@subsubsection Anonymity level -@c %**end of header - -The anonymity level determines how hard it should be for an adversary to -determine the identity of the publisher or the searcher/downloader. An -anonymity level of zero means that anonymity is not required. The default -anonymity level of "1" means that anonymous routing is desired, but no -particular amount of cover traffic is necessary. A powerful adversary might thus -still be able to deduce the origin of the traffic using traffic analysis. -Specifying higher anonymity levels increases the amount of cover traffic -required. While this offers better privacy, it can also significantly hurt -performance. - -@node Content Priority -@subsubsection Content Priority -@c %**end of header - -Depending on the peer's configuration, GNUnet peers migrate content between -peers. Content in this sense are individual blocks of a file, not necessarily -entire files. When peers run out of space (due to local publishing operations or -due to migration of content from other peers), blocks sometimes need to be -discarded. GNUnet first always discards expired blocks (typically, blocks are -published with an expiration of about two years in the future; this is another -option). If there is still not enough space, GNUnet discards the blocks with the -lowest priority. The priority of a block is decided by its popularity (in terms -of requests from peers we trust) and, in case of blocks published locally, the -base-priority that was specified by the user when the block was published -initially. - -@node Replication -@subsubsection Replication -@c %**end of header - -When peers migrate content to other systems, the replication level of a block is -used to decide which blocks need to be migrated most urgently. GNUnet will -always push the block with the highest replication level into the network, and -then decrement the replication level by one. If all blocks reach replication -level zero, the selection is simply random. - -@node File-sharing Publishing -@subsection File-sharing Publishing -@c %**end of header - -The command @code{gnunet-publish} can be used to add content to the network. -The basic format of the command is -@example -$ gnunet-publish [-n] [-k KEYWORDS]* [-m TYPE:VALUE] FILENAME -@end example - - -@menu -* Important command-line options:: -* Indexing vs. Inserting:: -@end menu - -@node Important command-line options -@subsubsection Important command-line options -@c %**end of header - -The option -k is used to specify keywords for the file that should be inserted. -You can supply any number of keywords, and each of the keywords will be -sufficient to locate and retrieve the file. - -The -m option is used to specify meta-data, such as descriptions. You can use -m -multiple times. The TYPE passed must be from the list of meta-data types known -to libextractor. You can obtain this list by running @code{extract -L}. -Use quotes around the entire meta-data argument if the value contains spaces. -The meta-data is displayed to other users when they select which files to -download. The meta-data and the keywords are optional and maybe inferred using -@code{GNU libextractor}. - -gnunet-publish has a few additional options to handle namespaces and -directories. -See the man-page for details. - -@node Indexing vs. Inserting -@subsubsection Indexing vs Inserting -@c %**end of header - -By default, GNUnet indexes a file instead of making a full copy. This is much -more efficient, but requries the file to stay unaltered at the location where it -was when it was indexed. If you intend to move, delete or alter a file, consider -using the option @code{-n} which will force GNUnet to make a copy of the file in -the database. - -Since it is much less efficient, this is strongly discouraged for large files. -When GNUnet indexes a file (default), GNUnet does @strong{not} create an -additional encrypted copy of the file but just computes a summary (or index) of -the file. That summary is approximately two percent of the size of the original -file and is stored in GNUnet's database. Whenever a request for a part of an -indexed file reaches GNUnet, this part is encrypted on-demand and send out. This -way, there is no need for an additional encrypted copy of the file to stay -anywhere on the drive. This is different from other systems, such as Freenet, -where each file that is put online must be in Freenet's database in encrypted -format, doubling the space requirements if the user wants to preseve a directly -accessible copy in plaintext. - -Thus indexing should be used for all files where the user will keep using this -file (at the location given to gnunet-publish) and does not want to retrieve it -back from GNUnet each time. If you want to remove a file that you have indexed -from the local peer, use the tool @code{gnunet-unindex} to un-index the file. - -The option @code{-n} may be used if the user fears that the file might be found -on his drive (assuming the computer comes under the control of an adversary). -When used with the @code{-n} flag, the user has a much better chance of denying -knowledge of the existence of the file, even if it is still (encrypted) on the -drive and the adversary is able to crack the encryption (e.g. by guessing the -keyword. - -@node File-sharing Searching -@subsection File-sharing Searching -@c %**end of header - -The command @code{gnunet-search} can be used to search for content on GNUnet. -The format is: -@example -$ gnunet-search [-t TIMEOUT] KEYWORD -@end example - -The -t option specifies that the query should timeout after approximately -TIMEOUT seconds. A value of zero is interpreted as @emph{no timeout}, which is -also the default. In this case, gnunet-search will never terminate (unless you -press CTRL-C). - -If multiple words are passed as keywords, they will all be considered optional. -Prefix keywords with a "+" to make them mandatory. - -Note that searching using -@example -$ gnunet-search Das Kapital -@end example - -is not the same as searching for -@example -$ gnunet-search "Das Kapital" -@end example - -as the first will match files shared under the keywords "Das" or "Kapital" -whereas the second will match files shared under the keyword "Das Kapital". - -Search results are printed by gnunet-search like this: -@example -$ gnunet-download -o "COPYING" --- gnunet://fs/chk/N8...C92.17992 -=> The GNU Public License <= (mimetype: text/plain) -@end example - -The first line is the command you would have to enter to download the file. -The argument passed to @code{-o} is the suggested filename (you may change it to -whatever you like). -The @code{--} is followed by key for decrypting the file, the query for -searching the file, a checksum (in hexadecimal) finally the size of the file in -bytes. -The second line contains the description of the file; here this is -"The GNU Public License" and the mime-type (see the options for gnunet-publish -on how to specify these). - -@node File-sharing Downloading -@subsection File-sharing Downloading -@c %**end of header - -In order to download a file, you need the three values returned by -@code{gnunet-search}. -You can then use the tool @code{gnunet-download} to obtain the file: -@example -$ gnunet-download -o FILENAME --- GNUNETURL -@end example - -FILENAME specifies the name of the file where GNUnet is supposed to write the -result. Existing files are overwritten. If the existing file contains blocks -that are identical to the desired download, those blocks will not be downloaded -again (automatic resume). - -If you want to download the GPL from the previous example, you do the following: -@example -$ gnunet-download -o "COPYING" --- gnunet://fs/chk/N8...92.17992 -@end example - -If you ever have to abort a download, you can continue it at any time by -re-issuing @command{gnunet-download} with the same filename. In that case, GNUnet -will @strong{not} download blocks again that are already present. - -GNUnet's file-encoding mechanism will ensure file integrity, even if the -existing file was not downloaded from GNUnet in the first place. - -You may want to use the @command{-V} switch (must be added before the @command{--}) to -turn on verbose reporting. In this case, @command{gnunet-download} will print the -current number of bytes downloaded whenever new data was received. - -@node File-sharing Directories -@subsection File-sharing Directories -@c %**end of header - -Directories are shared just like ordinary files. If you download a directory -with @command{gnunet-download}, you can use @command{gnunet-directory} to list its -contents. The canonical extension for GNUnet directories when stored as files in -your local file-system is ".gnd". The contents of a directory are URIs and -meta data. -The URIs contain all the information required by @command{gnunet-download} to -retrieve the file. The meta data typically includes the mime-type, description, -a filename and other meta information, and possibly even the full original file -(if it was small). - -@node File-sharing Namespace Management -@subsection File-sharing Namespace Management -@c %**end of header - -THIS TEXT IS OUTDATED AND NEEDS TO BE REWRITTEN FOR 0.10! - -The gnunet-pseudonym tool can be used to create pseudonyms and to advertise -namespaces. By default, gnunet-pseudonym simply lists all locally available -pseudonyms. - - -@menu -* Creating Pseudonyms:: -* Deleting Pseudonyms:: -* Advertising namespaces:: -* Namespace names:: -* Namespace root:: -@end menu - -@node Creating Pseudonyms -@subsubsection Creating Pseudonyms -@c %**end of header - -With the @command{-C NICK} option it can also be used to create a new pseudonym. -A pseudonym is the virtual identity of the entity in control of a namespace. -Anyone can create any number of pseudonyms. Note that creating a pseudonym can -take a few minutes depending on the performance of the machine used. - -@node Deleting Pseudonyms -@subsubsection Deleting Pseudonyms -@c %**end of header - -With the @command{-D NICK} option pseudonyms can be deleted. Once the pseudonym has -been deleted it is impossible to add content to the corresponding namespace. -Deleting the pseudonym does not make the namespace or any content in it -unavailable. - -@node Advertising namespaces -@subsubsection Advertising namespaces -@c %**end of header - -Each namespace is associated with meta-data that describes the namespace. -This meta data is provided by the user at the time that the namespace is -advertised. Advertisements are published under keywords so that they can be -found using normal keyword-searches. This way, users can learn about new -namespaces without relying on out-of-band communication or directories. -A suggested keyword to use for all namespaces is simply "namespace". -When a keyword-search finds a namespace advertisement, it is automatically -stored in a local list of known namespaces. Users can then associate a rank with -the namespace to remember the quality of the content found in it. - -@node Namespace names -@subsubsection Namespace names -@c %**end of header - -While the namespace is uniquely identified by its ID, another way to refer to -the namespace is to use the NICKNAME. The NICKNAME can be freely chosen by the -creator of the namespace and hence conflicts are possible. If a GNUnet client -learns about more than one namespace using the same NICKNAME, the ID is appended -to the NICKNAME to get a unique identifier. - -@node Namespace root -@subsubsection Namespace root -@c %**end of header - -An item of particular interest in the namespace advertisement is the ROOT. -The ROOT is the identifier of a designated entry in the namespace. The idea is -that the ROOT can be used to advertise an entry point to the content of the -namespace. - -@node File-Sharing URIs -@subsection File-Sharing URIs -@c %**end of header - -GNUnet (currently) uses four different types of URIs for file-sharing. They all -begin with "gnunet://fs/". This section describes the four different URI types -in detail. - - -@menu -* Encoding of hash values in URIs:: -* Content Hash Key (chk):: -* Location identifiers (loc):: -* Keyword queries (ksk):: -* Namespace content (sks):: -@end menu - -@node Encoding of hash values in URIs -@subsubsection Encoding of hash values in URIs -@c %**end of header - -Most URIs include some hash values. Hashes are encoded using base32hex -(RFC 2938). - -@node Content Hash Key (chk) -@subsubsection Content Hash Key (chk) -@c %**end of header - -A chk-URI is used to (uniquely) identify a file or directory and to allow peers -to download the file. Files are stored in GNUnet as a tree of encrypted blocks. -The chk-URI thus contains the information to download and decrypt those blocks. -A chk-URI has the format "gnunet://fs/chk/KEYHASH.QUERYHASH.SIZE". Here, "SIZE" -is the size of the file (which allows a peer to determine the shape of the -tree), KEYHASH is the key used to decrypt the file (also the hash of the -plaintext of the top block) and QUERYHASH is the query used to request the -top-level block (also the hash of the encrypted block). - -@node Location identifiers (loc) -@subsubsection Location identifiers (loc) -@c %**end of header - -For non-anonymous file-sharing, loc-URIs are used to specify which peer is -offering the data (in addition to specifying all of the data from a chk-URI). -Location identifiers include a digital signature of the peer to affirm that the -peer is truly the origin of the data. The format is -"gnunet://fs/loc/KEYHASH.QUERYHASH.SIZE.PEER.SIG.EXPTIME". Here, "PEER" is the -public key of the peer (in GNUnet format in base32hex), SIG is the RSA signature -(in GNUnet format in base32hex) and EXPTIME specifies when the signature expires -(in milliseconds after 1970). - -@node Keyword queries (ksk) -@subsubsection Keyword queries (ksk) -@c %**end of header - -A keyword-URI is used to specify that the desired operation is the search using -a particular keyword. The format is simply "gnunet://fs/ksk/KEYWORD". Non-ASCII -characters can be specified using the typical URI-encoding (using hex values) -from HTTP. "+" can be used to specify multiple keywords (which are then -logically "OR"-ed in the search, results matching both keywords are given a -higher rank): "gnunet://fs/ksk/KEYWORD1+KEYWORD2". - -@node Namespace content (sks) -@subsubsection Namespace content (sks) -@c %**end of header - -Namespaces are sets of files that have been approved by some (usually -pseudonymous) user --- typically by that user publishing all of the files -together. A file can be in many namespaces. A file is in a namespace if the -owner of the ego (aka the namespace's private key) signs the CHK of the file -cryptographically. An SKS-URI is used to search a namespace. The result is a -block containing meta data, the CHK and the namespace owner's signature. The -format of a sks-URI is "gnunet://fs/sks/NAMESPACE/IDENTIFIER". Here, "NAMESPACE" -is the public key for the namespace. "IDENTIFIER" is a freely chosen keyword -(or password!). A commonly used identifier is "root" which by convention refers -to some kind of index or other entry point into the namespace. - -@node The GNU Name System -@section The GNU Name System -@c %**end of header - - -The GNU Name System (GNS) is secure and decentralized naming system. -It allows its users to resolve and register names within the @code{.gnu} -@dfn{top-level domain} (TLD). - -GNS is designed to provide: -@itemize @bullet -@item Censorship resistance -@item Query privacy -@item Secure name resolution -@item Compatibility with DNS -@end itemize - -For the initial configuration and population of your GNS installation, please -follow the GNS setup instructions. The remainder of this chapter will provide -some background on GNS and then describe how to use GNS in more detail. - -Unlike DNS, GNS does not rely on central root zones or authorities. Instead any -user administers his own root and can can create arbitrary name value mappings. -Furthermore users can delegate resolution to other users' zones just like DNS NS -records do. Zones are uniquely identified via public keys and resource records -are signed using the corresponding public key. Delegation to another user's zone -is done using special PKEY records and petnames. A petname is a name that can be -freely chosen by the user. This results in non-unique name-value mappings as -@code{@uref{http://www.bob.gnu/, www.bob.gnu}} to one user might be -@code{@uref{http://www.friend.gnu/, www.friend.gnu}} for someone else. - - -@menu -* Maintaining your own Zones:: -* Obtaining your Zone Key:: -* Adding Links to Other Zones:: -* The Three Local Zones of GNS:: -* The Master Zone:: -* The Private Zone:: -* The Shorten Zone:: -* The ZKEY Top Level Domain in GNS:: -* Resource Records in GNS:: -@end menu - - -@node Maintaining your own Zones -@subsection Maintaining your own Zones - -To setup your GNS system you must execute: - -@example -$ gnunet-gns-import.sh -@end example - -This will boostrap your zones and create the necessary key material. -Your keys can be listed using the gnunet-identity command line tool: - -@example -$ gnunet-identity -d -@end example - -You can arbitrarily create your own zones using the gnunet-identity tool using: - -@example -$ gnunet-identity -C "new_zone" -@end example - -Now you can add (or edit, or remove) records in your GNS zone using the -gnunet-setup GUI or using the gnunet-namestore command-line tool. In either -case, your records will be stored in an SQL database under control of the -gnunet-service-namestore. Note that if mutliple users use one peer, the -namestore database will include the combined records of all users. However, -users will not be able to see each other's records if they are marked as -private. - -To provide a simple example for editing your own zone, suppose you have your own -web server with IP 1.2.3.4. Then you can put an A record (A records in DNS are -for IPv4 IP addresses) into your local zone using the command:@ - -@example -$ gnunet-namestore -z master-zone -a -n www -t A -V 1.2.3.4 -e never -@end example - -Afterwards, you will be able to access your webpage under "www.gnu" (assuming -your webserver does not use virtual hosting, if it does, please read up on -setting up the GNS proxy). - -Similar commands will work for other types of DNS and GNS records, the syntax -largely depending on the type of the record. Naturally, most users may find -editing the zones using the gnunet-setup GUI to be easier. - -@node Obtaining your Zone Key -@subsection Obtaining your Zone Key - -Each zone in GNS has a public-private key. Usually, gnunet-namestore and -gnunet-setup will access your private key as necessary, so you do not have to -worry about those. What is important is your public key (or rather, the hash of -your public key), as you will likely want to give it to others so that they can -securely link to you. - -You can usually get the hash of your public key using@ - -@example -$ gnunet-identity -d $options | grep master-zone | awk '@{print $3@}' -@end example - -For example, the output might be something like: - -@example -DC3SEECJORPHQNVRH965A6N74B1M37S721IG4RBQ15PJLLPJKUE0 -@end example - -Alternatively, you can obtain a QR code with your zone key AND your pseudonym -from gnunet-gtk. The QR code is displayed in the GNS tab and can be stored to -disk using the Save as button next to the image. - -@node Adding Links to Other Zones -@subsection Adding Links to Other Zones - - -A central operation in GNS is the ability to securely delegate to other zones. -Basically, by adding a delegation you make all of the names from the other zone -available to yourself. This section describes how to create delegations. - -Suppose you have a friend who you call 'bob' who also uses GNS. You can then -delegate resolution of names to Bob's zone by adding a PKEY record to his local -zone: - -@example -$ gnunet-namestore -a -n bob --type PKEY -V XXXX -e never -@end example - -Note that XXXX in the command above must be replaced with the hash of Bob's -public key (the output your friend obtained using the gnunet-identity command -from the previous section and told you, for example by giving you a business -card containing this information as a QR code). - -Assuming Bob has an A record for his website under the name of www in his zone, -you can then access Bob's website under www.bob.gnu --- as well as any (public) -GNS record that Bob has in his zone by replacing www with the respective name of -the record in Bob's zone. - -Furthermore, if Bob has himself a (public) delegation to Carol's zone under -"carol", you can access Carol's records under NAME.carol.bob.gnu (where NAME is -the name of Carol's record you want to access). - -@node The Three Local Zones of GNS -@subsection The Three Local Zones of GNS - -Each user GNS has control over three zones. Each of the zones has a different -purpose. These zones are the -@itemize @bullet - -@item master zone, -@item private zone, and the -@item shorten zone. -@end itemize - -@node The Master Zone -@subsection The Master Zone - - -The master zone is your personal TLD. Names within the @code{.gnu} namespace are -resolved relative to this zone. You can arbitrarily add records to this zone and -selectively publish those records. - -@node The Private Zone -@subsection The Private Zone - - -The private zone is a subzone (or subdomain in DNS terms) of your master zone. -It should be used for records that you want to keep private. For example -@code{bank.private.gnu}. The key idea is that you want to keep your private -records separate, if just to know that those names are not available to other -users. - -@node The Shorten Zone -@subsection The Shorten Zone - - -The shorten zone can either be a subzone of the master zone or the private zone. -It is different from the other zones in that GNS will automatically populate -this zone with other users' zones based on their PSEU records whenever you -resolve a name. - -For example if you go to -@code{@uref{http://www.bob.alice.dave.gnu/, www.bob.alice.dave.gnu}}, GNS will -try to import @code{bob} into your shorten zone. Having obtained Bob's PKEY from -@code{alice.dave.gnu}, GNS will lookup the PSEU record for @code{+} in Bob's -zone. If it exists and the specified pseudonym is not taken, Bob's PKEY will be -automatically added under that pseudonym (i.e. "bob") into your shorten zone. -From then on, Bob's webpage will also be available for you as -@code{@uref{http://www.bob.short.gnu/, www.bob.short.gnu}}. This feature is -called automatic name shortening and is supposed to keep GNS names as short and -memorable as possible. - -@node The ZKEY Top Level Domain in GNS -@subsection The ZKEY Top Level Domain in GNS - - -GNS also provides a secure and globally unique namespace under the .zkey -top-level domain. A name in the .zkey TLD corresponds to the (printable) public -key of a zone. Names in the .zkey TLD are then resolved by querying the -respective zone. The .zkey TLD is expected to be used under rare circumstances -where globally unique names are required and for integration with legacy -systems. - -@node Resource Records in GNS -@subsection Resource Records in GNS - - -GNS supports the majority of the DNS records as defined in -@uref{http://www.ietf.org/rfc/rfc1035.txt, RFC 1035}. Additionally, GNS defines -some new record types the are unique to the GNS system. For example, -GNS-specific resource records are use to give petnames for zone delegation, -revoke zone keys and provide some compatibility features. - -For some DNS records, GNS does extended processing to increase their usefulness -in GNS. In particular, GNS introduces special names referred to as -"zone relative names". Zone relative names are allowed in some resource record -types (for example, in NS and CNAME records) and can also be used in links on -webpages. Zone relative names end in ".+" which indicates that the name needs to -be resolved relative to the current authoritative zone. The extended processing -of those names will expand the ".+" with the correct delegation chain to the -authoritative zone (replacing ".+" with the name of the location where the name -was encountered) and hence generate a valid @code{.gnu} name. - -GNS currently supports the following record types: - -@menu -* NICK:: -* PKEY:: -* BOX:: -* LEHO:: -* VPN:: -* A AAAA and TXT:: -* CNAME:: -* GNS2DNS:: -* SOA SRV PTR and MX:: -@end menu - -@node NICK -@subsubsection NICK - -A NICK record is used to give a zone a name. With a NICK record, you can -essentially specify how you would like to be called. GNS expects this record -under the name "+" in the zone's database (NAMESTORE); however, it will then -automatically be copied into each record set, so that clients never need to do a -separate lookup to discover the NICK record. - -@b{Example}@ - -@example -Name: +; RRType: NICK; Value: bob -@end example - -This record in Bob's zone will tell other users that this zone wants to be -referred to as 'bob'. Note that nobody is obliged to call Bob's zone 'bob' in -their own zones. It can be seen as a recommendation ("Please call me 'bob'"). - -@node PKEY -@subsubsection PKEY - -PKEY records are used to add delegation to other users' zones and give those -zones a petname. - -@b{Example}@ - -Let Bob's zone be identified by the hash "ABC012". Bob is your friend so you -want to give him the petname "friend". Then you add the following record to your -zone: - -@example -Name: friend; RRType: PKEY; Value: ABC012; -@end example - -This will allow you to resolve records in bob's zone under "*.friend.gnu". - -@node BOX -@subsubsection BOX - -BOX records are there to integrate information from TLSA or SRV records under -the main label. In DNS, TLSA and SRV records use special names of the form -@code{_port._proto.(label.)*tld} to indicate the port number and protocol -(i.e. tcp or udp) for which the TLSA or SRV record is valid. This causes various -problems, and is elegantly solved in GNS by integrating the protocol and port -numbers together with the respective value into a "BOX" record. Note that in the -GUI, you do not get to edit BOX records directly right now --- the GUI will -provide the illusion of directly editing the TLSA and SRV records, even though -they internally are BOXed up. - -@node LEHO -@subsubsection LEHO - -The LEgacy HOstname of a server. Some webservers expect a specific hostname to -provide a service (virtiual hosting). Also SSL certificates usually contain DNS -names. To provide the expected legacy DNS name for a server, the LEHO record can -be used. To mitigate the just mentioned issues the GNS proxy has to be used. The -GNS proxy will use the LEHO information to apply the necessary transformations. - -@node VPN -@subsubsection VPN - -GNS allows easy access to services provided by the GNUnet Virtual Public -Network. When the GNS resolver encounters a VPN record it will contact the VPN -service to try and allocate an IPv4/v6 address (if the queries record type is an -IP address) that can be used to contact the service. - -@b{Example}@ - -I want to provide access to the VPN service "web.gnu." on port 80 on peer -ABC012:@ -Name: www; RRType: VPN; Value: 80 ABC012 web.gnu. - -The peer ABC012 is configured to provide an exit point for the service -"web.gnu." on port 80 to it's server running locally on port 8080 by having the -following lines in the @file{gnunet.conf} configuration file:@ -@example -[web.gnunet.] -TCP_REDIRECTS = 80:localhost4:8080 -@end example - -@node A AAAA and TXT -@subsubsection A AAAA and TXT - -Those records work in exactly the same fashion as in traditional DNS. - -@node CNAME -@subsubsection CNAME - -As specified in RFC 1035 whenever a CNAME is encountered the query needs to be -restarted with the specified name. In GNS a CNAME can either be: - -@itemize @bullet -@item A zone relative name, -@item A zkey name or -@item A DNS name (in which case resolution will continue outside of GNS with the systems DNS resolver) -@end itemize - -@node GNS2DNS -@subsubsection GNS2DNS - -GNS can delegate authority to a legacy DNS zone. For this, the name of the DNS -nameserver and the name of the DNS zone are specified in a GNS2DNS record. - -@b{Example} - -@example -Name: pet; RRType: GNS2DNS; Value: gnunet.org@@a.ns.joker.com -@end example - -Any query to @code{pet.gnu} will then be delegated to the DNS server at -@code{a.ns.joker.com}.@ -For example, @code{@uref{http://www.pet.gnu/, www.pet.gnu}} will result in a -DNS query for @code{@uref{http://www.gnunet.org/, www.gnunet.org}} to the server -at @code{a.ns.joker.com}. Delegation to DNS via NS records in GNS can be useful -if you do not want to start resolution in the DNS root zone (due to issues such -as censorship or availability). - -Note that you would typically want to use a relative name for the nameserver, -i.e. -@example -Name: pet; RRType: GNS2DNS; Value: gnunet.org@@ns-joker.+@ -Name: ns-joker; RRType: A; Value: 184.172.157.218 -@end example - -This way, you can avoid involving the DNS hierarchy in the resolution of -@code{a.ns.joker.com}. In the example above, the problem may not be obvious as -the nameserver for "gnunet.org" is in the ".com" zone. However, imagine the -nameserver was "ns.gnunet.org". In this case, delegating to "ns.gnunet.org" -would mean that despite using GNS, censorship in the DNS ".org" zone would still -be effective. - -@node SOA SRV PTR and MX -@subsubsection SOA SRV PTR and MX - -The domain names in those records can, again, be either -@itemize @bullet -@item A zone relative name, -@item A zkey name or -@item A DNS name -@end itemize - -The resolver will expand the zone relative name if possible. Note that when -using MX records within GNS, the target mail server might still refuse to accept -e-mails to the resulting domain as the name might not match. GNS-enabled mail -clients should use the ZKEY zone as the destination hostname and GNS-enabled -mail servers should be configured to accept e-mails to the ZKEY-zones of all -local users. - -@node Using the Virtual Public Network -@section Using the Virtual Public Network - -@menu -* Setting up an Exit node:: -* Fedora and the Firewall:: -* Setting up VPN node for protocol translation and tunneling:: -@end menu - -Using the GNUnet Virtual Public Network (VPN) application you can tunnel IP -traffic over GNUnet. Moreover, the VPN comes with built-in protocol translation -and DNS-ALG support, enabling IPv4-to-IPv6 protocol translation -(in both directions). This chapter documents how to use the GNUnet VPN. - -The first thing to note about the GNUnet VPN is that it is a public network. All -participating peers can participate and there is no secret key to control -access. So unlike common virtual private networks, the GNUnet VPN is not useful -as a means to provide a "private" network abstraction over the Internet. The -GNUnet VPN is a virtual network in the sense that it is an overlay over the -Internet, using its own routing mechanisms and can also use an internal -addressing scheme. The GNUnet VPN is an Internet underlay --- TCP/IP -applications run on top of it. - -The VPN is currently only supported on GNU/Linux systems. Support for operating -systems that support TUN (such as FreeBSD) should be easy to add (or might not -even require any coding at all --- we just did not test this so far). Support -for other operating systems would require re-writing the code to create virtual -network interfaces and to intercept DNS requests. - -The VPN does not provide good anonymity. While requests are routed over the -GNUnet network, other peers can directly see the source and destination of each -(encapsulated) IP packet. Finally, if you use the VPN to access Internet -services, the peer sending the request to the Internet will be able to observe -and even alter the IP traffic. We will discuss additional security implications -of using the VPN later in this chapter. - -@node Setting up an Exit node -@subsection Setting up an Exit node - -Any useful operation with the VPN requires the existence of an exit node in the -GNUnet Peer-to-Peer network. Exit functionality can only be enabled on peers -that have regular Internet access. If you want to play around with the VPN or -support the network, we encourage you to setup exit nodes. This chapter -documents how to setup an exit node. - -There are four types of exit functions an exit node can provide, and using the -GNUnet VPN to access the Internet will only work nicely if the first three types -are provided somewhere in the network. The four exit functions are: -@itemize @bullet -@item DNS: allow other peers to use your DNS resolver -@item IPv4: allow other peers to access your IPv4 Internet connection -@item IPv6: allow other peers to access your IPv6 Internet connection -@item Local service: allow other peers to access a specific TCP or UDP service your peer is providing -@end itemize - -By enabling "exit" in gnunet-setup and checking the respective boxes in the -"exit" tab, you can easily choose which of the above exit functions you want to -support. - -Note, however, that by supporting the first three functions you will allow -arbitrary other GNUnet users to access the Internet via your system. This is -somewhat similar to running a Tor exit node. The torproject has a nice article -about what to consider if you want to do this here. We believe that generally -running a DNS exit node is completely harmless. - -The exit node configuration does currently not allow you to restrict the -Internet traffic that leaves your system. In particular, you cannot exclude SMTP -traffic (or block port 25) or limit to HTTP traffic using the GNUnet -configuration. However, you can use your host firewall to restrict outbound -connections from the virtual tunnel interface. This is highly recommended. In -the future, we plan to offer a wider range of configuration options for exit -nodes. - -Note that by running an exit node GNUnet will configure your kernel to perform -IP-forwarding (for IPv6) and NAT (for IPv4) so that the traffic from the virtual -interface can be routed to the Internet. In order to provide an IPv6-exit, you -need to have a subnet routed to your host's external network interface and -assign a subrange of that subnet to the GNUnet exit's TUN interface. - -When running a local service, you should make sure that the local service is -(also) bound to the IP address of your EXIT interface (i.e. 169.254.86.1). It -will NOT work if your local service is just bound to loopback. You may also want -to create a "VPN" record in your zone of the GNU Name System to make it easy for -others to access your service via a name instead of just the full service -descriptor. Note that the identifier you assign the service can serve as a -passphrase or shared secret, clients connecting to the service must somehow -learn the service's name. VPN records in the GNU Name System can make this -easier. - -@node Fedora and the Firewall -@subsection Fedora and the Firewall - - -When using an exit node on Fedora 15, the standard firewall can create trouble -even when not really exiting the local system! For IPv4, the standard rules seem -fine. However, for IPv6 the standard rules prohibit traffic from the network -range of the virtual interface created by the exit daemon to the local IPv6 -address of the same interface (which is essentially loopback traffic, so you -might suspect that a standard firewall would leave this traffic alone). However, -as somehow for IPv6 the traffic is not recognized as originating from the local -system (and as the connection is not already "established"), the firewall drops -the traffic. You should still get ICMPv6 packets back, but that's obviously not -very useful. - -Possible ways to fix this include disabling the firewall (do you have a good -reason for having it on?) or disabling the firewall at least for the GNUnet exit -interface (or the respective IPv4/IPv6 address range). The best way to diagnose -these kinds of problems in general involves setting the firewall to REJECT -instead of DROP and to watch the traffic using wireshark (or tcpdump) to see if -ICMP messages are generated when running some tests that should work. - -@node Setting up VPN node for protocol translation and tunneling -@subsection Setting up VPN node for protocol translation and tunneling - - -The GNUnet VPN/PT subsystem enables you to tunnel IP traffic over the VPN to an -exit node, from where it can then be forwarded to the Internet. This section -documents how to setup VPN/PT on a node. Note that you can enable both the VPN -and an exit on the same peer. In this case, IP traffic from your system may -enter your peer's VPN and leave your peer's exit. This can be useful as a means -to do protocol translation. For example, you might have an application that -supports only IPv4 but needs to access an IPv6-only site. In this case, GNUnet -would perform 4to6 protocol translation between the VPN (IPv4) and the -Exit (IPv6). Similarly, 6to4 protocol translation is also possible. However, the -primary use for GNUnet would be to access an Internet service running with an -IP version that is not supported by your ISP. In this case, your IP traffic -would be routed via GNUnet to a peer that has access to the Internet with the -desired IP version. - -Setting up an entry node into the GNUnet VPN primarily requires you to enable -the "VPN/PT" option in "gnunet-setup". This will launch the -"gnunet-service-vpn", "gnunet-service-dns" and "gnunet-daemon-pt" processes. -The "gnunet-service-vpn" will create a virtual interface which will be used as -the target for your IP traffic that enters the VPN. Additionally, a second -virtual interface will be created by the "gnunet-service-dns" for your DNS -traffic. You will then need to specify which traffic you want to tunnel over -GNUnet. If your ISP only provides you with IPv4 or IPv6-access, you may choose -to tunnel the other IP protocol over the GNUnet VPN. If you do not have an ISP -(and are connected to other GNUnet peers via WLAN), you can also choose to -tunnel all IP traffic over GNUnet. This might also provide you with some -anonymity. After you enable the respective options and restart your peer, your -Internet traffic should be tunneled over the GNUnet VPN. - -The GNUnet VPN uses DNS-ALG to hijack your IP traffic. Whenever an application -resolves a hostname (i.e. 'gnunet.org'), the "gnunet-daemon-pt" will instruct -the "gnunet-service-dns" to intercept the request (possibly route it over GNUnet -as well) and replace the normal answer with an IP in the range of the VPN's -interface. "gnunet-daemon-pt" will then tell "gnunet-service-vpn" to forward all -traffic it receives on the TUN interface via the VPN to the original -destination. - -For applications that do not use DNS, you can also manually create such a -mapping using the gnunet-vpn command-line tool. Here, you specfiy the desired -address family of the result (i.e. "-4"), and the intended target IP on the -Internet ("-i 131.159.74.67") and "gnunet-vpn" will tell you which IP address in -the range of your VPN tunnel was mapped. - -gnunet-vpn can also be used to access "internal" services offered by GNUnet -nodes. So if you happen to know a peer and a service offered by that peer, you -can create an IP tunnel to that peer by specifying the peer's identity, service -name and protocol (--tcp or --udp) and you will again receive an IP address that -will terminate at the respective peer's service. diff --git a/doc/documentation/Makefile.am b/doc/documentation/Makefile.am new file mode 100644 index 0000000000..571999731d --- /dev/null +++ b/doc/documentation/Makefile.am @@ -0,0 +1,233 @@ +# This Makefile.am is in the public domain +docdir = $(datadir)/doc/gnunet/ + +infoimagedir = $(infodir)/images + +#DOT_FILES = images/$(wildcard *.dot) + +#DOT_VECTOR_GRAPHICS = \ +# $(DOT_FILES:%.dot=%.eps) \ +# $(DOT_FILES:%.dot=%.pdf) + +AM_MAKEINFOHTMLFLAGS = --no-split --css-ref=docstyle.css + +dist_infoimage_DATA = \ + images/gnunet-gtk-0-10-gns-a-done.png \ + images/gnunet-gtk-0-10-gns-a.png \ + images/daemon_lego_block.png \ + images/gnunet-gtk-0-10-gns.png \ + images/gnunet-0-10-peerinfo.png \ + images/gnunet-gtk-0-10-identity.png \ + images/gnunet-fs-gtk-0-10-star-tab.png \ + images/gnunet-gtk-0-10.png \ + images/gnunet-gtk-0-10-download-area.png \ + images/gnunet-gtk-0-10-search-selected.png \ + images/gnunet-gtk-0-10-fs-menu.png \ + images/gnunet-gtk-0-10-traffic.png \ + images/gnunet-gtk-0-10-fs.png \ + images/gnunet-namestore-gtk-phone.png \ + images/gnunet-gtk-0-10-fs-publish-editing.png \ + images/gnunet-namestore-gtk-vpn.png \ + images/gnunet-gtk-0-10-fs-published.png \ + images/gnunet-setup-exit.png \ + images/gnunet-gtk-0-10-fs-publish.png \ + images/iceweasel-preferences.png \ + images/gnunet-gtk-0-10-fs-publish-select.png \ + images/iceweasel-proxy.png \ + images/gnunet-gtk-0-10-fs-publish-with-file_0.png \ + images/service_lego_block.png \ + images/gnunet-gtk-0-10-fs-publish-with-file.png \ + images/service_stack.png \ + images/gnunet-gtk-0-10-fs-search.png \ + images/gnunet-tutorial-service.png \ + images/gnunet-tutorial-system.png \ + images/daemon_lego_block.svg \ + images/lego_stack.svg \ + images/service_lego_block.svg \ + images/structure.dot + +# images/$(wildcard *.png) \ +# images/$(wildcard *.svg) +# $(DOT_FILES:%.dot=%.png) + +#DOT_OPTIONS = \ +# -Gratio=.9 -Gnodesep=.005 -Granksep=.00005 \ +# -Nfontsite=9 -Nheight=.1 -Nwidth=.1 + +# .dot.png: +# $(AM_V_DOT)$(DOT) -Tpng $(DOT_OPTIONS) < "$<" > "$(srcdir)/$@.tmp"; \ +# mv "$(srcdir)/$@.tmp" "$(srcdir)/$@" + +# .dot.pdf: +# $(AM_V_DOT)$(DOT) -Tpdf $(DOT_OPTIONS) < "$<" > "$(srcdir)/$@.tmp"; \ +# mv "$(srcdir)/$@.tmp" "$(srcdir)/$@" + +# .dot.eps: +# $(AM_V_DOT)$(DOT) -Teps $(DOT_OPTIONS) < "$<" > "$(srcdir)/$@.tmp"; \ +# mv "$(srcdir)/$@.tmp" "$(srcdir)/$@" + +# .png.eps: +# $(AM_V_GEN)convert "$<" "$@-tmp.eps"; \ +# mv "$@-tmp.eps" "$@" + +# pdf-local: $(DOT_FILES=%.dot=$(top_srcdir)/%.pdf) +# info-local: $(DOT_FILES=%.dot=$(top_srcdir)/%.png) +# ps-local: $(DOT_FILES=%.dot=$(top_srcdir)/%.eps) \ +# $(top_srcdir)/%D%/images/coreutils-size-map.eps +# dvi-local: ps-local + +gnunet_tutorial_examples = \ + 001.c \ + 002.c \ + 003.c \ + 004.c \ + 005.c \ + 006.c \ + 007.c \ + 008.c \ + 009.c \ + 010.c \ + 011.c \ + 012.c \ + 013.c \ + 013.1.c \ + 014.c \ + 015.c \ + 016.c \ + 017.c \ + 018.c \ + 019.c \ + 020.c \ + 021.c \ + 022.c \ + 023.c \ + 024.c \ + 025.c \ + 026.c + +info_TEXINFOS = \ + gnunet.texi \ + gnunet-c-tutorial.texi + +gnunet_TEXINFOS = \ + chapters/developer.texi \ + chapters/installation.texi \ + chapters/philosophy.texi \ + chapters/user.texi \ + chapters/vocabulary.texi \ + chapters/configuration.texi \ + chapters/contributing.texi \ + fdl-1.3.texi \ + gpl-3.0.texi + +EXTRA_DIST = \ + $(gnunet_TEXINFOS) \ + $(gnunet_tutorial_examples) \ + htmlxref.cnf \ + run-gendocs.sh \ + docstyle.css + + +# $(DOT_FILES) \ +# $(DOT_VECTOR_GRAPHICS) + +DISTCLEANFILES = \ + gnunet.cps \ + gnunet-c-tutorial.cps \ + chapters/developer.cps \ + chapters/installation.cps \ + chapter/philosophy.cps \ + chapters/user.cps \ + chapters/configuration.cps \ + chapters/vocabulary.cps \ + fdl-1.3.cps \ + gpl-3.0.cps + +# if HAVE_EXTENDED_DOCUMENTATION_BUILDING +daemon_lego_block.png: images/daemon_lego_block.svg + convert images/daemon_lego_block.svg images/daemon_lego_block.png && + pngcrush images/daemon_lego_block.png images/daemon_lego_block.png + +service_lego_block.png: images/service_lego_block.svg + convert images/service_lego_block.svg images/service_lego_block.png && + pngcrush images/service_lego_block.png images/serivce_lego_block.png + +lego_stack.png: images/lego_stack.svg + convert images/lego_stack.svg images/lego_stack.png && + pngcrush images/lego_stack.png images/lego_stack.png + +# FIXME: The usage of 'date' strings causes a warning. +# version.texi: +# echo "@set UPDATED $(date +'%d %B %Y')" > $@ +# echo "@set UPDATED-MONTH $(date +'%B %Y')" >> $@ +# echo "@set EDITION $(PACKAGE_VERSION)" >> $@ +# echo "@set VERSION $(PACKAGE_VERSION)" >> $@ + +# Workaround for makeinfo error. Whcih in turn introduces more +# date-related 'warnings'. Well. +version2.texi: + echo "@set UPDATED $(date +'%d %B %Y')" > $@ + echo "@set UPDATED-MONTH $(date +'%B %Y')" >> $@ + echo "@set EDITION $(PACKAGE_VERSION)" >> $@ + echo "@set VERSION $(PACKAGE_VERSION)" >> $@ + +# FIXME: rm *.html and *.pdf +#doc-clean: +# @rm *.aux *.log *.toc *.cp *.cps + +doc-all-install: + @mkdir -p $(DESTDIR)/$(docdir) + @mkdir -p $(DESTDIR)/$(infoimagedir) + @mkdir -p $(DESTDIR)/$(infodir) + @install -m 0755 gnunet.pdf $(DESTDIR)/$(docdir) + @install -m 0755 gnunet-c-tutorial.pdf $(DESTDIR)/$(docdir) + @install -m 0755 gnunet-c-tutorial.info $(DESTDIR)/$(infodir) + @install -m 0755 gnunet.info $(DESTDIR)/$(infodir) + @install gnunet.html $(DESTDIR)/$(docdir) + @install gnunet-c-tutorial.html $(DESTDIR)/$(docdir) + +doc-gendoc-install: + @mkdir -p $(DESTDIR)/$(docdir) + @cp -r manual $(DESTDIR)/$(docdir) + +# @cp -r images $(DESTDIR)/$(infoimagedir) + +dev-build: version.texi version2.texi + @makeinfo --pdf gnunet.texi + @makeinfo --pdf gnunet-c-tutorial.texi + @makeinfo --html gnunet.texi + @makeinfo --html gnunet-c-tutorial.texi + @makeinfo --no-split gnunet.texi + @makeinfo --no-split gnunet-c-tutorial.texi + +# TODO: Add more to clean. +clean: + @rm gnunet.pdf + @rm gnunet.html + @rm gnunet.info + @rm gnunet.info-1 + @rm gnunet.info-2 + @rm gnunet.info-3 + @rm gnunet-c-tutorial.pdf + @rm gnunet-c-tutorial.info + @rm gnunet-c-tutorial.html + @rm -r gnunet.t2p + @rm -r gnunet-c-tutorial.t2p + @rm -r manual + +# CLEANFILES = \ +# gnunet.log \ +# gnunet-c-tutorial.log \ +# $(wildcard *.aux) \ +# $(wildcard *.toc) \ +# $(wildcard *.cp) \ +# $(wildcard *.cps) + +#.PHONY: version.texi +# if HAVE_EXTENDED_DOCUMENTATION_BUILDING_PDF + +# if HAVE_EXTENDED_DOCUMENTATION_BUILDING_HTML + +# endif +# endif +# endif diff --git a/doc/documentation/README.txt b/doc/documentation/README.txt new file mode 100644 index 0000000000..b2a7815580 --- /dev/null +++ b/doc/documentation/README.txt @@ -0,0 +1,99 @@ +To be moved to an appropriate section of "how to write documentation" or +"how to contribute to the documentation": + +1. When writing documentation, please use gender-neutral wording when + referring to people, such as singular “theyâ€, “theirâ€, “themâ€, and + so forth. -> https://en.wikipedia.org/wiki/Singular_they + +2. Keep line length below 74 characters. + - Expection by texi2pdf output so far: URLs will break + (inserted whitespace) when they contain linebreaks + within the @url{} / @uref{}. + +3. Do not use tab characters (see chapter 2.1 texinfo manual) + +4. Use neutral language and third person perspective in the text + +4.1 So, when you refer to a user in general or addressing the user, + refer to (1). +4.1.1 Unsolved exceptions for canonical reasons: + When refering to Alice, use "she". + When refering to Bob, use "he". + These are long established examples and they + should either be replaced (avoid Alice and Bob + examples when you can) or followed. + +5. Use 2 spaces between sentences, so instead of: + + We do this and the other thing. This is done by foo. + + Write: + + We do this and the other thing. This is done by foo. + +6. Use @footnote{} instead of putting an @*ref{} to the footnote on a + collected footnote-page. + In a 200+ pages handbook it's better to have footnotes accessible + without having to skip over to the end. + +6.1 Avoid unnecessary footnotes, keep the text self-explanatory and + in a simple language where possible/necessary. + +* Completion Levels: + +** chapters/philosophy: around 100% fixed after initial export. + +* What's left to do + +- Which Texlive modules are needed? Decrease the size. +- Update the content of gnunet documentation. + +* How to use (hack) on this + +** with guix + +Adjust accordingly, ie read the Guix Documentation: +setenv GUIX_PACKAGE_PATH "gnunet/contrib/packages/guix/packages" +guix environment gnunet-doc +and +guix build -f contrib/packages/guix/gnunet-doc.scm + +** without guix + +You need to have Texinfo and Texlive in your path. +sh bootstrap +./configure --enable-documentation +cd doc +make (format you want) + +for example: make html, make info, make pdf + +* structure (relations) + +** gnunet.texi + -> chapters/developer.texi + -> chapters/installation.texi + -> chapters/philosophy.texi + -> chapters/user.texi + -> chapters/vocabulary.texi + -> images/* + -> gpl-3.0.texi + -> fdl-1.3.texi + +** gnunet-c-tutorial.texi + -> figs/Service.pdf + -> figs/System.pdf + -> tutorial-examples/*.c + -> gpl-3.0.texi + -> fdl-1.3.texi + +- gnunet-c-tutorial-v1.pdf: original LaTeX "gnunet-c-tutorial.pdf". +- man folder: the man pages. +- doxygen folder +- outdated-and-old-installation-instructions.txt: self described within the file. + + +Use `gendocs', add to the manual/ directory of the web site. + + $ cd doc + $ gendocs.sh gnunet "GNUnet 0.10.X Reference Manual" diff --git a/doc/documentation/chapters/configuration.texi b/doc/documentation/chapters/configuration.texi new file mode 100644 index 0000000000..286c72e7ab --- /dev/null +++ b/doc/documentation/chapters/configuration.texi @@ -0,0 +1,5 @@ +@node Configuration Handbook +@chapter Configuration Handbook + +This chapter has yet to be written. It is intended to be about in-depth +configuration of GNUnet. diff --git a/doc/documentation/chapters/contributing.texi b/doc/documentation/chapters/contributing.texi new file mode 100644 index 0000000000..5844fb497f --- /dev/null +++ b/doc/documentation/chapters/contributing.texi @@ -0,0 +1,102 @@ +@node GNUnet Contributors Handbook +@chapter GNUnet Contributors Handbook + +@menu +* Contributing to GNUnet:: +* Licenses of contributions:: +* Copyright Assignment:: +* Contributing to the Reference Manual:: +@end menu + +@node Contributing to GNUnet +@section Contributing to GNUnet + +@node Licenses of contributions +@section Licenses of contributions + +GNUnet is a @uref{https://www.gnu.org/, GNU} package. +All code contributions must thus be put under the +@uref{https://www.gnu.org/copyleft/gpl.html, GNU Public License (GPL)}. +All documentation should be put under FSF approved licenses +(see @uref{https://www.gnu.org/copyleft/fdl.html, fdl}). + +By submitting documentation, translations, and other content to GNUnet +you automatically grant the right to publish code under the +GNU Public License and documentation under either or both the +GNU Public License or the GNU Free Documentation License. +When contributing to the GNUnet project, GNU standards and the +@uref{https://www.gnu.org/philosophy/philosophy.html, GNU philosophy} +should be adhered to. + +@cindex copyright assignment +@node Copyright Assignment +@section Copyright Assignment +We require a formal copyright assignment for GNUnet contributors +to GNUnet e.V.; nevertheless, we do allow pseudonymous contributions. +By signing the copyright agreement and submitting your code (or +documentation) to us, you agree to share the rights to your code +with GNUnet e.V.; GNUnet e.V. receives non-exclusive ownership +rights, and in particular is allowed to dual-license the code. You +retain non-exclusive rights to your contributions, so you can also +share your contributions freely with other projects. + +GNUnet e.V. will publish all accepted contributions under the GPLv3 +or any later version. The association may decide to publish +contributions under additional licenses (dual-licensing). + +We do not intentionally remove your name from your contributions; +however, due to extensive editing it is not always trivial to +attribute contributors properly. If you find that you significantly +contributed to a file (or the project as a whole) and are not listed +in the respective authors file or section, please do let us know. + +@node Contributing to the Reference Manual +@section Contributing to the Reference Manual + +@itemize @bullet + +@item When writing documentation, please use +@uref{https://en.wikipedia.org/wiki/Singular_they, gender-neutral wording} +when referring to people, such as singular “theyâ€, “theirâ€, “themâ€, and so +forth. + +@item Keep line length below 74 characters, except for URLs. +URLs break in the PDF output when they contain linebreaks. + +@item Do not use tab characters (see chapter 2.1 texinfo manual) + +@item Use neutral language and third person perspective in the text + +@item So, when you refer to a user in general or addressing the user, +refer to (1). +@itemize @bullet +@item Unsolved exceptions for canonical reasons: When refering to Alice, +use "she". When refering to Bob, use "he". These are long established +examples and they should either be replaced (avoid Alice and Bob +examples when you can) or followed. +@end itemize + +@c FIXME: This is questionable, it feels like bike shed painging to do +@c this for several k lines. It only helps to jump between sentences in +@c editors afaik. +@c @item Use 2 spaces between sentences, so instead of: + +@c @example +@c We do this and the other thing. This is done by foo. +@c @end example + +@c Write: + +@c @example +@c We do this and the other thing. This is done by foo. +@c @end example + +@item Use @@footnote@{@} instead of putting an @@*ref@{@} to the +footnote on a collected footnote-page. +In a 200+ pages handbook it's better to have footnotes accessible +without having to skip over to the end. + +@item Avoid unnecessary footnotes, keep the text self-explanatory and +in a simple language where possible/necessary. + +@end itemize diff --git a/doc/documentation/chapters/developer.texi b/doc/documentation/chapters/developer.texi new file mode 100644 index 0000000000..70fd7c7ebe --- /dev/null +++ b/doc/documentation/chapters/developer.texi @@ -0,0 +1,8341 @@ +@c *********************************************************************** +@node GNUnet Developer Handbook +@chapter GNUnet Developer Handbook + +This book is intended to be an introduction for programmers that want to +extend the GNUnet framework. GNUnet is more than a simple peer-to-peer +application. + +For developers, GNUnet is: + +@itemize @bullet +@item developed by a community that believes in the GNU philosophy +@item Free Software (Free as in Freedom), licensed under the +GNU General Public License +@item A set of standards, including coding conventions and +architectural rules +@item A set of layered protocols, both specifying the communication +between peers as well as the communication between components +of a single peer +@item A set of libraries with well-defined APIs suitable for +writing extensions +@end itemize + +In particular, the architecture specifies that a peer consists of many +processes communicating via protocols. Processes can be written in almost +any language. +C and Java @footnote{As well as Guile} APIs exist for accessing existing +services and for writing extensions. +It is possible to write extensions in other languages by +implementing the necessary IPC protocols. + +GNUnet can be extended and improved along many possible dimensions, and +anyone interested in Free Software and Freedom-enhancing Networking is +welcome to join the effort. This Developer Handbook attempts to provide +an initial introduction to some of the key design choices and central +components of the system. +This part of the GNUNet documentation is far from complete, +and we welcome informed contributions, be it in the form of +new chapters, sections or insightful comments. + +@menu +* Developer Introduction:: +* Code overview:: +* System Architecture:: +* Subsystem stability:: +* Naming conventions and coding style guide:: +* Build-system:: +* Developing extensions for GNUnet using the gnunet-ext template:: +* Writing testcases:: +* TESTING library:: +* Performance regression analysis with Gauger:: +* TESTBED Subsystem:: +* libgnunetutil:: +* Automatic Restart Manager (ARM):: +* TRANSPORT Subsystem:: +* NAT library:: +* Distance-Vector plugin:: +* SMTP plugin:: +* Bluetooth plugin:: +* WLAN plugin:: +* ATS Subsystem:: +* CORE Subsystem:: +* CADET Subsystem:: +* NSE Subsystem:: +* HOSTLIST Subsystem:: +* IDENTITY Subsystem:: +* NAMESTORE Subsystem:: +* PEERINFO Subsystem:: +* PEERSTORE Subsystem:: +* SET Subsystem:: +* STATISTICS Subsystem:: +* Distributed Hash Table (DHT):: +* GNU Name System (GNS):: +* GNS Namecache:: +* REVOCATION Subsystem:: +* File-sharing (FS) Subsystem:: +* REGEX Subsystem:: +@end menu + +@node Developer Introduction +@section Developer Introduction + +This Developer Handbook is intended as first introduction to GNUnet for +new developers that want to extend the GNUnet framework. After the +introduction, each of the GNUnet subsystems (directories in the +@file{src/} tree) is (supposed to be) covered in its own chapter. In +addition to this documentation, GNUnet developers should be aware of the +services available on the GNUnet server to them. + +New developers can have a look a the GNUnet tutorials for C and java +available in the @file{src/} directory of the repository or under the +following links: + +@c ** FIXME: Link to files in source, not online. +@c ** FIXME: Where is the Java tutorial? +@itemize @bullet +@item @uref{https://gnunet.org/git/gnunet.git/plain/doc/gnunet-c-tutorial.pdf, GNUnet C tutorial} +@item GNUnet Java tutorial +@end itemize + +In addition to the GNUnet Reference Documentation you are reading, +the GNUnet server at @uref{https://gnunet.org} contains +various resources for GNUnet developers and those +who aspire to become regular contributors. +They are all conveniently reachable via the "Developer" +entry in the navigation menu. Some additional tools (such as static +analysis reports) require a special developer access to perform certain +operations. If you want (or require) access, you should contact +@uref{http://grothoff.org/christian/, Christian Grothoff}, +GNUnet's maintainer. + +The public subsystems on the GNUnet server that help developers are: + +@itemize @bullet + +@item The version control system (git) keeps our code and enables +distributed development. +It is pubclicly accessible at @uref{https://gnunet.org/git/}. +Only developers with write access can commit code, everyone else is +encouraged to submit patches to the +@uref{https://lists.gnu.org/mailman/listinfo/gnunet-developers, GNUnet-developers mailinglist}. + +@item The bugtracking system (Mantis). +We use it to track feature requests, open bug reports and their +resolutions. +It can be accessed at @uref{https://gnunet.org/bugs/}. +Anyone can report bugs, but only developers can claim to have fixed them. + +@item Our site installation of the +CI@footnote{Continuous Integration} system @code{Buildbot} is used +to check GNUnet builds automatically on a range of platforms. +The web interface of this CI is exposed at +@uref{https://gnunet.org/buildbot/}. +Builds are triggered automatically 30 minutes after the last commit to +our repository was made. + +@item The current quality of our automated test suite is assessed using +Code coverage analysis. This analysis is run daily; however the webpage +is only updated if all automated tests pass at that time. Testcases that +improve our code coverage are always welcome. + +@item We try to automatically find bugs using a static analysis scan. +This scan is run daily; however the webpage is only updated if all +automated tests pass at the time. Note that not everything that is +flagged by the analysis is a bug, sometimes even good code can be marked +as possibly problematic. Nevertheless, developers are encouraged to at +least be aware of all issues in their code that are listed. + +@item We use Gauger for automatic performance regression visualization. +Details on how to use Gauger are here. + +@item We use @uref{http://junit.org/, junit} to automatically test +@command{gnunet-java}. +Automatically generated, current reports on the test suite are here. + +@item We use Cobertura to generate test coverage reports for gnunet-java. +Current reports on test coverage are here. + +@end itemize + + + +@c *********************************************************************** +@menu +* Project overview:: +@end menu + +@node Project overview +@subsection Project overview + +The GNUnet project consists at this point of several sub-projects. This +section is supposed to give an initial overview about the various +sub-projects. Note that this description also lists projects that are far +from complete, including even those that have literally not a single line +of code in them yet. + +GNUnet sub-projects in order of likely relevance are currently: + +@table @asis + +@item @command{gnunet} +Core of the P2P framework, including file-sharing, VPN and +chat applications; this is what the Developer Handbook covers mostly +@item @command{gnunet-gtk} +Gtk+-based user interfaces, including: + +@itemize @bullet +@item @command{gnunet-fs-gtk} (file-sharing), +@item @command{gnunet-statistics-gtk} (statistics over time), +@item @command{gnunet-peerinfo-gtk} +(information about current connections and known peers), +@item @command{gnunet-chat-gtk} (chat GUI) and +@item @command{gnunet-setup} (setup tool for "everything") +@end itemize + +@item @command{gnunet-fuse} +Mounting directories shared via GNUnet's file-sharing +on GNU/Linux distributions +@item @command{gnunet-update} +Installation and update tool +@item @command{gnunet-ext} +Template for starting 'external' GNUnet projects +@item @command{gnunet-java} +Java APIs for writing GNUnet services and applications +@c ** FIXME: Point to new website repository once we have it: +@c ** @item svn/gnunet-www/ Code and media helping drive the GNUnet +@c website +@item @command{eclectic} +Code to run GNUnet nodes on testbeds for research, development, +testing and evaluation +@c ** FIXME: Solve the status and location of gnunet-qt +@item @command{gnunet-qt} +Qt-based GNUnet GUI (is it depreacated?) +@item @command{gnunet-cocoa} +cocoa-based GNUnet GUI (is it depreacated?) +@item @command{gnunet-guile} + +@end table + +We are also working on various supporting libraries and tools: +@c ** FIXME: What about gauger, and what about libmwmodem? + +@table @asis +@item @command{libextractor} +GNU libextractor (meta data extraction) +@item @command{libmicrohttpd} +GNU libmicrohttpd (embedded HTTP(S) server library) +@item @command{gauger} +Tool for performance regression analysis +@item @command{monkey} +Tool for automated debugging of distributed systems +@item @command{libmwmodem} +Library for accessing satellite connection quality +reports +@item @command{libgnurl} +gnURL (feature-restricted variant of cURL/libcurl) +@end table + +Finally, there are various external projects (see links for a list of +those that have a public website) which build on top of the GNUnet +framework. + +@c *********************************************************************** +@node Code overview +@section Code overview + +This section gives a brief overview of the GNUnet source code. +Specifically, we sketch the function of each of the subdirectories in +the @file{gnunet/src/} directory. The order given is roughly bottom-up +(in terms of the layers of the system). + +@table @asis +@item @file{util/} --- libgnunetutil +Library with general utility functions, all +GNUnet binaries link against this library. Anything from memory +allocation and data structures to cryptography and inter-process +communication. The goal is to provide an OS-independent interface and +more 'secure' or convenient implementations of commonly used primitives. +The API is spread over more than a dozen headers, developers should study +those closely to avoid duplicating existing functions. +@pxref{libgnunetutil}. +@item @file{hello/} --- libgnunethello +HELLO messages are used to +describe under which addresses a peer can be reached (for example, +protocol, IP, port). This library manages parsing and generating of HELLO +messages. +@item @file{block/} --- libgnunetblock +The DHT and other components of GNUnet +store information in units called 'blocks'. Each block has a type and the +type defines a particular format and how that binary format is to be +linked to a hash code (the key for the DHT and for databases). The block +library is a wapper around block plugins which provide the necessary +functions for each block type. +@item @file{statistics/} --- statistics service +The statistics service enables associating +values (of type uint64_t) with a componenet name and a string. The main +uses is debugging (counting events), performance tracking and user +entertainment (what did my peer do today?). +@item @file{arm/} --- Automatic Restart Manager (ARM) +The automatic-restart-manager (ARM) service +is the GNUnet master service. Its role is to start gnunet-services, to +re-start them when they crashed and finally to shut down the system when +requested. +@item @file{peerinfo/} --- peerinfo service +The peerinfo service keeps track of which peers are known +to the local peer and also tracks the validated addresses for each peer +(in the form of a HELLO message) for each of those peers. The peer is not +necessarily connected to all peers known to the peerinfo service. +Peerinfo provides persistent storage for peer identities --- peers are +not forgotten just because of a system restart. +@item @file{datacache/} --- libgnunetdatacache +The datacache library provides (temporary) block storage for the DHT. +Existing plugins can store blocks in Sqlite, Postgres or MySQL databases. +All data stored in the cache is lost when the peer is stopped or +restarted (datacache uses temporary tables). +@item @file{datastore/} --- datastore service +The datastore service stores file-sharing blocks in +databases for extended periods of time. In contrast to the datacache, data +is not lost when peers restart. However, quota restrictions may still +cause old, expired or low-priority data to be eventually discarded. +Existing plugins can store blocks in Sqlite, Postgres or MySQL databases. +@item @file{template/} --- service template +Template for writing a new service. Does nothing. +@item @file{ats/} --- Automatic Transport Selection +The automatic transport selection (ATS) service +is responsible for deciding which address (i.e. +which transport plugin) should be used for communication with other peers, +and at what bandwidth. +@item @file{nat/} --- libgnunetnat +Library that provides basic functions for NAT traversal. +The library supports NAT traversal with +manual hole-punching by the user, UPnP and ICMP-based autonomous NAT +traversal. The library also includes an API for testing if the current +configuration works and the @code{gnunet-nat-server} which provides an +external service to test the local configuration. +@item @file{fragmentation/} --- libgnunetfragmentation +Some transports (UDP and WLAN, mostly) have restrictions on the maximum +transfer unit (MTU) for packets. The fragmentation library can be used to +break larger packets into chunks of at most 1k and transmit the resulting +fragments reliabily (with acknowledgement, retransmission, timeouts, +etc.). +@item @file{transport/} --- transport service +The transport service is responsible for managing the +basic P2P communication. It uses plugins to support P2P communication +over TCP, UDP, HTTP, HTTPS and other protocols.The transport service +validates peer addresses, enforces bandwidth restrictions, limits the +total number of connections and enforces connectivity restrictions (i.e. +friends-only). +@item @file{peerinfo-tool/} --- gnunet-peerinfo +This directory contains the gnunet-peerinfo binary which can be used to +inspect the peers and HELLOs known to the peerinfo service. +@item @file{core/} +The core service is responsible for establishing encrypted, authenticated +connections with other peers, encrypting and decrypting messages and +forwarding messages to higher-level services that are interested in them. +@item @file{testing/} --- libgnunettesting +The testing library allows starting (and stopping) peers +for writing testcases. +It also supports automatic generation of configurations for peers +ensuring that the ports and paths are disjoint. libgnunettesting is also +the foundation for the testbed service +@item @file{testbed/} --- testbed service +The testbed service is used for creating small or large scale deployments +of GNUnet peers for evaluation of protocols. +It facilitates peer depolyments on multiple +hosts (for example, in a cluster) and establishing varous network +topologies (both underlay and overlay). +@item @file{nse/} --- Network Size Estimation +The network size estimation (NSE) service +implements a protocol for (securely) estimating the current size of the +P2P network. +@item @file{dht/} --- distributed hash table +The distributed hash table (DHT) service provides a +distributed implementation of a hash table to store blocks under hash +keys in the P2P network. +@item @file{hostlist/} --- hostlist service +The hostlist service allows learning about +other peers in the network by downloading HELLO messages from an HTTP +server, can be configured to run such an HTTP server and also implements +a P2P protocol to advertise and automatically learn about other peers +that offer a public hostlist server. +@item @file{topology/} --- topology service +The topology service is responsible for +maintaining the mesh topology. It tries to maintain connections to friends +(depending on the configuration) and also tries to ensure that the peer +has a decent number of active connections at all times. If necessary, new +connections are added. All peers should run the topology service, +otherwise they may end up not being connected to any other peer (unless +some other service ensures that core establishes the required +connections). The topology service also tells the transport service which +connections are permitted (for friend-to-friend networking) +@item @file{fs/} --- file-sharing +The file-sharing (FS) service implements GNUnet's +file-sharing application. Both anonymous file-sharing (using gap) and +non-anonymous file-sharing (using dht) are supported. +@item @file{cadet/} --- cadet service +The CADET service provides a general-purpose routing abstraction to create +end-to-end encrypted tunnels in mesh networks. We wrote a paper +documenting key aspects of the design. +@item @file{tun/} --- libgnunettun +Library for building IPv4, IPv6 packets and creating +checksums for UDP, TCP and ICMP packets. The header +defines C structs for common Internet packet formats and in particular +structs for interacting with TUN (virtual network) interfaces. +@item @file{mysql/} --- libgnunetmysql +Library for creating and executing prepared MySQL +statements and to manage the connection to the MySQL database. +Essentially a lightweight wrapper for the interaction between GNUnet +components and libmysqlclient. +@item @file{dns/} +Service that allows intercepting and modifying DNS requests of +the local machine. Currently used for IPv4-IPv6 protocol translation +(DNS-ALG) as implemented by "pt/" and for the GNUnet naming system. The +service can also be configured to offer an exit service for DNS traffic. +@item @file{vpn/} --- VPN service +The virtual public network (VPN) service provides a virtual +tunnel interface (VTUN) for IP routing over GNUnet. +Needs some other peers to run an "exit" service to work. +Can be activated using the "gnunet-vpn" tool or integrated with DNS using +the "pt" daemon. +@item @file{exit/} +Daemon to allow traffic from the VPN to exit this +peer to the Internet or to specific IP-based services of the local peer. +Currently, an exit service can only be restricted to IPv4 or IPv6, not to +specific ports and or IP address ranges. If this is not acceptable, +additional firewall rules must be added manually. exit currently only +works for normal UDP, TCP and ICMP traffic; DNS queries need to leave the +system via a DNS service. +@item @file{pt/} +protocol translation daemon. This daemon enables 4-to-6, +6-to-4, 4-over-6 or 6-over-4 transitions for the local system. It +essentially uses "DNS" to intercept DNS replies and then maps results to +those offered by the VPN, which then sends them using mesh to some daemon +offering an appropriate exit service. +@item @file{identity/} +Management of egos (alter egos) of a user; identities are +essentially named ECC private keys and used for zones in the GNU name +system and for namespaces in file-sharing, but might find other uses later +@item @file{revocation/} +Key revocation service, can be used to revoke the +private key of an identity if it has been compromised +@item @file{namecache/} +Cache for resolution results for the GNU name system; +data is encrypted and can be shared among users, +loss of the data should ideally only result in a +performance degradation (persistence not required) +@item @file{namestore/} +Database for the GNU name system with per-user private information, +persistence required +@item @file{gns/} +GNU name system, a GNU approach to DNS and PKI. +@item @file{dv/} +A plugin for distance-vector (DV)-based routing. +DV consists of a service and a transport plugin to provide peers +with the illusion of a direct P2P connection for connections +that use multiple (typically up to 3) hops in the actual underlay network. +@item @file{regex/} +Service for the (distributed) evaluation of regular expressions. +@item @file{scalarproduct/} +The scalar product service offers an API to perform a secure multiparty +computation which calculates a scalar product between two peers +without exposing the private input vectors of the peers to each other. +@item @file{consensus/} +The consensus service will allow a set of peers to agree +on a set of values via a distributed set union computation. +@item @file{rest/} +The rest API allows access to GNUnet services using RESTful interaction. +The services provide plugins that can exposed by the rest server. +@item @file{experimentation/} +The experimentation daemon coordinates distributed +experimentation to evaluate transport and ATS properties. +@end table + +@c *********************************************************************** +@node System Architecture +@section System Architecture + +GNUnet developers like LEGOs. The blocks are indestructible, can be +stacked together to construct complex buildings and it is generally easy +to swap one block for a different one that has the same shape. GNUnet's +architecture is based on LEGOs: + +@c images here + +This chapter documents the GNUnet LEGO system, also known as GNUnet's +system architecture. + +The most common GNUnet component is a service. Services offer an API (or +several, depending on what you count as "an API") which is implemented as +a library. The library communicates with the main process of the service +using a service-specific network protocol. The main process of the service +typically doesn't fully provide everything that is needed --- it has holes +to be filled by APIs to other services. + +A special kind of component in GNUnet are user interfaces and daemons. +Like services, they have holes to be filled by APIs of other services. +Unlike services, daemons do not implement their own network protocol and +they have no API: + +The GNUnet system provides a range of services, daemons and user +interfaces, which are then combined into a layered GNUnet instance (also +known as a peer). + +Note that while it is generally possible to swap one service for another +compatible service, there is often only one implementation. However, +during development we often have a "new" version of a service in parallel +with an "old" version. While the "new" version is not working, developers +working on other parts of the service can continue their development by +simply using the "old" service. Alternative design ideas can also be +easily investigated by swapping out individual components. This is +typically achieved by simply changing the name of the "BINARY" in the +respective configuration section. + +Key properties of GNUnet services are that they must be separate +processes and that they must protect themselves by applying tight error +checking against the network protocol they implement (thereby achieving a +certain degree of robustness). + +On the other hand, the APIs are implemented to tolerate failures of the +service, isolating their host process from errors by the service. If the +service process crashes, other services and daemons around it should not +also fail, but instead wait for the service process to be restarted by +ARM. + + +@c *********************************************************************** +@node Subsystem stability +@section Subsystem stability + +This section documents the current stability of the various GNUnet +subsystems. Stability here describes the expected degree of compatibility +with future versions of GNUnet. For each subsystem we distinguish between +compatibility on the P2P network level (communication protocol between +peers), the IPC level (communication between the service and the service +library) and the API level (stability of the API). P2P compatibility is +relevant in terms of which applications are likely going to be able to +communicate with future versions of the network. IPC communication is +relevant for the implementation of language bindings that re-implement the +IPC messages. Finally, API compatibility is relevant to developers that +hope to be able to avoid changes to applications build on top of the APIs +of the framework. + +The following table summarizes our current view of the stability of the +respective protocols or APIs: + +@multitable @columnfractions .20 .20 .20 .20 +@headitem Subsystem @tab P2P @tab IPC @tab C API +@item util @tab n/a @tab n/a @tab stable +@item arm @tab n/a @tab stable @tab stable +@item ats @tab n/a @tab unstable @tab testing +@item block @tab n/a @tab n/a @tab stable +@item cadet @tab testing @tab testing @tab testing +@item consensus @tab experimental @tab experimental @tab experimental +@item core @tab stable @tab stable @tab stable +@item datacache @tab n/a @tab n/a @tab stable +@item datastore @tab n/a @tab stable @tab stable +@item dht @tab stable @tab stable @tab stable +@item dns @tab stable @tab stable @tab stable +@item dv @tab testing @tab testing @tab n/a +@item exit @tab testing @tab n/a @tab n/a +@item fragmentation @tab stable @tab n/a @tab stable +@item fs @tab stable @tab stable @tab stable +@item gns @tab stable @tab stable @tab stable +@item hello @tab n/a @tab n/a @tab testing +@item hostlist @tab stable @tab stable @tab n/a +@item identity @tab stable @tab stable @tab n/a +@item multicast @tab experimental @tab experimental @tab experimental +@item mysql @tab stable @tab n/a @tab stable +@item namestore @tab n/a @tab stable @tab stable +@item nat @tab n/a @tab n/a @tab stable +@item nse @tab stable @tab stable @tab stable +@item peerinfo @tab n/a @tab stable @tab stable +@item psyc @tab experimental @tab experimental @tab experimental +@item pt @tab n/a @tab n/a @tab n/a +@item regex @tab stable @tab stable @tab stable +@item revocation @tab stable @tab stable @tab stable +@item social @tab experimental @tab experimental @tab experimental +@item statistics @tab n/a @tab stable @tab stable +@item testbed @tab n/a @tab testing @tab testing +@item testing @tab n/a @tab n/a @tab testing +@item topology @tab n/a @tab n/a @tab n/a +@item transport @tab stable @tab stable @tab stable +@item tun @tab n/a @tab n/a @tab stable +@item vpn @tab testing @tab n/a @tab n/a +@end multitable + +Here is a rough explanation of the values: + +@table @samp +@item stable +No incompatible changes are planned at this time; for IPC/APIs, if +there are incompatible changes, they will be minor and might only require +minimal changes to existing code; for P2P, changes will be avoided if at +all possible for the 0.10.x-series + +@item testing +No incompatible changes are +planned at this time, but the code is still known to be in flux; so while +we have no concrete plans, our expectation is that there will still be +minor modifications; for P2P, changes will likely be extensions that +should not break existing code + +@item unstable +Changes are planned and will happen; however, they +will not be totally radical and the result should still resemble what is +there now; nevertheless, anticipated changes will break protocol/API +compatibility + +@item experimental +Changes are planned and the result may look nothing like +what the API/protocol looks like today + +@item unknown +Someone should think about where this subsystem headed + +@item n/a +This subsystem does not have an API/IPC-protocol/P2P-protocol +@end table + +@c *********************************************************************** +@node Naming conventions and coding style guide +@section Naming conventions and coding style guide + +Here you can find some rules to help you write code for GNUnet. + +@c *********************************************************************** +@menu +* Naming conventions:: +* Coding style:: +@end menu + +@node Naming conventions +@subsection Naming conventions + + +@c *********************************************************************** +@menu +* include files:: +* binaries:: +* logging:: +* configuration:: +* exported symbols:: +* private (library-internal) symbols (including structs and macros):: +* testcases:: +* performance tests:: +* src/ directories:: +@end menu + +@node include files +@subsubsection include files + +@itemize @bullet +@item _lib: library without need for a process +@item _service: library that needs a service process +@item _plugin: plugin definition +@item _protocol: structs used in network protocol +@item exceptions: +@itemize @bullet +@item gnunet_config.h --- generated +@item platform.h --- first included +@item plibc.h --- external library +@item gnunet_common.h --- fundamental routines +@item gnunet_directories.h --- generated +@item gettext.h --- external library +@end itemize +@end itemize + +@c *********************************************************************** +@node binaries +@subsubsection binaries + +@itemize @bullet +@item gnunet-service-xxx: service process (has listen socket) +@item gnunet-daemon-xxx: daemon process (no listen socket) +@item gnunet-helper-xxx[-yyy]: SUID helper for module xxx +@item gnunet-yyy: command-line tool for end-users +@item libgnunet_plugin_xxx_yyy.so: plugin for API xxx +@item libgnunetxxx.so: library for API xxx +@end itemize + +@c *********************************************************************** +@node logging +@subsubsection logging + +@itemize @bullet +@item services and daemons use their directory name in +@code{GNUNET_log_setup} (i.e. 'core') and log using +plain 'GNUNET_log'. +@item command-line tools use their full name in +@code{GNUNET_log_setup} (i.e. 'gnunet-publish') and log using +plain 'GNUNET_log'. +@item service access libraries log using +'@code{GNUNET_log_from}' and use '@code{DIRNAME-api}' for the +component (i.e. 'core-api') +@item pure libraries (without associated service) use +'@code{GNUNET_log_from}' with the component set to their +library name (without lib or '@file{.so}'), +which should also be their directory name (i.e. '@file{nat}') +@item plugins should use '@code{GNUNET_log_from}' +with the directory name and the plugin name combined to produce +the component name (i.e. 'transport-tcp'). +@item logging should be unified per-file by defining a +@code{LOG} macro with the appropriate arguments, +along these lines: + +@example +#define LOG(kind,...) +GNUNET_log_from (kind, "example-api",__VA_ARGS__) +@end example + +@end itemize + +@c *********************************************************************** +@node configuration +@subsubsection configuration + +@itemize @bullet +@item paths (that are substituted in all filenames) are in PATHS +(have as few as possible) +@item all options for a particular module (@file{src/MODULE}) +are under @code{[MODULE]} +@item options for a plugin of a module +are under @code{[MODULE-PLUGINNAME]} +@end itemize + +@c *********************************************************************** +@node exported symbols +@subsubsection exported symbols + +@itemize @bullet +@item must start with @code{GNUNET_modulename_} and be defined in +@file{modulename.c} +@item exceptions: those defined in @file{gnunet_common.h} +@end itemize + +@c *********************************************************************** +@node private (library-internal) symbols (including structs and macros) +@subsubsection private (library-internal) symbols (including structs and macros) + +@itemize @bullet +@item must NOT start with any prefix +@item must not be exported in a way that linkers could use them or@ other +libraries might see them via headers; they must be either +declared/defined in C source files or in headers that are in the +respective directory under @file{src/modulename/} and NEVER be declared +in @file{src/include/}. +@end itemize + +@node testcases +@subsubsection testcases + +@itemize @bullet +@item must be called @file{test_module-under-test_case-description.c} +@item "case-description" maybe omitted if there is only one test +@end itemize + +@c *********************************************************************** +@node performance tests +@subsubsection performance tests + +@itemize @bullet +@item must be called @file{perf_module-under-test_case-description.c} +@item "case-description" maybe omitted if there is only one performance +test +@item Must only be run if @code{HAVE_BENCHMARKS} is satisfied +@end itemize + +@c *********************************************************************** +@node src/ directories +@subsubsection src/ directories + +@itemize @bullet +@item gnunet-NAME: end-user applications (i.e., gnunet-search, gnunet-arm) +@item gnunet-service-NAME: service processes with accessor library (i.e., +gnunet-service-arm) +@item libgnunetNAME: accessor library (_service.h-header) or standalone +library (_lib.h-header) +@item gnunet-daemon-NAME: daemon process without accessor library (i.e., +gnunet-daemon-hostlist) and no GNUnet management port +@item libgnunet_plugin_DIR_NAME: loadable plugins (i.e., +libgnunet_plugin_transport_tcp) +@end itemize + +@cindex Coding style +@node Coding style +@subsection Coding style + +@c XXX: Adjust examples to GNU Standards! +@itemize @bullet +@item We follow the GNU Coding Standards (@pxref{Top, The GNU Coding Standards,, standards, The GNU Coding Standards}); +@item Indentation is done with spaces, two per level, no tabs; +@item C99 struct initialization is fine; +@item declare only one variable per line, for example: + +@noindent +instead of + +@example +int i,j; +@end example + +@noindent +write: + +@example +int i; +int j; +@end example + +@c TODO: include actual example from a file in source + +@noindent +This helps keep diffs small and forces developers to think precisely about +the type of every variable. +Note that @code{char *} is different from @code{const char*} and +@code{int} is different from @code{unsigned int} or @code{uint32_t}. +Each variable type should be chosen with care. + +@item While @code{goto} should generally be avoided, having a +@code{goto} to the end of a function to a block of clean up +statements (free, close, etc.) can be acceptable. + +@item Conditions should be written with constants on the left (to avoid +accidental assignment) and with the 'true' target being either the +'error' case or the significantly simpler continuation. For example: + +@example +if (0 != stat ("filename," &sbuf)) @{ + error(); + @} + else @{ + /* handle normal case here */ + @} +@end example + +@noindent +instead of + +@example +if (stat ("filename," &sbuf) == 0) @{ + /* handle normal case here */ + @} else @{ + error(); + @} +@end example + +@noindent +If possible, the error clause should be terminated with a 'return' (or +'goto' to some cleanup routine) and in this case, the 'else' clause +should be omitted: + +@example +if (0 != stat ("filename," &sbuf)) @{ + error(); + return; + @} +/* handle normal case here */ +@end example + +This serves to avoid deep nesting. The 'constants on the left' rule +applies to all constants (including. @code{GNUNET_SCHEDULER_NO_TASK}), +NULL, and enums). With the two above rules (constants on left, errors in +'true' branch), there is only one way to write most branches correctly. + +@item Combined assignments and tests are allowed if they do not hinder +code clarity. For example, one can write: + +@example +if (NULL == (value = lookup_function())) @{ + error(); + return; + @} +@end example + +@item Use @code{break} and @code{continue} wherever possible to avoid +deep(er) nesting. Thus, we would write: + +@example +next = head; +while (NULL != (pos = next)) @{ + next = pos->next; + if (! should_free (pos)) + continue; + GNUNET_CONTAINER_DLL_remove (head, tail, pos); + GNUNET_free (pos); + @} +@end example + +instead of + +@example +next = head; while (NULL != (pos = next)) @{ + next = pos->next; + if (should_free (pos)) @{ + /* unnecessary nesting! */ + GNUNET_CONTAINER_DLL_remove (head, tail, pos); + GNUNET_free (pos); + @} + @} +@end example + +@item We primarily use @code{for} and @code{while} loops. +A @code{while} loop is used if the method for advancing in the loop is +not a straightforward increment operation. In particular, we use: + +@example +next = head; +while (NULL != (pos = next)) +@{ + next = pos->next; + if (! should_free (pos)) + continue; + GNUNET_CONTAINER_DLL_remove (head, tail, pos); + GNUNET_free (pos); +@} +@end example + +to free entries in a list (as the iteration changes the structure of the +list due to the free; the equivalent @code{for} loop does no longer +follow the simple @code{for} paradigm of @code{for(INIT;TEST;INC)}). +However, for loops that do follow the simple @code{for} paradigm we do +use @code{for}, even if it involves linked lists: + +@example +/* simple iteration over a linked list */ +for (pos = head; + NULL != pos; + pos = pos->next) +@{ + use (pos); +@} +@end example + + +@item The first argument to all higher-order functions in GNUnet must be +declared to be of type @code{void *} and is reserved for a closure. We do +not use inner functions, as trampolines would conflict with setups that +use non-executable stacks. +The first statement in a higher-order function, which unusually should +be part of the variable declarations, should assign the +@code{cls} argument to the precise expected type. For example: + +@example +int callback (void *cls, char *args) @{ + struct Foo *foo = cls; + int other_variables; + + /* rest of function */ +@} +@end example + + +@item It is good practice to write complex @code{if} expressions instead +of using deeply nested @code{if} statements. However, except for addition +and multiplication, all operators should use parens. This is fine: + +@example +if ( (1 == foo) || ((0 == bar) && (x != y)) ) + return x; +@end example + + +However, this is not: + +@example +if (1 == foo) + return x; +if (0 == bar && x != y) + return x; +@end example + +@noindent +Note that splitting the @code{if} statement above is debateable as the +@code{return x} is a very trivial statement. However, once the logic after +the branch becomes more complicated (and is still identical), the "or" +formulation should be used for sure. + +@item There should be two empty lines between the end of the function and +the comments describing the following function. There should be a single +empty line after the initial variable declarations of a function. If a +function has no local variables, there should be no initial empty line. If +a long function consists of several complex steps, those steps might be +separated by an empty line (possibly followed by a comment describing the +following step). The code should not contain empty lines in arbitrary +places; if in doubt, it is likely better to NOT have an empty line (this +way, more code will fit on the screen). +@end itemize + +@c *********************************************************************** +@node Build-system +@section Build-system + +If you have code that is likely not to compile or build rules you might +want to not trigger for most developers, use @code{if HAVE_EXPERIMENTAL} +in your @file{Makefile.am}. +Then it is OK to (temporarily) add non-compiling (or known-to-not-port) +code. + +If you want to compile all testcases but NOT run them, run configure with +the @code{--enable-test-suppression} option. + +If you want to run all testcases, including those that take a while, run +configure with the @code{--enable-expensive-testcases} option. + +If you want to compile and run benchmarks, run configure with the +@code{--enable-benchmarks} option. + +If you want to obtain code coverage results, run configure with the +@code{--enable-coverage} option and run the @file{coverage.sh} script in +the @file{contrib/} directory. + +@cindex gnunet-ext +@node Developing extensions for GNUnet using the gnunet-ext template +@section Developing extensions for GNUnet using the gnunet-ext template + +For developers who want to write extensions for GNUnet we provide the +gnunet-ext template to provide an easy to use skeleton. + +gnunet-ext contains the build environment and template files for the +development of GNUnet services, command line tools, APIs and tests. + +First of all you have to obtain gnunet-ext from git: + +@example +git clone https://gnunet.org/git/gnunet-ext.git +@end example + +The next step is to bootstrap and configure it. For configure you have to +provide the path containing GNUnet with +@code{--with-gnunet=/path/to/gnunet} and the prefix where you want the +install the extension using @code{--prefix=/path/to/install}: + +@example +./bootstrap +./configure --prefix=/path/to/install --with-gnunet=/path/to/gnunet +@end example + +When your GNUnet installation is not included in the default linker search +path, you have to add @code{/path/to/gnunet} to the file +@file{/etc/ld.so.conf} and run @code{ldconfig} or your add it to the +environmental variable @code{LD_LIBRARY_PATH} by using + +@example +export LD_LIBRARY_PATH=/path/to/gnunet/lib +@end example + +@cindex writing testcases +@node Writing testcases +@section Writing testcases + +Ideally, any non-trivial GNUnet code should be covered by automated +testcases. Testcases should reside in the same place as the code that is +being tested. The name of source files implementing tests should begin +with @code{test_} followed by the name of the file that contains +the code that is being tested. + +Testcases in GNUnet should be integrated with the autotools build system. +This way, developers and anyone building binary packages will be able to +run all testcases simply by running @code{make check}. The final +testcases shipped with the distribution should output at most some brief +progress information and not display debug messages by default. The +success or failure of a testcase must be indicated by returning zero +(success) or non-zero (failure) from the main method of the testcase. +The integration with the autotools is relatively straightforward and only +requires modifications to the @file{Makefile.am} in the directory +containing the testcase. For a testcase testing the code in @file{foo.c} +the @file{Makefile.am} would contain the following lines: + +@example +check_PROGRAMS = test_foo +TESTS = $(check_PROGRAMS) +test_foo_SOURCES = test_foo.c +test_foo_LDADD = $(top_builddir)/src/util/libgnunetutil.la +@end example + +Naturally, other libraries used by the testcase may be specified in the +@code{LDADD} directive as necessary. + +Often testcases depend on additional input files, such as a configuration +file. These support files have to be listed using the @code{EXTRA_DIST} +directive in order to ensure that they are included in the distribution. + +Example: + +@example +EXTRA_DIST = test_foo_data.conf +@end example + +Executing @code{make check} will run all testcases in the current +directory and all subdirectories. Testcases can be compiled individually +by running @code{make test_foo} and then invoked directly using +@code{./test_foo}. Note that due to the use of plugins in GNUnet, it is +typically necessary to run @code{make install} before running any +testcases. Thus the canonical command @code{make check install} has to be +changed to @code{make install check} for GNUnet. + +@cindex TESTING library +@node TESTING library +@section TESTING library + +The TESTING library is used for writing testcases which involve starting a +single or multiple peers. While peers can also be started by testcases +using the ARM subsystem, using TESTING library provides an elegant way to +do this. The configurations of the peers are auto-generated from a given +template to have non-conflicting port numbers ensuring that peers' +services do not run into bind errors. This is achieved by testing ports' +availability by binding a listening socket to them before allocating them +to services in the generated configurations. + +An another advantage while using TESTING is that it shortens the testcase +startup time as the hostkeys for peers are copied from a pre-computed set +of hostkeys instead of generating them at peer startup which may take a +considerable amount of time when starting multiple peers or on an embedded +processor. + +TESTING also allows for certain services to be shared among peers. This +feature is invaluable when testing with multiple peers as it helps to +reduce the number of services run per each peer and hence the total +number of processes run per testcase. + +TESTING library only handles creating, starting and stopping peers. +Features useful for testcases such as connecting peers in a topology are +not available in TESTING but are available in the TESTBED subsystem. +Furthermore, TESTING only creates peers on the localhost, however by +using TESTBED testcases can benefit from creating peers across multiple +hosts. + +@menu +* API:: +* Finer control over peer stop:: +* Helper functions:: +* Testing with multiple processes:: +@end menu + +@cindex TESTING API +@node API +@subsection API + +TESTING abstracts a group of peers as a TESTING system. All peers in a +system have common hostname and no two services of these peers have a +same port or a UNIX domain socket path. + +TESTING system can be created with the function +@code{GNUNET_TESTING_system_create()} which returns a handle to the +system. This function takes a directory path which is used for generating +the configurations of peers, an IP address from which connections to the +peers' services should be allowed, the hostname to be used in peers' +configuration, and an array of shared service specifications of type +@code{struct GNUNET_TESTING_SharedService}. + +The shared service specification must specify the name of the service to +share, the configuration pertaining to that shared service and the +maximum number of peers that are allowed to share a single instance of +the shared service. + +TESTING system created with @code{GNUNET_TESTING_system_create()} chooses +ports from the default range @code{12000} - @code{56000} while +auto-generating configurations for peers. +This range can be customised with the function +@code{GNUNET_TESTING_system_create_with_portrange()}. This function is +similar to @code{GNUNET_TESTING_system_create()} except that it take 2 +additional parameters --- the start and end of the port range to use. + +A TESTING system is destroyed with the funciton +@code{GNUNET_TESTING_system_destory()}. This function takes the handle of +the system and a flag to remove the files created in the directory used +to generate configurations. + +A peer is created with the function +@code{GNUNET_TESTING_peer_configure()}. This functions takes the system +handle, a configuration template from which the configuration for the peer +is auto-generated and the index from where the hostkey for the peer has to +be copied from. When successfull, this function returs a handle to the +peer which can be used to start and stop it and to obtain the identity of +the peer. If unsuccessful, a NULL pointer is returned with an error +message. This function handles the generated configuration to have +non-conflicting ports and paths. + +Peers can be started and stopped by calling the functions +@code{GNUNET_TESTING_peer_start()} and @code{GNUNET_TESTING_peer_stop()} +respectively. A peer can be destroyed by calling the function +@code{GNUNET_TESTING_peer_destroy}. When a peer is destroyed, the ports +and paths in allocated in its configuration are reclaimed for usage in new +peers. + +@c *********************************************************************** +@node Finer control over peer stop +@subsection Finer control over peer stop + +Using @code{GNUNET_TESTING_peer_stop()} is normally fine for testcases. +However, calling this function for each peer is inefficient when trying to +shutdown multiple peers as this function sends the termination signal to +the given peer process and waits for it to terminate. It would be faster +in this case to send the termination signals to the peers first and then +wait on them. This is accomplished by the functions +@code{GNUNET_TESTING_peer_kill()} which sends a termination signal to the +peer, and the function @code{GNUNET_TESTING_peer_wait()} which waits on +the peer. + +Further finer control can be achieved by choosing to stop a peer +asynchronously with the function @code{GNUNET_TESTING_peer_stop_async()}. +This function takes a callback parameter and a closure for it in addition +to the handle to the peer to stop. The callback function is called with +the given closure when the peer is stopped. Using this function +eliminates blocking while waiting for the peer to terminate. + +An asynchronous peer stop can be cancelled by calling the function +@code{GNUNET_TESTING_peer_stop_async_cancel()}. Note that calling this +function does not prevent the peer from terminating if the termination +signal has already been sent to it. It does, however, cancels the +callback to be called when the peer is stopped. + +@c *********************************************************************** +@node Helper functions +@subsection Helper functions + +Most of the testcases can benefit from an abstraction which configures a +peer and starts it. This is provided by the function +@code{GNUNET_TESTING_peer_run()}. This function takes the testing +directory pathname, a configuration template, a callback and its closure. +This function creates a peer in the given testing directory by using the +configuration template, starts the peer and calls the given callback with +the given closure. + +The function @code{GNUNET_TESTING_peer_run()} starts the ARM service of +the peer which starts the rest of the configured services. A similar +function @code{GNUNET_TESTING_service_run} can be used to just start a +single service of a peer. In this case, the peer's ARM service is not +started; instead, only the given service is run. + +@c *********************************************************************** +@node Testing with multiple processes +@subsection Testing with multiple processes + +When testing GNUnet, the splitting of the code into a services and clients +often complicates testing. The solution to this is to have the testcase +fork @code{gnunet-service-arm}, ask it to start the required server and +daemon processes and then execute appropriate client actions (to test the +client APIs or the core module or both). If necessary, multiple ARM +services can be forked using different ports (!) to simulate a network. +However, most of the time only one ARM process is needed. Note that on +exit, the testcase should shutdown ARM with a @code{TERM} signal (to give +it the chance to cleanly stop its child processes). + +The following code illustrates spawning and killing an ARM process from a +testcase: + +@example +static void run (void *cls, + char *const *args, + const char *cfgfile, + const struct GNUNET_CONFIGURATION_Handle *cfg) @{ + struct GNUNET_OS_Process *arm_pid; + arm_pid = GNUNET_OS_start_process (NULL, + NULL, + "gnunet-service-arm", + "gnunet-service-arm", + "-c", + cfgname, + NULL); + /* do real test work here */ + if (0 != GNUNET_OS_process_kill (arm_pid, SIGTERM)) + GNUNET_log_strerror + (GNUNET_ERROR_TYPE_WARNING, "kill"); + GNUNET_assert (GNUNET_OK == GNUNET_OS_process_wait (arm_pid)); + GNUNET_OS_process_close (arm_pid); @} + +GNUNET_PROGRAM_run (argc, argv, + "NAME-OF-TEST", + "nohelp", + options, + &run, + cls); +@end example + + +An alternative way that works well to test plugins is to implement a +mock-version of the environment that the plugin expects and then to +simply load the plugin directly. + +@c *********************************************************************** +@node Performance regression analysis with Gauger +@section Performance regression analysis with Gauger + +To help avoid performance regressions, GNUnet uses Gauger. Gauger is a +simple logging tool that allows remote hosts to send performance data to +a central server, where this data can be analyzed and visualized. Gauger +shows graphs of the repository revisions and the performace data recorded +for each revision, so sudden performance peaks or drops can be identified +and linked to a specific revision number. + +In the case of GNUnet, the buildbots log the performance data obtained +during the tests after each build. The data can be accesed on GNUnet's +Gauger page. + +The menu on the left allows to select either the results of just one +build bot (under "Hosts") or review the data from all hosts for a given +test result (under "Metrics"). In case of very different absolute value +of the results, for instance arm vs. amd64 machines, the option +"Normalize" on a metric view can help to get an idea about the +performance evolution across all hosts. + +Using Gauger in GNUnet and having the performance of a module tracked over +time is very easy. First of course, the testcase must generate some +consistent metric, which makes sense to have logged. Highly volatile or +random dependant metrics probably are not ideal candidates for meaningful +regression detection. + +To start logging any value, just include @code{gauger.h} in your testcase +code. Then, use the macro @code{GAUGER()} to make the Buildbots log +whatever value is of interest for you to @code{gnunet.org}'s Gauger +server. No setup is necessary as most Buildbots have already everything +in place and new metrics are created on demand. To delete a metric, you +need to contact a member of the GNUnet development team (a file will need +to be removed manually from the respective directory). + +The code in the test should look like this: + +@example +[other includes] +#include <gauger.h> + +int main (int argc, char *argv[]) @{ + + [run test, generate data] + GAUGER("YOUR_MODULE", + "METRIC_NAME", + (float)value, + "UNIT"); @} +@end example + +Where: + +@table @asis + +@item @strong{YOUR_MODULE} is a category in the gauger page and should be +the name of the module or subsystem like "Core" or "DHT" +@item @strong{METRIC} is +the name of the metric being collected and should be concise and +descriptive, like "PUT operations in sqlite-datastore". +@item @strong{value} is the value +of the metric that is logged for this run. +@item @strong{UNIT} is the unit in +which the value is measured, for instance "kb/s" or "kb of RAM/node". +@end table + +If you wish to use Gauger for your own project, you can grab a copy of the +latest stable release or check out Gauger's Subversion repository. + +@cindex TESTBED Subsystem +@node TESTBED Subsystem +@section TESTBED Subsystem + +The TESTBED subsystem facilitates testing and measuring of multi-peer +deployments on a single host or over multiple hosts. + +The architecture of the testbed module is divided into the following: +@itemize @bullet + +@item Testbed API: An API which is used by the testing driver programs. It +provides with functions for creating, destroying, starting, stopping +peers, etc. + +@item Testbed service (controller): A service which is started through the +Testbed API. This service handles operations to create, destroy, start, +stop peers, connect them, modify their configurations. + +@item Testbed helper: When a controller has to be started on a host, the +testbed API starts the testbed helper on that host which in turn starts +the controller. The testbed helper receives a configuration for the +controller through its stdin and changes it to ensure the controller +doesn't run into any port conflict on that host. +@end itemize + + +The testbed service (controller) is different from the other GNUnet +services in that it is not started by ARM and is not supposed to be run +as a daemon. It is started by the testbed API through a testbed helper. +In a typical scenario involving multiple hosts, a controller is started +on each host. Controllers take up the actual task of creating peers, +starting and stopping them on the hosts they run. + +While running deployments on a single localhost the testbed API starts the +testbed helper directly as a child process. When running deployments on +remote hosts the testbed API starts Testbed Helpers on each remote host +through remote shell. By default testbed API uses SSH as a remote shell. +This can be changed by setting the environmental variable +GNUNET_TESTBED_RSH_CMD to the required remote shell program. This +variable can also contain parameters which are to be passed to the remote +shell program. For e.g: + +@example +export GNUNET_TESTBED_RSH_CMD="ssh -o BatchMode=yes \ +-o NoHostAuthenticationForLocalhost=yes %h" +@end example + +Substitutions are allowed in the command string above, +this allows for substitutions through placemarks which begin with a `%'. +At present the following substitutions are supported + +@itemize @bullet +@item %h: hostname +@item %u: username +@item %p: port +@end itemize + +Note that the substitution placemark is replaced only when the +corresponding field is available and only once. Specifying + +@example +%u@atchar{}%h +@end example + +doesn't work either. If you want to user username substitutions for +@command{SSH}, use the argument @code{-l} before the +username substitution. + +For example: +@example +ssh -l %u -p %p %h +@end example + +The testbed API and the helper communicate through the helpers stdin and +stdout. As the helper is started through a remote shell on remote hosts +any output messages from the remote shell interfere with the communication +and results in a failure while starting the helper. For this reason, it is +suggested to use flags to make the remote shells produce no output +messages and to have password-less logins. The default remote shell, SSH, +the default options are: + +@example +-o BatchMode=yes -o NoHostBasedAuthenticationForLocalhost=yes" +@end example + +Password-less logins should be ensured by using SSH keys. + +Since the testbed API executes the remote shell as a non-interactive +shell, certain scripts like .bashrc, .profiler may not be executed. If +this is the case testbed API can be forced to execute an interactive +shell by setting up the environmental variable +@code{GNUNET_TESTBED_RSH_CMD_SUFFIX} to a shell program. + +An example could be: + +@example +export GNUNET_TESTBED_RSH_CMD_SUFFIX="sh -lc" +@end example + +The testbed API will then execute the remote shell program as: + +@example +$GNUNET_TESTBED_RSH_CMD -p $port $dest $GNUNET_TESTBED_RSH_CMD_SUFFIX \ +gnunet-helper-testbed +@end example + +On some systems, problems may arise while starting testbed helpers if +GNUnet is installed into a custom location since the helper may not be +found in the standard path. This can be addressed by setting the variable +`@code{HELPER_BINARY_PATH}' to the path of the testbed helper. +Testbed API will then use this path to start helper binaries both +locally and remotely. + +Testbed API can accessed by including the +@file{gnunet_testbed_service.h} file and linking with +@code{-lgnunettestbed}. + +@c *********************************************************************** +@menu +* Supported Topologies:: +* Hosts file format:: +* Topology file format:: +* Testbed Barriers:: +* Automatic large-scale deployment in the PlanetLab testbed:: +* TESTBED Caveats:: +@end menu + +@node Supported Topologies +@subsection Supported Topologies + +While testing multi-peer deployments, it is often needed that the peers +are connected in some topology. This requirement is addressed by the +function @code{GNUNET_TESTBED_overlay_connect()} which connects any given +two peers in the testbed. + +The API also provides a helper function +@code{GNUNET_TESTBED_overlay_configure_topology()} to connect a given set +of peers in any of the following supported topologies: + +@itemize @bullet + +@item @code{GNUNET_TESTBED_TOPOLOGY_CLIQUE}: All peers are connected with +each other + +@item @code{GNUNET_TESTBED_TOPOLOGY_LINE}: Peers are connected to form a +line + +@item @code{GNUNET_TESTBED_TOPOLOGY_RING}: Peers are connected to form a +ring topology + +@item @code{GNUNET_TESTBED_TOPOLOGY_2D_TORUS}: Peers are connected to +form a 2 dimensional torus topology. The number of peers may not be a +perfect square, in that case the resulting torus may not have the uniform +poloidal and toroidal lengths + +@item @code{GNUNET_TESTBED_TOPOLOGY_ERDOS_RENYI}: Topology is generated +to form a random graph. The number of links to be present should be given + +@item @code{GNUNET_TESTBED_TOPOLOGY_SMALL_WORLD}: Peers are connected to +form a 2D Torus with some random links among them. The number of random +links are to be given + +@item @code{GNUNET_TESTBED_TOPOLOGY_SMALL_WORLD_RING}: Peers are +connected to form a ring with some random links among them. The number of +random links are to be given + +@item @code{GNUNET_TESTBED_TOPOLOGY_SCALE_FREE}: Connects peers in a +topology where peer connectivity follows power law - new peers are +connected with high probabililty to well connected peers. +@footnote{See Emergence of Scaling in Random Networks. Science 286, +509-512, 1999 +(@uref{https://gnunet.org/git/bibliography.git/plain/docs/emergence_of_scaling_in_random_networks__barabasi_albert_science_286__1999.pdf, pdf})} + +@item @code{GNUNET_TESTBED_TOPOLOGY_FROM_FILE}: The topology information +is loaded from a file. The path to the file has to be given. +@xref{Topology file format}, for the format of this file. + +@item @code{GNUNET_TESTBED_TOPOLOGY_NONE}: No topology +@end itemize + + +The above supported topologies can be specified respectively by setting +the variable @code{OVERLAY_TOPOLOGY} to the following values in the +configuration passed to Testbed API functions +@code{GNUNET_TESTBED_test_run()} and +@code{GNUNET_TESTBED_run()}: + +@itemize @bullet +@item @code{CLIQUE} +@item @code{RING} +@item @code{LINE} +@item @code{2D_TORUS} +@item @code{RANDOM} +@item @code{SMALL_WORLD} +@item @code{SMALL_WORLD_RING} +@item @code{SCALE_FREE} +@item @code{FROM_FILE} +@item @code{NONE} +@end itemize + + +Topologies @code{RANDOM}, @code{SMALL_WORLD} and @code{SMALL_WORLD_RING} +require the option @code{OVERLAY_RANDOM_LINKS} to be set to the number of +random links to be generated in the configuration. The option will be +ignored for the rest of the topologies. + +Topology @code{SCALE_FREE} requires the options +@code{SCALE_FREE_TOPOLOGY_CAP} to be set to the maximum number of peers +which can connect to a peer and @code{SCALE_FREE_TOPOLOGY_M} to be set to +how many peers a peer should be atleast connected to. + +Similarly, the topology @code{FROM_FILE} requires the option +@code{OVERLAY_TOPOLOGY_FILE} to contain the path of the file containing +the topology information. This option is ignored for the rest of the +topologies. @xref{Topology file format}, for the format of this file. + +@c *********************************************************************** +@node Hosts file format +@subsection Hosts file format + +The testbed API offers the function +@code{GNUNET_TESTBED_hosts_load_from_file()} to load from a given file +details about the hosts which testbed can use for deploying peers. +This function is useful to keep the data about hosts +separate instead of hard coding them in code. + +Another helper function from testbed API, @code{GNUNET_TESTBED_run()} +also takes a hosts file name as its parameter. It uses the above +function to populate the hosts data structures and start controllers to +deploy peers. + +These functions require the hosts file to be of the following format: +@itemize @bullet +@item Each line is interpreted to have details about a host +@item Host details should include the username to use for logging into the +host, the hostname of the host and the port number to use for the remote +shell program. All thee values should be given. +@item These details should be given in the following format: +@example +<username>@@<hostname>:<port> +@end example +@end itemize + +Note that having canonical hostnames may cause problems while resolving +the IP addresses (See this bug). Hence it is advised to provide the hosts' +IP numerical addresses as hostnames whenever possible. + +@c *********************************************************************** +@node Topology file format +@subsection Topology file format + +A topology file describes how peers are to be connected. It should adhere +to the following format for testbed to parse it correctly. + +Each line should begin with the target peer id. This should be followed by +a colon(`:') and origin peer ids seperated by `|'. All spaces except for +newline characters are ignored. The API will then try to connect each +origin peer to the target peer. + +For example, the following file will result in 5 overlay connections: +[2->1], [3->1],[4->3], [0->3], [2->0]@ +@code{@ 1:2|3@ 3:4| 0@ 0: 2@ } + +@c *********************************************************************** +@node Testbed Barriers +@subsection Testbed Barriers + +The testbed subsystem's barriers API facilitates coordination among the +peers run by the testbed and the experiment driver. The concept is +similar to the barrier synchronisation mechanism found in parallel +programming or multi-threading paradigms - a peer waits at a barrier upon +reaching it until the barrier is reached by a predefined number of peers. +This predefined number of peers required to cross a barrier is also called +quorum. We say a peer has reached a barrier if the peer is waiting for the +barrier to be crossed. Similarly a barrier is said to be reached if the +required quorum of peers reach the barrier. A barrier which is reached is +deemed as crossed after all the peers waiting on it are notified. + +The barriers API provides the following functions: +@itemize @bullet +@item @strong{@code{GNUNET_TESTBED_barrier_init()}:} function to +initialse a barrier in the experiment +@item @strong{@code{GNUNET_TESTBED_barrier_cancel()}:} function to cancel +a barrier which has been initialised before +@item @strong{@code{GNUNET_TESTBED_barrier_wait()}:} function to signal +barrier service that the caller has reached a barrier and is waiting for +it to be crossed +@item @strong{@code{GNUNET_TESTBED_barrier_wait_cancel()}:} function to +stop waiting for a barrier to be crossed +@end itemize + + +Among the above functions, the first two, namely +@code{GNUNET_TESTBED_barrier_init()} and +@code{GNUNET_TESTBED_barrier_cancel()} are used by experiment drivers. All +barriers should be initialised by the experiment driver by calling +@code{GNUNET_TESTBED_barrier_init()}. This function takes a name to +identify the barrier, the quorum required for the barrier to be crossed +and a notification callback for notifying the experiment driver when the +barrier is crossed. @code{GNUNET_TESTBED_barrier_cancel()} cancels an +initialised barrier and frees the resources allocated for it. This +function can be called upon a initialised barrier before it is crossed. + +The remaining two functions @code{GNUNET_TESTBED_barrier_wait()} and +@code{GNUNET_TESTBED_barrier_wait_cancel()} are used in the peer's +processes. @code{GNUNET_TESTBED_barrier_wait()} connects to the local +barrier service running on the same host the peer is running on and +registers that the caller has reached the barrier and is waiting for the +barrier to be crossed. Note that this function can only be used by peers +which are started by testbed as this function tries to access the local +barrier service which is part of the testbed controller service. Calling +@code{GNUNET_TESTBED_barrier_wait()} on an uninitialised barrier results +in failure. @code{GNUNET_TESTBED_barrier_wait_cancel()} cancels the +notification registered by @code{GNUNET_TESTBED_barrier_wait()}. + + +@c *********************************************************************** +@menu +* Implementation:: +@end menu + +@node Implementation +@subsubsection Implementation + +Since barriers involve coordination between experiment driver and peers, +the barrier service in the testbed controller is split into two +components. The first component responds to the message generated by the +barrier API used by the experiment driver (functions +@code{GNUNET_TESTBED_barrier_init()} and +@code{GNUNET_TESTBED_barrier_cancel()}) and the second component to the +messages generated by barrier API used by peers (functions +@code{GNUNET_TESTBED_barrier_wait()} and +@code{GNUNET_TESTBED_barrier_wait_cancel()}). + +Calling @code{GNUNET_TESTBED_barrier_init()} sends a +@code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_INIT} message to the master +controller. The master controller then registers a barrier and calls +@code{GNUNET_TESTBED_barrier_init()} for each its subcontrollers. In this +way barrier initialisation is propagated to the controller hierarchy. +While propagating initialisation, any errors at a subcontroller such as +timeout during further propagation are reported up the hierarchy back to +the experiment driver. + +Similar to @code{GNUNET_TESTBED_barrier_init()}, +@code{GNUNET_TESTBED_barrier_cancel()} propagates +@code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_CANCEL} message which causes +controllers to remove an initialised barrier. + +The second component is implemented as a separate service in the binary +`gnunet-service-testbed' which already has the testbed controller service. +Although this deviates from the gnunet process architecture of having one +service per binary, it is needed in this case as this component needs +access to barrier data created by the first component. This component +responds to @code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_WAIT} messages from +local peers when they call @code{GNUNET_TESTBED_barrier_wait()}. Upon +receiving @code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_WAIT} message, the +service checks if the requested barrier has been initialised before and +if it was not initialised, an error status is sent through +@code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_STATUS} message to the local +peer and the connection from the peer is terminated. If the barrier is +initialised before, the barrier's counter for reached peers is incremented +and a notification is registered to notify the peer when the barrier is +reached. The connection from the peer is left open. + +When enough peers required to attain the quorum send +@code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_WAIT} messages, the controller +sends a @code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_STATUS} message to its +parent informing that the barrier is crossed. If the controller has +started further subcontrollers, it delays this message until it receives +a similar notification from each of those subcontrollers. Finally, the +barriers API at the experiment driver receives the +@code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_STATUS} when the barrier is +reached at all the controllers. + +The barriers API at the experiment driver responds to the +@code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_STATUS} message by echoing it +back to the master controller and notifying the experiment controller +through the notification callback that a barrier has been crossed. The +echoed @code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_STATUS} message is +propagated by the master controller to the controller hierarchy. This +propagation triggers the notifications registered by peers at each of the +controllers in the hierarchy. Note the difference between this downward +propagation of the @code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_STATUS} +message from its upward propagation --- the upward propagation is needed +for ensuring that the barrier is reached by all the controllers and the +downward propagation is for triggering that the barrier is crossed. + +@cindex PlanetLab testbed +@node Automatic large-scale deployment in the PlanetLab testbed +@subsection Automatic large-scale deployment in the PlanetLab testbed + +PlanetLab is a testbed for computer networking and distributed systems +research. It was established in 2002 and as of June 2010 was composed of +1090 nodes at 507 sites worldwide. + +To automate the GNUnet we created a set of automation tools to simplify +the large-scale deployment. We provide you a set of scripts you can use +to deploy GNUnet on a set of nodes and manage your installation. + +Please also check @uref{https://gnunet.org/installation-fedora8-svn} and +@uref{https://gnunet.org/installation-fedora12-svn} to find detailled +instructions how to install GNUnet on a PlanetLab node. + + +@c *********************************************************************** +@menu +* PlanetLab Automation for Fedora8 nodes:: +* Install buildslave on PlanetLab nodes running fedora core 8:: +* Setup a new PlanetLab testbed using GPLMT:: +* Why do i get an ssh error when using the regex profiler?:: +@end menu + +@node PlanetLab Automation for Fedora8 nodes +@subsubsection PlanetLab Automation for Fedora8 nodes + +@c *********************************************************************** +@node Install buildslave on PlanetLab nodes running fedora core 8 +@subsubsection Install buildslave on PlanetLab nodes running fedora core 8 +@c ** Actually this is a subsubsubsection, but must be fixed differently +@c ** as subsubsection is the lowest. + +Since most of the PlanetLab nodes are running the very old Fedora core 8 +image, installing the buildslave software is quite some pain. For our +PlanetLab testbed we figured out how to install the buildslave software +best. + +@c This is a vvery terrible way to suggest installing software. +@c FIXME: Is there an official, safer way instead of blind-piping a +@c script? +@c FIXME: Use newer pypi URLs below. +Install Distribute for Python: + +@example +curl http://python-distribute.org/distribute_setup.py | sudo python +@end example + +Install Distribute for zope.interface <= 3.8.0 (4.0 and 4.0.1 will not +work): + +@example +export PYPI=@value{PYPI-URL} +wget $PYPI/z/zope.interface/zope.interface-3.8.0.tar.gz +tar zvfz zope.interface-3.8.0.tar.gz +cd zope.interface-3.8.0 +sudo python setup.py install +@end example + +Install the buildslave software (0.8.6 was the latest version): + +@example +export GCODE="http://buildbot.googlecode.com/files" +wget $GCODE/buildbot-slave-0.8.6p1.tar.gz +tar xvfz buildbot-slave-0.8.6p1.tar.gz +cd buildslave-0.8.6p1 +sudo python setup.py install +@end example + +The setup will download the matching twisted package and install it. +It will also try to install the latest version of zope.interface which +will fail to install. Buildslave will work anyway since version 3.8.0 +was installed before! + +@c *********************************************************************** +@node Setup a new PlanetLab testbed using GPLMT +@subsubsection Setup a new PlanetLab testbed using GPLMT + +@itemize @bullet +@item Get a new slice and assign nodes +Ask your PlanetLab PI to give you a new slice and assign the nodes you +need +@item Install a buildmaster +You can stick to the buildbot documentation:@ +@uref{http://buildbot.net/buildbot/docs/current/manual/installation.html} +@item Install the buildslave software on all nodes +To install the buildslave on all nodes assigned to your slice you can use +the tasklist @code{install_buildslave_fc8.xml} provided with GPLMT: + +@example +./gplmt.py -c contrib/tumple_gnunet.conf -t \ +contrib/tasklists/install_buildslave_fc8.xml -a -p <planetlab password> +@end example + +@item Create the buildmaster configuration and the slave setup commands + +The master and the and the slaves have need to have credentials and the +master has to have all nodes configured. This can be done with the +@file{create_buildbot_configuration.py} script in the @file{scripts} +directory. + +This scripts takes a list of nodes retrieved directly from PlanetLab or +read from a file and a configuration template and creates: + +@itemize @bullet +@item a tasklist which can be executed with gplmt to setup the slaves +@item a master.cfg file containing a PlanetLab nodes +@end itemize + +A configuration template is included in the <contrib>, most important is +that the script replaces the following tags in the template: + +%GPLMT_BUILDER_DEFINITION :@ GPLMT_BUILDER_SUMMARY@ GPLMT_SLAVES@ +%GPLMT_SCHEDULER_BUILDERS + +Create configuration for all nodes assigned to a slice: + +@example +./create_buildbot_configuration.py -u <planetlab username> \ +-p <planetlab password> -s <slice> -m <buildmaster+port> \ +-t <template> +@end example + +Create configuration for some nodes in a file: + +@example +./create_buildbot_configuration.p -f <node_file> \ +-m <buildmaster+port> -t <template> +@end example + +@item Copy the @file{master.cfg} to the buildmaster and start it +Use @code{buildbot start <basedir>} to start the server +@item Setup the buildslaves +@end itemize + +@c *********************************************************************** +@node Why do i get an ssh error when using the regex profiler? +@subsubsection Why do i get an ssh error when using the regex profiler? + +Why do i get an ssh error "Permission denied (publickey,password)." when +using the regex profiler although passwordless ssh to localhost works +using publickey and ssh-agent? + +You have to generate a public/private-key pair with no password:@ +@code{ssh-keygen -t rsa -b 4096 -f ~/.ssh/id_localhost}@ +and then add the following to your ~/.ssh/config file: + +@code{Host 127.0.0.1@ IdentityFile ~/.ssh/id_localhost} + +now make sure your hostsfile looks like + +@example +[USERNAME]@@127.0.0.1:22@ +[USERNAME]@@127.0.0.1:22 +@end example + +You can test your setup by running @code{ssh 127.0.0.1} in a +terminal and then in the opened session run it again. +If you were not asked for a password on either login, +then you should be good to go. + +@cindex TESTBED Caveats +@node TESTBED Caveats +@subsection TESTBED Caveats + +This section documents a few caveats when using the GNUnet testbed +subsystem. + +@c *********************************************************************** +@menu +* CORE must be started:: +* ATS must want the connections:: +@end menu + +@node CORE must be started +@subsubsection CORE must be started + +A simple issue is #3993: Your configuration MUST somehow ensure that for +each peer the CORE service is started when the peer is setup, otherwise +TESTBED may fail to connect peers when the topology is initialized, as +TESTBED will start some CORE services but not necessarily all (but it +relies on all of them running). The easiest way is to set +'FORCESTART = YES' in the '[core]' section of the configuration file. +Alternatively, having any service that directly or indirectly depends on +CORE being started with FORCESTART will also do. This issue largely arises +if users try to over-optimize by not starting any services with +FORCESTART. + +@c *********************************************************************** +@node ATS must want the connections +@subsubsection ATS must want the connections + +When TESTBED sets up connections, it only offers the respective HELLO +information to the TRANSPORT service. It is then up to the ATS service to +@strong{decide} to use the connection. The ATS service will typically +eagerly establish any connection if the number of total connections is +low (relative to bandwidth). Details may further depend on the +specific ATS backend that was configured. If ATS decides to NOT establish +a connection (even though TESTBED provided the required information), then +that connection will count as failed for TESTBED. Note that you can +configure TESTBED to tolerate a certain number of connection failures +(see '-e' option of gnunet-testbed-profiler). This issue largely arises +for dense overlay topologies, especially if you try to create cliques +with more than 20 peers. + +@cindex libgnunetutil +@node libgnunetutil +@section libgnunetutil + +libgnunetutil is the fundamental library that all GNUnet code builds upon. +Ideally, this library should contain most of the platform dependent code +(except for user interfaces and really special needs that only few +applications have). It is also supposed to offer basic services that most +if not all GNUnet binaries require. The code of libgnunetutil is in the +@file{src/util/} directory. The public interface to the library is in the +gnunet_util.h header. The functions provided by libgnunetutil fall +roughly into the following categories (in roughly the order of importance +for new developers): + +@itemize @bullet +@item logging (common_logging.c) +@item memory allocation (common_allocation.c) +@item endianess conversion (common_endian.c) +@item internationalization (common_gettext.c) +@item String manipulation (string.c) +@item file access (disk.c) +@item buffered disk IO (bio.c) +@item time manipulation (time.c) +@item configuration parsing (configuration.c) +@item command-line handling (getopt*.c) +@item cryptography (crypto_*.c) +@item data structures (container_*.c) +@item CPS-style scheduling (scheduler.c) +@item Program initialization (program.c) +@item Networking (network.c, client.c, server*.c, service.c) +@item message queueing (mq.c) +@item bandwidth calculations (bandwidth.c) +@item Other OS-related (os*.c, plugin.c, signal.c) +@item Pseudonym management (pseudonym.c) +@end itemize + +It should be noted that only developers that fully understand this entire +API will be able to write good GNUnet code. + +Ideally, porting GNUnet should only require porting the gnunetutil +library. More testcases for the gnunetutil APIs are therefore a great +way to make porting of GNUnet easier. + +@menu +* Logging:: +* Interprocess communication API (IPC):: +* Cryptography API:: +* Message Queue API:: +* Service API:: +* Optimizing Memory Consumption of GNUnet's (Multi-) Hash Maps:: +* CONTAINER_MDLL API:: +@end menu + +@cindex Logging +@cindex log levels +@node Logging +@subsection Logging + +GNUnet is able to log its activity, mostly for the purposes of debugging +the program at various levels. + +@file{gnunet_common.h} defines several @strong{log levels}: +@table @asis + +@item ERROR for errors (really problematic situations, often leading to +crashes) +@item WARNING for warnings (troubling situations that might have +negative consequences, although not fatal) +@item INFO for various information. +Used somewhat rarely, as GNUnet statistics is used to hold and display +most of the information that users might find interesting. +@item DEBUG for debugging. +Does not produce much output on normal builds, but when extra logging is +enabled at compile time, a staggering amount of data is outputted under +this log level. +@end table + + +Normal builds of GNUnet (configured with @code{--enable-logging[=yes]}) +are supposed to log nothing under DEBUG level. The +@code{--enable-logging=verbose} configure option can be used to create a +build with all logging enabled. However, such build will produce large +amounts of log data, which is inconvenient when one tries to hunt down a +specific problem. + +To mitigate this problem, GNUnet provides facilities to apply a filter to +reduce the logs: +@table @asis + +@item Logging by default When no log levels are configured in any other +way (see below), GNUnet will default to the WARNING log level. This +mostly applies to GNUnet command line utilities, services and daemons; +tests will always set log level to WARNING or, if +@code{--enable-logging=verbose} was passed to configure, to DEBUG. The +default level is suggested for normal operation. +@item The -L option Most GNUnet executables accept an "-L loglevel" or +"--log=loglevel" option. If used, it makes the process set a global log +level to "loglevel". Thus it is possible to run some processes +with -L DEBUG, for example, and others with -L ERROR to enable specific +settings to diagnose problems with a particular process. +@item Configuration files. Because GNUnet +service and deamon processes are usually launched by gnunet-arm, it is not +possible to pass different custom command line options directly to every +one of them. The options passed to @code{gnunet-arm} only affect +gnunet-arm and not the rest of GNUnet. However, one can specify a +configuration key "OPTIONS" in the section that corresponds to a service +or a daemon, and put a value of "-L loglevel" there. This will make the +respective service or daemon set its log level to "loglevel" (as the +value of OPTIONS will be passed as a command-line argument). + +To specify the same log level for all services without creating separate +"OPTIONS" entries in the configuration for each one, the user can specify +a config key "GLOBAL_POSTFIX" in the [arm] section of the configuration +file. The value of GLOBAL_POSTFIX will be appended to all command lines +used by the ARM service to run other services. It can contain any option +valid for all GNUnet commands, thus in particular the "-L loglevel" +option. The ARM service itself is, however, unaffected by GLOBAL_POSTFIX; +to set log level for it, one has to specify "OPTIONS" key in the [arm] +section. +@item Environment variables. +Setting global per-process log levels with "-L loglevel" does not offer +sufficient log filtering granularity, as one service will call interface +libraries and supporting libraries of other GNUnet services, potentially +producing lots of debug log messages from these libraries. Also, changing +the config file is not always convenient (especially when running the +GNUnet test suite).@ To fix that, and to allow GNUnet to use different +log filtering at runtime without re-compiling the whole source tree, the +log calls were changed to be configurable at run time. To configure them +one has to define environment variables "GNUNET_FORCE_LOGFILE", +"GNUNET_LOG" and/or "GNUNET_FORCE_LOG": +@itemize @bullet + +@item "GNUNET_LOG" only affects the logging when no global log level is +configured by any other means (that is, the process does not explicitly +set its own log level, there are no "-L loglevel" options on command line +or in configuration files), and can be used to override the default +WARNING log level. + +@item "GNUNET_FORCE_LOG" will completely override any other log +configuration options given. + +@item "GNUNET_FORCE_LOGFILE" will completely override the location of the +file to log messages to. It should contain a relative or absolute file +name. Setting GNUNET_FORCE_LOGFILE is equivalent to passing +"--log-file=logfile" or "-l logfile" option (see below). It supports "[]" +format in file names, but not "@{@}" (see below). +@end itemize + + +Because environment variables are inherited by child processes when they +are launched, starting or re-starting the ARM service with these +variables will propagate them to all other services. + +"GNUNET_LOG" and "GNUNET_FORCE_LOG" variables must contain a specially +formatted @strong{logging definition} string, which looks like this:@ + +@c FIXME: Can we close this with [/component] instead? +@example +[component];[file];[function];[from_line[-to_line]];loglevel[/component...] +@end example + +That is, a logging definition consists of definition entries, separated by +slashes ('/'). If only one entry is present, there is no need to add a +slash to its end (although it is not forbidden either).@ All definition +fields (component, file, function, lines and loglevel) are mandatory, but +(except for the loglevel) they can be empty. An empty field means +"match anything". Note that even if fields are empty, the semicolon (';') +separators must be present.@ The loglevel field is mandatory, and must +contain one of the log level names (ERROR, WARNING, INFO or DEBUG).@ +The lines field might contain one non-negative number, in which case it +matches only one line, or a range "from_line-to_line", in which case it +matches any line in the interval [from_line;to_line] (that is, including +both start and end line).@ GNUnet mostly defaults component name to the +name of the service that is implemented in a process ('transport', +'core', 'peerinfo', etc), but logging calls can specify custom component +names using @code{GNUNET_log_from}.@ File name and function name are +provided by the compiler (__FILE__ and __FUNCTION__ built-ins). + +Component, file and function fields are interpreted as non-extended +regular expressions (GNU libc regex functions are used). Matching is +case-sensitive, "^" and "$" will match the beginning and the end of the +text. If a field is empty, its contents are automatically replaced with +a ".*" regular expression, which matches anything. Matching is done in +the default way, which means that the expression matches as long as it's +contained anywhere in the string. Thus "GNUNET_" will match both +"GNUNET_foo" and "BAR_GNUNET_BAZ". Use '^' and/or '$' to make sure that +the expression matches at the start and/or at the end of the string. +The semicolon (';') can't be escaped, and GNUnet will not use it in +component names (it can't be used in function names and file names +anyway). + +@end table + + +Every logging call in GNUnet code will be (at run time) matched against +the log definitions passed to the process. If a log definition fields are +matching the call arguments, then the call log level is compared the the +log level of that definition. If the call log level is less or equal to +the definition log level, the call is allowed to proceed. Otherwise the +logging call is forbidden, and nothing is logged. If no definitions +matched at all, GNUnet will use the global log level or (if a global log +level is not specified) will default to WARNING (that is, it will allow +the call to proceed, if its level is less or equal to the global log +level or to WARNING). + +That is, definitions are evaluated from left to right, and the first +matching definition is used to allow or deny the logging call. Thus it is +advised to place narrow definitions at the beginning of the logdef +string, and generic definitions - at the end. + +Whether a call is allowed or not is only decided the first time this +particular call is made. The evaluation result is then cached, so that +any attempts to make the same call later will be allowed or disallowed +right away. Because of that runtime log level evaluation should not +significantly affect the process performance. +Log definition parsing is only done once, at the first call to +GNUNET_log_setup () made by the process (which is usually done soon after +it starts). + +At the moment of writing there is no way to specify logging definitions +from configuration files, only via environment variables. + +At the moment GNUnet will stop processing a log definition when it +encounters an error in definition formatting or an error in regular +expression syntax, and will not report the failure in any way. + + +@c *********************************************************************** +@menu +* Examples:: +* Log files:: +* Updated behavior of GNUNET_log:: +@end menu + +@node Examples +@subsubsection Examples + +@table @asis + +@item @code{GNUNET_FORCE_LOG=";;;;DEBUG" gnunet-arm -s} Start GNUnet +process tree, running all processes with DEBUG level (one should be +careful with it, as log files will grow at alarming rate!) +@item @code{GNUNET_FORCE_LOG="core;;;;DEBUG" gnunet-arm -s} Start GNUnet +process tree, running the core service under DEBUG level (everything else +will use configured or default level). + +@item Start GNUnet process tree, allowing any logging calls from +gnunet-service-transport_validation.c (everything else will use +configured or default level). + +@example +GNUNET_FORCE_LOG=";gnunet-service-transport_validation.c;;; DEBUG" \ +gnunet-arm -s +@end example + +@item Start GNUnet process tree, allowing any logging calls from +gnunet-gnunet-service-fs_push.c (everything else will use configured or +default level). + +@example +GNUNET_FORCE_LOG="fs;gnunet-service-fs_push.c;;;DEBUG" gnunet-arm -s +@end example + +@item Start GNUnet process tree, allowing any logging calls from the +GNUNET_NETWORK_socket_select function (everything else will use +configured or default level). + +@example +GNUNET_FORCE_LOG=";;GNUNET_NETWORK_socket_select;;DEBUG" gnunet-arm -s +@end example + +@item Start GNUnet process tree, allowing any logging calls from the +components that have "transport" in their names, and are made from +function that have "send" in their names. Everything else will be allowed +to be logged only if it has WARNING level. + +@example +GNUNET_FORCE_LOG="transport.*;;.*send.*;;DEBUG/;;;;WARNING" gnunet-arm -s +@end example + +@end table + + +On Windows, one can use batch files to run GNUnet processes with special +environment variables, without affecting the whole system. Such batch +file will look like this: + +@example +set GNUNET_FORCE_LOG=;;do_transmit;;DEBUG@ gnunet-arm -s +@end example + +(note the absence of double quotes in the environment variable definition, +as opposed to earlier examples, which use the shell). +Another limitation, on Windows, GNUNET_FORCE_LOGFILE @strong{MUST} be set +in order to GNUNET_FORCE_LOG to work. + + +@cindex Log files +@node Log files +@subsubsection Log files + +GNUnet can be told to log everything into a file instead of stderr (which +is the default) using the "--log-file=logfile" or "-l logfile" option. +This option can also be passed via command line, or from the "OPTION" and +"GLOBAL_POSTFIX" configuration keys (see above). The file name passed +with this option is subject to GNUnet filename expansion. If specified in +"GLOBAL_POSTFIX", it is also subject to ARM service filename expansion, +in particular, it may contain "@{@}" (left and right curly brace) +sequence, which will be replaced by ARM with the name of the service. +This is used to keep logs from more than one service separate, while only +specifying one template containing "@{@}" in GLOBAL_POSTFIX. + +As part of a secondary file name expansion, the first occurrence of "[]" +sequence ("left square brace" followed by "right square brace") in the +file name will be replaced with a process identifier or the process when +it initializes its logging subsystem. As a result, all processes will log +into different files. This is convenient for isolating messages of a +particular process, and prevents I/O races when multiple processes try to +write into the file at the same time. This expansion is done +independently of "@{@}" expansion that ARM service does (see above). + +The log file name that is specified via "-l" can contain format characters +from the 'strftime' function family. For example, "%Y" will be replaced +with the current year. Using "basename-%Y-%m-%d.log" would include the +current year, month and day in the log file. If a GNUnet process runs for +long enough to need more than one log file, it will eventually clean up +old log files. Currently, only the last three log files (plus the current +log file) are preserved. So once the fifth log file goes into use (so +after 4 days if you use "%Y-%m-%d" as above), the first log file will be +automatically deleted. Note that if your log file name only contains "%Y", +then log files would be kept for 4 years and the logs from the first year +would be deleted once year 5 begins. If you do not use any date-related +string format codes, logs would never be automatically deleted by GNUnet. + + +@c *********************************************************************** + +@node Updated behavior of GNUNET_log +@subsubsection Updated behavior of GNUNET_log + +It's currently quite common to see constructions like this all over the +code: + +@example +#if MESH_DEBUG +GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "MESH: client disconnected\n"); +#endif +@end example + +The reason for the #if is not to avoid displaying the message when +disabled (GNUNET_ERROR_TYPE takes care of that), but to avoid the +compiler including it in the binary at all, when compiling GNUnet for +platforms with restricted storage space / memory (MIPS routers, +ARM plug computers / dev boards, etc). + +This presents several problems: the code gets ugly, hard to write and it +is very easy to forget to include the #if guards, creating non-consistent +code. A new change in GNUNET_log aims to solve these problems. + +@strong{This change requires to @file{./configure} with at least +@code{--enable-logging=verbose} to see debug messages.} + +Here is an example of code with dense debug statements: + +@example +switch (restrict_topology) @{ +case GNUNET_TESTING_TOPOLOGY_CLIQUE:#if VERBOSE_TESTING +GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, _("Blacklisting all but clique +topology\n")); #endif unblacklisted_connections = create_clique (pg, +&remove_connections, BLACKLIST, GNUNET_NO); break; case +GNUNET_TESTING_TOPOLOGY_SMALL_WORLD_RING: #if VERBOSE_TESTING GNUNET_log +(GNUNET_ERROR_TYPE_DEBUG, _("Blacklisting all but small world (ring) +topology\n")); #endif unblacklisted_connections = create_small_world_ring +(pg,&remove_connections, BLACKLIST); break; +@end example + + +Pretty hard to follow, huh? + +From now on, it is not necessary to include the #if / #endif statements to +achieve the same behavior. The GNUNET_log and GNUNET_log_from macros take +care of it for you, depending on the configure option: + +@itemize @bullet +@item If @code{--enable-logging} is set to @code{no}, the binary will +contain no log messages at all. +@item If @code{--enable-logging} is set to @code{yes}, the binary will +contain no DEBUG messages, and therefore running with -L DEBUG will have +no effect. Other messages (ERROR, WARNING, INFO, etc) will be included. +@item If @code{--enable-logging} is set to @code{verbose}, or +@code{veryverbose} the binary will contain DEBUG messages (still, it will +be neccessary to run with -L DEBUG or set the DEBUG config option to show +them). +@end itemize + + +If you are a developer: +@itemize @bullet +@item please make sure that you @code{./configure +--enable-logging=@{verbose,veryverbose@}}, so you can see DEBUG messages. +@item please remove the @code{#if} statements around @code{GNUNET_log +(GNUNET_ERROR_TYPE_DEBUG, ...)} lines, to improve the readibility of your +code. +@end itemize + +Since now activating DEBUG automatically makes it VERBOSE and activates +@strong{all} debug messages by default, you probably want to use the +https://gnunet.org/logging functionality to filter only relevant messages. +A suitable configuration could be: + +@example +$ export GNUNET_FORCE_LOG="^YOUR_SUBSYSTEM$;;;;DEBUG/;;;;WARNING" +@end example + +Which will behave almost like enabling DEBUG in that subsytem before the +change. Of course you can adapt it to your particular needs, this is only +a quick example. + +@cindex Interprocess communication API +@cindex ICP +@node Interprocess communication API (IPC) +@subsection Interprocess communication API (IPC) + +In GNUnet a variety of new message types might be defined and used in +interprocess communication, in this tutorial we use the +@code{struct AddressLookupMessage} as a example to introduce how to +construct our own message type in GNUnet and how to implement the message +communication between service and client. +(Here, a client uses the @code{struct AddressLookupMessage} as a request +to ask the server to return the address of any other peer connecting to +the service.) + + +@c *********************************************************************** +@menu +* Define new message types:: +* Define message struct:: +* Client - Establish connection:: +* Client - Initialize request message:: +* Client - Send request and receive response:: +* Server - Startup service:: +* Server - Add new handles for specified messages:: +* Server - Process request message:: +* Server - Response to client:: +* Server - Notification of clients:: +* Conversion between Network Byte Order (Big Endian) and Host Byte Order:: +@end menu + +@node Define new message types +@subsubsection Define new message types + +First of all, you should define the new message type in +@file{gnunet_protocols.h}: + +@example + // Request to look addresses of peers in server. +#define GNUNET_MESSAGE_TYPE_TRANSPORT_ADDRESS_LOOKUP 29 + // Response to the address lookup request. +#define GNUNET_MESSAGE_TYPE_TRANSPORT_ADDRESS_REPLY 30 +@end example + +@c *********************************************************************** +@node Define message struct +@subsubsection Define message struct + +After the type definition, the specified message structure should also be +described in the header file, e.g. transport.h in our case. + +@example +struct AddressLookupMessage @{ + struct GNUNET_MessageHeader header; + int32_t numeric_only GNUNET_PACKED; + struct GNUNET_TIME_AbsoluteNBO timeout; + uint32_t addrlen GNUNET_PACKED; + /* followed by 'addrlen' bytes of the actual address, then + followed by the 0-terminated name of the transport */ @}; +GNUNET_NETWORK_STRUCT_END +@end example + + +Please note @code{GNUNET_NETWORK_STRUCT_BEGIN} and @code{GNUNET_PACKED} +which both ensure correct alignment when sending structs over the network. + +@menu +@end menu + +@c *********************************************************************** +@node Client - Establish connection +@subsubsection Client - Establish connection +@c %**end of header + + +At first, on the client side, the underlying API is employed to create a +new connection to a service, in our example the transport service would be +connected. + +@example +struct GNUNET_CLIENT_Connection *client; +client = GNUNET_CLIENT_connect ("transport", cfg); +@end example + +@c *********************************************************************** +@node Client - Initialize request message +@subsubsection Client - Initialize request message +@c %**end of header + +When the connection is ready, we initialize the message. In this step, +all the fields of the message should be properly initialized, namely the +size, type, and some extra user-defined data, such as timeout, name of +transport, address and name of transport. + +@example +struct AddressLookupMessage *msg; +size_t len = sizeof (struct AddressLookupMessage) + + addressLen + + strlen (nameTrans) + + 1; +msg->header->size = htons (len); +msg->header->type = htons +(GNUNET_MESSAGE_TYPE_TRANSPORT_ADDRESS_LOOKUP); +msg->timeout = GNUNET_TIME_absolute_hton (abs_timeout); +msg->addrlen = htonl (addressLen); +char *addrbuf = (char *) &msg[1]; +memcpy (addrbuf, address, addressLen); +char *tbuf = &addrbuf[addressLen]; +memcpy (tbuf, nameTrans, strlen (nameTrans) + 1); +@end example + +Note that, here the functions @code{htonl}, @code{htons} and +@code{GNUNET_TIME_absolute_hton} are applied to convert little endian +into big endian, about the usage of the big/small edian order and the +corresponding conversion function please refer to Introduction of +Big Endian and Little Endian. + +@c *********************************************************************** +@node Client - Send request and receive response +@subsubsection Client - Send request and receive response +@c %**end of header + +@b{FIXME: This is very outdated, see the tutorial for the current API!} + +Next, the client would send the constructed message as a request to the +service and wait for the response from the service. To accomplish this +goal, there are a number of API calls that can be used. In this example, +@code{GNUNET_CLIENT_transmit_and_get_response} is chosen as the most +appropriate function to use. + +@example +GNUNET_CLIENT_transmit_and_get_response +(client, msg->header, timeout, GNUNET_YES, &address_response_processor, +arp_ctx); +@end example + +the argument @code{address_response_processor} is a function with +@code{GNUNET_CLIENT_MessageHandler} type, which is used to process the +reply message from the service. + +@node Server - Startup service +@subsubsection Server - Startup service + +After receiving the request message, we run a standard GNUnet service +startup sequence using @code{GNUNET_SERVICE_run}, as follows, + +@example +int main(int argc, char**argv) @{ + GNUNET_SERVICE_run(argc, argv, "transport" + GNUNET_SERVICE_OPTION_NONE, &run, NULL)); @} +@end example + +@c *********************************************************************** +@node Server - Add new handles for specified messages +@subsubsection Server - Add new handles for specified messages +@c %**end of header + +in the function above the argument @code{run} is used to initiate +transport service,and defined like this: + +@example +static void run (void *cls, +struct GNUNET_SERVER_Handle *serv, +const struct GNUNET_CONFIGURATION_Handle *cfg) @{ + GNUNET_SERVER_add_handlers (serv, handlers); @} +@end example + + +Here, @code{GNUNET_SERVER_add_handlers} must be called in the run +function to add new handlers in the service. The parameter +@code{handlers} is a list of @code{struct GNUNET_SERVER_MessageHandler} +to tell the service which function should be called when a particular +type of message is received, and should be defined in this way: + +@example +static struct GNUNET_SERVER_MessageHandler handlers[] = @{ + @{&handle_start, + NULL, + GNUNET_MESSAGE_TYPE_TRANSPORT_START, + 0@}, + @{&handle_send, + NULL, + GNUNET_MESSAGE_TYPE_TRANSPORT_SEND, + 0@}, + @{&handle_try_connect, + NULL, + GNUNET_MESSAGE_TYPE_TRANSPORT_TRY_CONNECT, + sizeof (struct TryConnectMessage) + @}, + @{&handle_address_lookup, + NULL, + GNUNET_MESSAGE_TYPE_TRANSPORT_ADDRESS_LOOKUP, + 0@}, + @{NULL, + NULL, + 0, + 0@} +@}; +@end example + + +As shown, the first member of the struct in the first area is a callback +function, which is called to process the specified message types, given +as the third member. The second parameter is the closure for the callback +function, which is set to @code{NULL} in most cases, and the last +parameter is the expected size of the message of this type, usually we +set it to 0 to accept variable size, for special cases the exact size of +the specified message also can be set. In addition, the terminator sign +depicted as @code{@{NULL, NULL, 0, 0@}} is set in the last aera. + +@c *********************************************************************** +@node Server - Process request message +@subsubsection Server - Process request message +@c %**end of header + +After the initialization of transport service, the request message would +be processed. Before handling the main message data, the validity of this +message should be checked out, e.g., to check whether the size of message +is correct. + +@example +size = ntohs (message->size); +if (size < sizeof (struct AddressLookupMessage)) @{ + GNUNET_break_op (0); + GNUNET_SERVER_receive_done (client, GNUNET_SYSERR); + return; @} +@end example + + +Note that, opposite to the construction method of the request message in +the client, in the server the function @code{nothl} and @code{ntohs} +should be employed during the extraction of the data from the message, so +that the data in big endian order can be converted back into little +endian order. See more in detail please refer to Introduction of +Big Endian and Little Endian. + +Moreover in this example, the name of the transport stored in the message +is a 0-terminated string, so we should also check whether the name of the +transport in the received message is 0-terminated: + +@example +nameTransport = (const char *) &address[addressLen]; +if (nameTransport[size - sizeof + (struct AddressLookupMessage) + - addressLen - 1] != '\0') @{ + GNUNET_break_op (0); + GNUNET_SERVER_receive_done (client, + GNUNET_SYSERR); + return; @} +@end example + +Here, @code{GNUNET_SERVER_receive_done} should be called to tell the +service that the request is done and can receive the next message. The +argument @code{GNUNET_SYSERR} here indicates that the service didn't +understand the request message, and the processing of this request would +be terminated. + +In comparison to the aforementioned situation, when the argument is equal +to @code{GNUNET_OK}, the service would continue to process the requst +message. + +@c *********************************************************************** +@node Server - Response to client +@subsubsection Server - Response to client +@c %**end of header + +Once the processing of current request is done, the server should give the +response to the client. A new @code{struct AddressLookupMessage} would be +produced by the server in a similar way as the client did and sent to the +client, but here the type should be +@code{GNUNET_MESSAGE_TYPE_TRANSPORT_ADDRESS_REPLY} rather than +@code{GNUNET_MESSAGE_TYPE_TRANSPORT_ADDRESS_LOOKUP} in client. +@example +struct AddressLookupMessage *msg; +size_t len = sizeof (struct AddressLookupMessage) + + addressLen + + strlen (nameTrans) + 1; +msg->header->size = htons (len); +msg->header->type = htons + (GNUNET_MESSAGE_TYPE_TRANSPORT_ADDRESS_REPLY); + +// ... + +struct GNUNET_SERVER_TransmitContext *tc; +tc = GNUNET_SERVER_transmit_context_create (client); +GNUNET_SERVER_transmit_context_append_data +(tc, + NULL, + 0, + GNUNET_MESSAGE_TYPE_TRANSPORT_ADDRESS_REPLY); +GNUNET_SERVER_transmit_context_run (tc, rtimeout); +@end example + + +Note that, there are also a number of other APIs provided to the service +to send the message. + +@c *********************************************************************** +@node Server - Notification of clients +@subsubsection Server - Notification of clients +@c %**end of header + +Often a service needs to (repeatedly) transmit notifications to a client +or a group of clients. In these cases, the client typically has once +registered for a set of events and then needs to receive a message +whenever such an event happens (until the client disconnects). The use of +a notification context can help manage message queues to clients and +handle disconnects. Notification contexts can be used to send +individualized messages to a particular client or to broadcast messages +to a group of clients. An individualized notification might look like +this: + +@example +GNUNET_SERVER_notification_context_unicast(nc, + client, + msg, + GNUNET_YES); +@end example + + +Note that after processing the original registration message for +notifications, the server code still typically needs to call +@code{GNUNET_SERVER_receive_done} so that the client can transmit further +messages to the server. + +@c *********************************************************************** +@node Conversion between Network Byte Order (Big Endian) and Host Byte Order +@subsubsection Conversion between Network Byte Order (Big Endian) and Host Byte Order +@c %** subsub? it's a referenced page on the ipc document. +@c %**end of header + +Here we can simply comprehend big endian and little endian as Network Byte +Order and Host Byte Order respectively. What is the difference between +both two? + +Usually in our host computer we store the data byte as Host Byte Order, +for example, we store a integer in the RAM which might occupies 4 Byte, +as Host Byte Order the higher Byte would be stored at the lower address +of RAM, and the lower Byte would be stored at the higher address of RAM. +However, contrast to this, Network Byte Order just take the totally +opposite way to store the data, says, it will store the lower Byte at the +lower address, and the higher Byte will stay at higher address. + +For the current communication of network, we normally exchange the +information by surveying the data package, every two host wants to +communicate with each other must send and receive data package through +network. In order to maintain the identity of data through the +transmission in the network, the order of the Byte storage must changed +before sending and after receiving the data. + +There ten convenient functions to realize the conversion of Byte Order in +GNUnet, as following: + +@table @asis + +@item uint16_t htons(uint16_t hostshort) Convert host byte order to net +byte order with short int +@item uint32_t htonl(uint32_t hostlong) Convert host byte +order to net byte order with long int +@item uint16_t ntohs(uint16_t netshort) +Convert net byte order to host byte order with short int +@item uint32_t +ntohl(uint32_t netlong) Convert net byte order to host byte order with +long int +@item unsigned long long GNUNET_ntohll (unsigned long long netlonglong) +Convert net byte order to host byte order with long long int +@item unsigned long long GNUNET_htonll (unsigned long long hostlonglong) +Convert host byte order to net byte order with long long int +@item struct GNUNET_TIME_RelativeNBO GNUNET_TIME_relative_hton +(struct GNUNET_TIME_Relative a) Convert relative time to network byte +order. +@item struct GNUNET_TIME_Relative GNUNET_TIME_relative_ntoh +(struct GNUNET_TIME_RelativeNBO a) Convert relative time from network +byte order. +@item struct GNUNET_TIME_AbsoluteNBO GNUNET_TIME_absolute_hton +(struct GNUNET_TIME_Absolute a) Convert relative time to network byte +order. +@item struct GNUNET_TIME_Absolute GNUNET_TIME_absolute_ntoh +(struct GNUNET_TIME_AbsoluteNBO a) Convert relative time from network +byte order. +@end table + +@cindex Cryptography API +@node Cryptography API +@subsection Cryptography API +@c %**end of header + +The gnunetutil APIs provides the cryptographic primitives used in GNUnet. +GNUnet uses 2048 bit RSA keys for the session key exchange and for signing +messages by peers and most other public-key operations. Most researchers +in cryptography consider 2048 bit RSA keys as secure and practically +unbreakable for a long time. The API provides functions to create a fresh +key pair, read a private key from a file (or create a new file if the +file does not exist), encrypt, decrypt, sign, verify and extraction of +the public key into a format suitable for network transmission. + +For the encryption of files and the actual data exchanged between peers +GNUnet uses 256-bit AES encryption. Fresh, session keys are negotiated +for every new connection.@ Again, there is no published technique to +break this cipher in any realistic amount of time. The API provides +functions for generation of keys, validation of keys (important for +checking that decryptions using RSA succeeded), encryption and decryption. + +GNUnet uses SHA-512 for computing one-way hash codes. The API provides +functions to compute a hash over a block in memory or over a file on disk. + +The crypto API also provides functions for randomizing a block of memory, +obtaining a single random number and for generating a permuation of the +numbers 0 to n-1. Random number generation distinguishes between WEAK and +STRONG random number quality; WEAK random numbers are pseudo-random +whereas STRONG random numbers use entropy gathered from the operating +system. + +Finally, the crypto API provides a means to deterministically generate a +1024-bit RSA key from a hash code. These functions should most likely not +be used by most applications; most importantly, +GNUNET_CRYPTO_rsa_key_create_from_hash does not create an RSA-key that +should be considered secure for traditional applications of RSA. + +@cindex Message Queue API +@node Message Queue API +@subsection Message Queue API +@c %**end of header + +@strong{ Introduction }@ +Often, applications need to queue messages that +are to be sent to other GNUnet peers, clients or services. As all of +GNUnet's message-based communication APIs, by design, do not allow +messages to be queued, it is common to implement custom message queues +manually when they are needed. However, writing very similar code in +multiple places is tedious and leads to code duplication. + +MQ (for Message Queue) is an API that provides the functionality to +implement and use message queues. We intend to eventually replace all of +the custom message queue implementations in GNUnet with MQ. + +@strong{ Basic Concepts }@ +The two most important entities in MQ are queues and envelopes. + +Every queue is backed by a specific implementation (e.g. for mesh, stream, +connection, server client, etc.) that will actually deliver the queued +messages. For convenience,@ some queues also allow to specify a list of +message handlers. The message queue will then also wait for incoming +messages and dispatch them appropriately. + +An envelope holds the the memory for a message, as well as metadata +(Where is the envelope queued? What should happen after it has been +sent?). Any envelope can only be queued in one message queue. + +@strong{ Creating Queues }@ +The following is a list of currently available message queues. Note that +to avoid layering issues, message queues for higher level APIs are not +part of @code{libgnunetutil}, but@ the respective API itself provides the +queue implementation. + +@table @asis + +@item @code{GNUNET_MQ_queue_for_connection_client} +Transmits queued messages over a @code{GNUNET_CLIENT_Connection} handle. +Also supports receiving with message handlers. + +@item @code{GNUNET_MQ_queue_for_server_client} +Transmits queued messages over a @code{GNUNET_SERVER_Client} handle. Does +not support incoming message handlers. + +@item @code{GNUNET_MESH_mq_create} Transmits queued messages over a +@code{GNUNET_MESH_Tunnel} handle. Does not support incoming message +handlers. + +@item @code{GNUNET_MQ_queue_for_callbacks} This is the most general +implementation. Instead of delivering and receiving messages with one of +GNUnet's communication APIs, implementation callbacks are called. Refer to +"Implementing Queues" for a more detailed explanation. +@end table + + +@strong{ Allocating Envelopes }@ +A GNUnet message (as defined by the GNUNET_MessageHeader) has three +parts: The size, the type, and the body. + +MQ provides macros to allocate an envelope containing a message +conveniently, automatically setting the size and type fields of the +message. + +Consider the following simple message, with the body consisting of a +single number value. +@c why the empy code function? +@code{} + +@example +struct NumberMessage @{ + /** Type: GNUNET_MESSAGE_TYPE_EXAMPLE_1 */ + struct GNUNET_MessageHeader header; + uint32_t number GNUNET_PACKED; +@}; +@end example + +An envelope containing an instance of the NumberMessage can be +constructed like this: + +@example +struct GNUNET_MQ_Envelope *ev; +struct NumberMessage *msg; +ev = GNUNET_MQ_msg (msg, GNUNET_MESSAGE_TYPE_EXAMPLE_1); +msg->number = htonl (42); +@end example + +In the above code, @code{GNUNET_MQ_msg} is a macro. The return value is +the newly allocated envelope. The first argument must be a pointer to some +@code{struct} containing a @code{struct GNUNET_MessageHeader header} +field, while the second argument is the desired message type, in host +byte order. + +The @code{msg} pointer now points to an allocated message, where the +message type and the message size are already set. The message's size is +inferred from the type of the @code{msg} pointer: It will be set to +'sizeof(*msg)', properly converted to network byte order. + +If the message body's size is dynamic, the the macro +@code{GNUNET_MQ_msg_extra} can be used to allocate an envelope whose +message has additional space allocated after the @code{msg} structure. + +If no structure has been defined for the message, +@code{GNUNET_MQ_msg_header_extra} can be used to allocate additional space +after the message header. The first argument then must be a pointer to a +@code{GNUNET_MessageHeader}. + +@strong{Envelope Properties}@ +A few functions in MQ allow to set additional properties on envelopes: + +@table @asis + +@item @code{GNUNET_MQ_notify_sent} Allows to specify a function that will +be called once the envelope's message has been sent irrevocably. +An envelope can be canceled precisely up to the@ point where the notify +sent callback has been called. + +@item @code{GNUNET_MQ_disable_corking} No corking will be used when +sending the message. Not every@ queue supports this flag, per default, +envelopes are sent with corking.@ + +@end table + + +@strong{Sending Envelopes}@ +Once an envelope has been constructed, it can be queued for sending with +@code{GNUNET_MQ_send}. + +Note that in order to avoid memory leaks, an envelope must either be sent +(the queue will free it) or destroyed explicitly with +@code{GNUNET_MQ_discard}. + +@strong{Canceling Envelopes}@ +An envelope queued with @code{GNUNET_MQ_send} can be canceled with +@code{GNUNET_MQ_cancel}. Note that after the notify sent callback has +been called, canceling a message results in undefined behavior. +Thus it is unsafe to cancel an envelope that does not have a notify sent +callback. When canceling an envelope, it is not necessary@ to call +@code{GNUNET_MQ_discard}, and the envelope can't be sent again. + +@strong{ Implementing Queues }@ +@code{TODO} + +@cindex Service API +@node Service API +@subsection Service API +@c %**end of header + +Most GNUnet code lives in the form of services. Services are processes +that offer an API for other components of the system to build on. Those +other components can be command-line tools for users, graphical user +interfaces or other services. Services provide their API using an IPC +protocol. For this, each service must listen on either a TCP port or a +UNIX domain socket; for this, the service implementation uses the server +API. This use of server is exposed directly to the users of the service +API. Thus, when using the service API, one is usually also often using +large parts of the server API. The service API provides various +convenience functions, such as parsing command-line arguments and the +configuration file, which are not found in the server API. +The dual to the service/server API is the client API, which can be used to +access services. + +The most common way to start a service is to use the +@code{GNUNET_SERVICE_run} function from the program's main function. +@code{GNUNET_SERVICE_run} will then parse the command line and +configuration files and, based on the options found there, +start the server. It will then give back control to the main +program, passing the server and the configuration to the +@code{GNUNET_SERVICE_Main} callback. @code{GNUNET_SERVICE_run} +will also take care of starting the scheduler loop. +If this is inappropriate (for example, because the scheduler loop +is already running), @code{GNUNET_SERVICE_start} and +related functions provide an alternative to @code{GNUNET_SERVICE_run}. + +When starting a service, the service_name option is used to determine +which sections in the configuration file should be used to configure the +service. A typical value here is the name of the @file{src/} +sub-directory, for example @file{statistics}. +The same string would also be given to +@code{GNUNET_CLIENT_connect} to access the service. + +Once a service has been initialized, the program should use the +@code{GNUNET_SERVICE_Main} callback to register message handlers +using @code{GNUNET_SERVER_add_handlers}. +The service will already have registered a handler for the +"TEST" message. + +@fnindex GNUNET_SERVICE_Options +The option bitfield (@code{enum GNUNET_SERVICE_Options}) +determines how a service should behave during shutdown. +There are three key strategies: + +@table @asis + +@item instant (@code{GNUNET_SERVICE_OPTION_NONE}) +Upon receiving the shutdown +signal from the scheduler, the service immediately terminates the server, +closing all existing connections with clients. +@item manual (@code{GNUNET_SERVICE_OPTION_MANUAL_SHUTDOWN}) +The service does nothing by itself +during shutdown. The main program will need to take the appropriate +action by calling GNUNET_SERVER_destroy or GNUNET_SERVICE_stop (depending +on how the service was initialized) to terminate the service. This method +is used by gnunet-service-arm and rather uncommon. +@item soft (@code{GNUNET_SERVICE_OPTION_SOFT_SHUTDOWN}) +Upon receiving the shutdown signal from the scheduler, +the service immediately tells the server to stop +listening for incoming clients. Requests from normal existing clients are +still processed and the server/service terminates once all normal clients +have disconnected. Clients that are not expected to ever disconnect (such +as clients that monitor performance values) can be marked as 'monitor' +clients using GNUNET_SERVER_client_mark_monitor. Those clients will +continue to be processed until all 'normal' clients have disconnected. +Then, the server will terminate, closing the monitor connections. +This mode is for example used by 'statistics', allowing existing 'normal' +clients to set (possibly persistent) statistic values before terminating. + +@end table + +@c *********************************************************************** +@node Optimizing Memory Consumption of GNUnet's (Multi-) Hash Maps +@subsection Optimizing Memory Consumption of GNUnet's (Multi-) Hash Maps +@c %**end of header + +A commonly used data structure in GNUnet is a (multi-)hash map. It is most +often used to map a peer identity to some data structure, but also to map +arbitrary keys to values (for example to track requests in the distributed +hash table or in file-sharing). As it is commonly used, the DHT is +actually sometimes responsible for a large share of GNUnet's overall +memory consumption (for some processes, 30% is not uncommon). The +following text documents some API quirks (and their implications for +applications) that were recently introduced to minimize the footprint of +the hash map. + + +@c *********************************************************************** +@menu +* Analysis:: +* Solution:: +* Migration:: +* Conclusion:: +* Availability:: +@end menu + +@node Analysis +@subsubsection Analysis +@c %**end of header + +The main reason for the "excessive" memory consumption by the hash map is +that GNUnet uses 512-bit cryptographic hash codes --- and the +(multi-)hash map also uses the same 512-bit 'struct GNUNET_HashCode'. As +a result, storing just the keys requires 64 bytes of memory for each key. +As some applications like to keep a large number of entries in the hash +map (after all, that's what maps are good for), 64 bytes per hash is +significant: keeping a pointer to the value and having a linked list for +collisions consume between 8 and 16 bytes, and 'malloc' may add about the +same overhead per allocation, putting us in the 16 to 32 byte per entry +ballpark. Adding a 64-byte key then triples the overall memory +requirement for the hash map. + +To make things "worse", most of the time storing the key in the hash map +is not required: it is typically already in memory elsewhere! In most +cases, the values stored in the hash map are some application-specific +struct that _also_ contains the hash. Here is a simplified example: + +@example +struct MyValue @{ +struct GNUNET_HashCode key; +unsigned int my_data; @}; + +// ... +val = GNUNET_malloc (sizeof (struct MyValue)); +val->key = key; +val->my_data = 42; +GNUNET_CONTAINER_multihashmap_put (map, &key, val, ...); +@end example + +This is a common pattern as later the entries might need to be removed, +and at that time it is convenient to have the key immediately at hand: + +@example +GNUNET_CONTAINER_multihashmap_remove (map, &val->key, val); +@end example + + +Note that here we end up with two times 64 bytes for the key, plus maybe +64 bytes total for the rest of the 'struct MyValue' and the map entry in +the hash map. The resulting redundant storage of the key increases +overall memory consumption per entry from the "optimal" 128 bytes to 192 +bytes. This is not just an extreme example: overheads in practice are +actually sometimes close to those highlighted in this example. This is +especially true for maps with a significant number of entries, as there +we tend to really try to keep the entries small. + +@c *********************************************************************** +@node Solution +@subsubsection Solution +@c %**end of header + +The solution that has now been implemented is to @strong{optionally} +allow the hash map to not make a (deep) copy of the hash but instead have +a pointer to the hash/key in the entry. This reduces the memory +consumption for the key from 64 bytes to 4 to 8 bytes. However, it can +also only work if the key is actually stored in the entry (which is the +case most of the time) and if the entry does not modify the key (which in +all of the code I'm aware of has been always the case if there key is +stored in the entry). Finally, when the client stores an entry in the +hash map, it @strong{must} provide a pointer to the key within the entry, +not just a pointer to a transient location of the key. If +the client code does not meet these requirements, the result is a dangling +pointer and undefined behavior of the (multi-)hash map API. + +@c *********************************************************************** +@node Migration +@subsubsection Migration +@c %**end of header + +To use the new feature, first check that the values contain the respective +key (and never modify it). Then, all calls to +@code{GNUNET_CONTAINER_multihashmap_put} on the respective map must be +audited and most likely changed to pass a pointer into the value's struct. +For the initial example, the new code would look like this: + +@example +struct MyValue @{ +struct GNUNET_HashCode key; +unsigned int my_data; @}; + +// ... +val = GNUNET_malloc (sizeof (struct MyValue)); +val->key = key; val->my_data = 42; +GNUNET_CONTAINER_multihashmap_put (map, &val->key, val, ...); +@end example + + +Note that @code{&val} was changed to @code{&val->key} in the argument to +the @code{put} call. This is critical as often @code{key} is on the stack +or in some other transient data structure and thus having the hash map +keep a pointer to @code{key} would not work. Only the key inside of +@code{val} has the same lifetime as the entry in the map (this must of +course be checked as well). Naturally, @code{val->key} must be +intiialized before the @code{put} call. Once all @code{put} calls have +been converted and double-checked, you can change the call to create the +hash map from + +@example +map = +GNUNET_CONTAINER_multihashmap_create (SIZE, GNUNET_NO); +@end example + +to + +@example +map = GNUNET_CONTAINER_multihashmap_create (SIZE, GNUNET_YES); +@end example + +If everything was done correctly, you now use about 60 bytes less memory +per entry in @code{map}. However, if now (or in the future) any call to +@code{put} does not ensure that the given key is valid until the entry is +removed from the map, undefined behavior is likely to be observed. + +@c *********************************************************************** +@node Conclusion +@subsubsection Conclusion +@c %**end of header + +The new optimization can is often applicable and can result in a +reduction in memory consumption of up to 30% in practice. However, it +makes the code less robust as additional invariants are imposed on the +multi hash map client. Thus applications should refrain from enabling the +new mode unless the resulting performance increase is deemed significant +enough. In particular, it should generally not be used in new code (wait +at least until benchmarks exist). + +@c *********************************************************************** +@node Availability +@subsubsection Availability +@c %**end of header + +The new multi hash map code was committed in SVN 24319 (will be in GNUnet +0.9.4). Various subsystems (transport, core, dht, file-sharing) were +previously audited and modified to take advantage of the new capability. +In particular, memory consumption of the file-sharing service is expected +to drop by 20-30% due to this change. + + +@cindex CONTAINER_MDLL API +@node CONTAINER_MDLL API +@subsection CONTAINER_MDLL API +@c %**end of header + +This text documents the GNUNET_CONTAINER_MDLL API. The +GNUNET_CONTAINER_MDLL API is similar to the GNUNET_CONTAINER_DLL API in +that it provides operations for the construction and manipulation of +doubly-linked lists. The key difference to the (simpler) DLL-API is that +the MDLL-version allows a single element (instance of a "struct") to be +in multiple linked lists at the same time. + +Like the DLL API, the MDLL API stores (most of) the data structures for +the doubly-linked list with the respective elements; only the 'head' and +'tail' pointers are stored "elsewhere" --- and the application needs to +provide the locations of head and tail to each of the calls in the +MDLL API. The key difference for the MDLL API is that the "next" and +"previous" pointers in the struct can no longer be simply called "next" +and "prev" --- after all, the element may be in multiple doubly-linked +lists, so we cannot just have one "next" and one "prev" pointer! + +The solution is to have multiple fields that must have a name of the +format "next_XX" and "prev_XX" where "XX" is the name of one of the +doubly-linked lists. Here is a simple example: + +@example +struct MyMultiListElement @{ + struct MyMultiListElement *next_ALIST; + struct MyMultiListElement *prev_ALIST; + struct MyMultiListElement *next_BLIST; + struct MyMultiListElement *prev_BLIST; + void + *data; +@}; +@end example + + +Note that by convention, we use all-uppercase letters for the list names. +In addition, the program needs to have a location for the head and tail +pointers for both lists, for example: + +@example +static struct MyMultiListElement *head_ALIST; +static struct MyMultiListElement *tail_ALIST; +static struct MyMultiListElement *head_BLIST; +static struct MyMultiListElement *tail_BLIST; +@end example + + +Using the MDLL-macros, we can now insert an element into the ALIST: + +@example +GNUNET_CONTAINER_MDLL_insert (ALIST, head_ALIST, tail_ALIST, element); +@end example + + +Passing "ALIST" as the first argument to MDLL specifies which of the +next/prev fields in the 'struct MyMultiListElement' should be used. The +extra "ALIST" argument and the "_ALIST" in the names of the +next/prev-members are the only differences between the MDDL and DLL-API. +Like the DLL-API, the MDLL-API offers functions for inserting (at head, +at tail, after a given element) and removing elements from the list. +Iterating over the list should be done by directly accessing the +"next_XX" and/or "prev_XX" members. + +@cindex Automatic Restart Manager +@cindex ARM +@node Automatic Restart Manager (ARM) +@section Automatic Restart Manager (ARM) +@c %**end of header + +GNUnet's Automated Restart Manager (ARM) is the GNUnet service responsible +for system initialization and service babysitting. ARM starts and halts +services, detects configuration changes and restarts services impacted by +the changes as needed. It's also responsible for restarting services in +case of crashes and is planned to incorporate automatic debugging for +diagnosing service crashes providing developers insights about crash +reasons. The purpose of this document is to give GNUnet developer an idea +about how ARM works and how to interact with it. + +@menu +* Basic functionality:: +* Key configuration options:: +* ARM - Availability:: +* Reliability:: +@end menu + +@c *********************************************************************** +@node Basic functionality +@subsection Basic functionality +@c %**end of header + +@itemize @bullet +@item ARM source code can be found under "src/arm".@ Service processes are +managed by the functions in "gnunet-service-arm.c" which is controlled +with "gnunet-arm.c" (main function in that file is ARM's entry point). + +@item The functions responsible for communicating with ARM , starting and +stopping services -including ARM service itself- are provided by the +ARM API "arm_api.c".@ Function: GNUNET_ARM_connect() returns to the caller +an ARM handle after setting it to the caller's context (configuration and +scheduler in use). This handle can be used afterwards by the caller to +communicate with ARM. Functions GNUNET_ARM_start_service() and +GNUNET_ARM_stop_service() are used for starting and stopping services +respectively. + +@item A typical example of using these basic ARM services can be found in +file test_arm_api.c. The test case connects to ARM, starts it, then uses +it to start a service "resolver", stops the "resolver" then stops "ARM". +@end itemize + +@c *********************************************************************** +@node Key configuration options +@subsection Key configuration options +@c %**end of header + +Configurations for ARM and services should be available in a .conf file +(As an example, see test_arm_api_data.conf). When running ARM, the +configuration file to use should be passed to the command: + +@example +$ gnunet-arm -s -c configuration_to_use.conf +@end example + +If no configuration is passed, the default configuration file will be used +(see GNUNET_PREFIX/share/gnunet/defaults.conf which is created from +contrib/defaults.conf).@ Each of the services is having a section starting +by the service name between square brackets, for example: "[arm]". +The following options configure how ARM configures or interacts with the +various services: + +@table @asis + +@item PORT Port number on which the service is listening for incoming TCP +connections. ARM will start the services should it notice a request at +this port. + +@item HOSTNAME Specifies on which host the service is deployed. Note +that ARM can only start services that are running on the local system +(but will not check that the hostname matches the local machine name). +This option is used by the @code{gnunet_client_lib.h} implementation to +determine which system to connect to. The default is "localhost". + +@item BINARY The name of the service binary file. + +@item OPTIONS To be passed to the service. + +@item PREFIX A command to pre-pend to the actual command, for example, +running a service with "valgrind" or "gdb" + +@item DEBUG Run in debug mode (much verbosity). + +@item AUTOSTART ARM will listen to UNIX domain socket and/or TCP port of +the service and start the service on-demand. + +@item FORCESTART ARM will always start this service when the peer +is started. + +@item ACCEPT_FROM IPv4 addresses the service accepts connections from. + +@item ACCEPT_FROM6 IPv6 addresses the service accepts connections from. + +@end table + + +Options that impact the operation of ARM overall are in the "[arm]" +section. ARM is a normal service and has (except for AUTOSTART) all of the +options that other services do. In addition, ARM has the +following options: + +@table @asis + +@item GLOBAL_PREFIX Command to be pre-pended to all services that are +going to run. + +@item GLOBAL_POSTFIX Global option that will be supplied to all the +services that are going to run. + +@end table + +@c *********************************************************************** +@node ARM - Availability +@subsection ARM - Availability +@c %**end of header + +As mentioned before, one of the features provided by ARM is starting +services on demand. Consider the example of one service "client" that +wants to connect to another service a "server". The "client" will ask ARM +to run the "server". ARM starts the "server". The "server" starts +listening to incoming connections. The "client" will establish a +connection with the "server". And then, they will start to communicate +together.@ One problem with that scheme is that it's slow!@ +The "client" service wants to communicate with the "server" service at +once and is not willing wait for it to be started and listening to +incoming connections before serving its request.@ One solution for that +problem will be that ARM starts all services as default services. That +solution will solve the problem, yet, it's not quite practical, for some +services that are going to be started can never be used or are going to +be used after a relatively long time.@ +The approach followed by ARM to solve this problem is as follows: + +@itemize @bullet + +@item For each service having a PORT field in the configuration file and +that is not one of the default services ( a service that accepts incoming +connections from clients), ARM creates listening sockets for all addresses +associated with that service. + +@item The "client" will immediately establish a connection with +the "server". + +@item ARM --- pretending to be the "server" --- will listen on the +respective port and notice the incoming connection from the "client" +(but not accept it), instead + +@item Once there is an incoming connection, ARM will start the "server", +passing on the listen sockets (now, the service is started and can do its +work). + +@item Other client services now can directly connect directly to the +"server". + +@end itemize + +@c *********************************************************************** +@node Reliability +@subsection Reliability + +One of the features provided by ARM, is the automatic restart of crashed +services.@ ARM needs to know which of the running services died. Function +"gnunet-service-arm.c/maint_child_death()" is responsible for that. The +function is scheduled to run upon receiving a SIGCHLD signal. The +function, then, iterates ARM's list of services running and monitors +which service has died (crashed). For all crashing services, ARM restarts +them.@ +Now, considering the case of a service having a serious problem causing it +to crash each time it's started by ARM. If ARM keeps blindly restarting +such a service, we are going to have the pattern: +start-crash-restart-crash-restart-crash and so forth!! Which is of course +not practical.@ +For that reason, ARM schedules the service to be restarted after waiting +for some delay that grows exponentially with each crash/restart of that +service.@ To clarify the idea, considering the following example: + +@itemize @bullet + +@item Service S crashed. + +@item ARM receives the SIGCHLD and inspects its list of services to find +the dead one(s). + +@item ARM finds S dead and schedules it for restarting after "backoff" +time which is initially set to 1ms. ARM will double the backoff time +correspondent to S (now backoff(S) = 2ms) + +@item Because there is a severe problem with S, it crashed again. + +@item Again ARM receives the SIGCHLD and detects that it's S again that's +crashed. ARM schedules it for restarting but after its new backoff time +(which became 2ms), and doubles its backoff time (now backoff(S) = 4). + +@item and so on, until backoff(S) reaches a certain threshold +(@code{EXPONENTIAL_BACKOFF_THRESHOLD} is set to half an hour), +after reaching it, backoff(S) will remain half an hour, +hence ARM won't be busy for a lot of time trying to restart a +problematic service. +@end itemize + +@cindex TRANSPORT Subsystem +@node TRANSPORT Subsystem +@section TRANSPORT Subsystem +@c %**end of header + +This chapter documents how the GNUnet transport subsystem works. The +GNUnet transport subsystem consists of three main components: the +transport API (the interface used by the rest of the system to access the +transport service), the transport service itself (most of the interesting +functions, such as choosing transports, happens here) and the transport +plugins. A transport plugin is a concrete implementation for how two +GNUnet peers communicate; many plugins exist, for example for +communication via TCP, UDP, HTTP, HTTPS and others. Finally, the +transport subsystem uses supporting code, especially the NAT/UPnP +library to help with tasks such as NAT traversal. + +Key tasks of the transport service include: + +@itemize @bullet + +@item Create our HELLO message, notify clients and neighbours if our HELLO +changes (using NAT library as necessary) + +@item Validate HELLOs from other peers (send PING), allow other peers to +validate our HELLO's addresses (send PONG) + +@item Upon request, establish connections to other peers (using address +selection from ATS subsystem) and maintain them (again using PINGs and +PONGs) as long as desired + +@item Accept incoming connections, give ATS service the opportunity to +switch communication channels + +@item Notify clients about peers that have connected to us or that have +been disconnected from us + +@item If a (stateful) connection goes down unexpectedly (without explicit +DISCONNECT), quickly attempt to recover (without notifying clients) but do +notify clients quickly if reconnecting fails + +@item Send (payload) messages arriving from clients to other peers via +transport plugins and receive messages from other peers, forwarding +those to clients + +@item Enforce inbound traffic limits (using flow-control if it is +applicable); outbound traffic limits are enforced by CORE, not by us (!) + +@item Enforce restrictions on P2P connection as specified by the blacklist +configuration and blacklisting clients +@end itemize + +Note that the term "clients" in the list above really refers to the +GNUnet-CORE service, as CORE is typically the only client of the +transport service. + +@menu +* Address validation protocol:: +@end menu + +@node Address validation protocol +@subsection Address validation protocol +@c %**end of header + +This section documents how the GNUnet transport service validates +connections with other peers. It is a high-level description of the +protocol necessary to understand the details of the implementation. It +should be noted that when we talk about PING and PONG messages in this +section, we refer to transport-level PING and PONG messages, which are +different from core-level PING and PONG messages (both in implementation +and function). + +The goal of transport-level address validation is to minimize the chances +of a successful man-in-the-middle attack against GNUnet peers on the +transport level. Such an attack would not allow the adversary to decrypt +the P2P transmissions, but a successful attacker could at least measure +traffic volumes and latencies (raising the adversaries capablities by +those of a global passive adversary in the worst case). The scenarios we +are concerned about is an attacker, Mallory, giving a @code{HELLO} to +Alice that claims to be for Bob, but contains Mallory's IP address +instead of Bobs (for some transport). +Mallory would then forward the traffic to Bob (by initiating a +connection to Bob and claiming to be Alice). As a further +complication, the scheme has to work even if say Alice is behind a NAT +without traversal support and hence has no address of her own (and thus +Alice must always initiate the connection to Bob). + +An additional constraint is that @code{HELLO} messages do not contain a +cryptographic signature since other peers must be able to edit +(i.e. remove) addresses from the @code{HELLO} at any time (this was +not true in GNUnet 0.8.x). A basic @strong{assumption} is that each peer +knows the set of possible network addresses that it @strong{might} +be reachable under (so for example, the external IP address of the +NAT plus the LAN address(es) with the respective ports). + +The solution is the following. If Alice wants to validate that a given +address for Bob is valid (i.e. is actually established @strong{directly} +with the intended target), she sends a PING message over that connection +to Bob. Note that in this case, Alice initiated the connection so only +Alice knows which address was used for sure (Alice may be behind NAT, so +whatever address Bob sees may not be an address Alice knows she has). +Bob checks that the address given in the @code{PING} is actually one +of Bob's addresses (ie: does not belong to Mallory), and if it is, +sends back a @code{PONG} (with a signature that says that Bob +owns/uses the address from the @code{PING}). +Alice checks the signature and is happy if it is valid and the address +in the @code{PONG} is the address Alice used. +This is similar to the 0.8.x protocol where the @code{HELLO} contained a +signature from Bob for each address used by Bob. +Here, the purpose code for the signature is +@code{GNUNET_SIGNATURE_PURPOSE_TRANSPORT_PONG_OWN}. After this, Alice will +remember Bob's address and consider the address valid for a while (12h in +the current implementation). Note that after this exchange, Alice only +considers Bob's address to be valid, the connection itself is not +considered 'established'. In particular, Alice may have many addresses +for Bob that Alice considers valid. + +@c TODO: reference Footnotes so that I don't have to duplicate the +@c footnotes or add them to an index at the end. Is this possible at +@c all in Texinfo? +The @code{PONG} message is protected with a nonce/challenge against replay +attacks@footnote{@uref{http://en.wikipedia.org/wiki/Replay_attack, replay}} +and uses an expiration time for the signature (but those are almost +implementation details). + +@cindex NAT library +@node NAT library +@section NAT library +@c %**end of header + +The goal of the GNUnet NAT library is to provide a general-purpose API for +NAT traversal @strong{without} third-party support. So protocols that +involve contacting a third peer to help establish a connection between +two peers are outside of the scope of this API. That does not mean that +GNUnet doesn't support involving a third peer (we can do this with the +distance-vector transport or using application-level protocols), it just +means that the NAT API is not concerned with this possibility. The API is +written so that it will work for IPv6-NAT in the future as well as +current IPv4-NAT. Furthermore, the NAT API is always used, even for peers +that are not behind NAT --- in that case, the mapping provided is simply +the identity. + +NAT traversal is initiated by calling @code{GNUNET_NAT_register}. Given a +set of addresses that the peer has locally bound to (TCP or UDP), the NAT +library will return (via callback) a (possibly longer) list of addresses +the peer @strong{might} be reachable under. Internally, depending on the +configuration, the NAT library will try to punch a hole (using UPnP) or +just "know" that the NAT was manually punched and generate the respective +external IP address (the one that should be globally visible) based on +the given information. + +The NAT library also supports ICMP-based NAT traversal. Here, the other +peer can request connection-reversal by this peer (in this special case, +the peer is even allowed to configure a port number of zero). If the NAT +library detects a connection-reversal request, it returns the respective +target address to the client as well. It should be noted that +connection-reversal is currently only intended for TCP, so other plugins +@strong{must} pass @code{NULL} for the reversal callback. Naturally, the +NAT library also supports requesting connection reversal from a remote +peer (@code{GNUNET_NAT_run_client}). + +Once initialized, the NAT handle can be used to test if a given address is +possibly a valid address for this peer (@code{GNUNET_NAT_test_address}). +This is used for validating our addresses when generating PONGs. + +Finally, the NAT library contains an API to test if our NAT configuration +is correct. Using @code{GNUNET_NAT_test_start} @strong{before} binding to +the respective port, the NAT library can be used to test if the +configuration works. The test function act as a local client, initialize +the NAT traversal and then contact a @code{gnunet-nat-server} (running by +default on @code{gnunet.org}) and ask for a connection to be established. +This way, it is easy to test if the current NAT configuration is valid. + +@node Distance-Vector plugin +@section Distance-Vector plugin +@c %**end of header + +The Distance Vector (DV) transport is a transport mechanism that allows +peers to act as relays for each other, thereby connecting peers that would +otherwise be unable to connect. This gives a larger connection set to +applications that may work better with more peers to choose from (for +example, File Sharing and/or DHT). + +The Distance Vector transport essentially has two functions. The first is +"gossiping" connection information about more distant peers to directly +connected peers. The second is taking messages intended for non-directly +connected peers and encapsulating them in a DV wrapper that contains the +required information for routing the message through forwarding peers. Via +gossiping, optimal routes through the known DV neighborhood are discovered +and utilized and the message encapsulation provides some benefits in +addition to simply getting the message from the correct source to the +proper destination. + +The gossiping function of DV provides an up to date routing table of +peers that are available up to some number of hops. We call this a +fisheye view of the network (like a fish, nearby objects are known while +more distant ones unknown). Gossip messages are sent only to directly +connected peers, but they are sent about other knowns peers within the +"fisheye distance". Whenever two peers connect, they immediately gossip +to each other about their appropriate other neighbors. They also gossip +about the newly connected peer to previously +connected neighbors. In order to keep the routing tables up to date, +disconnect notifications are propogated as gossip as well (because +disconnects may not be sent/received, timeouts are also used remove +stagnant routing table entries). + +Routing of messages via DV is straightforward. When the DV transport is +notified of a message destined for a non-direct neighbor, the appropriate +forwarding peer is selected, and the base message is encapsulated in a DV +message which contains information about the initial peer and the intended +recipient. At each forwarding hop, the initial peer is validated (the +forwarding peer ensures that it has the initial peer in its neighborhood, +otherwise the message is dropped). Next the base message is +re-encapsulated in a new DV message for the next hop in the forwarding +chain (or delivered to the current peer, if it has arrived at the +destination). + +Assume a three peer network with peers Alice, Bob and Carol. Assume that + +@example +Alice <-> Bob and Bob <-> Carol +@end example + +@noindent +are direct (e.g. over TCP or UDP transports) connections, but that +Alice cannot directly connect to Carol. +This may be the case due to NAT or firewall restrictions, or perhaps +based on one of the peers respective configurations. If the Distance +Vector transport is enabled on all three peers, it will automatically +discover (from the gossip protocol) that Alice and Carol can connect via +Bob and provide a "virtual" Alice <-> Carol connection. Routing between +Alice and Carol happens as follows; Alice creates a message destined for +Carol and notifies the DV transport about it. The DV transport at Alice +looks up Carol in the routing table and finds that the message must be +sent through Bob for Carol. The message is encapsulated setting Alice as +the initiator and Carol as the destination and sent to Bob. Bob receives +the messages, verifies that both Alice and Carol are known to Bob, and +re-wraps the message in a new DV message for Carol. +The DV transport at Carol receives this message, unwraps the original +message, and delivers it to Carol as though it came directly from Alice. + +@cindex SMTP plugin +@node SMTP plugin +@section SMTP plugin +@c %**end of header + +This section describes the new SMTP transport plugin for GNUnet as it +exists in the 0.7.x and 0.8.x branch. SMTP support is currently not +available in GNUnet 0.9.x. This page also describes the transport layer +abstraction (as it existed in 0.7.x and 0.8.x) in more detail and gives +some benchmarking results. The performance results presented are quite +old and maybe outdated at this point. + +@itemize @bullet +@item Why use SMTP for a peer-to-peer transport? +@item SMTPHow does it work? +@item How do I configure my peer? +@item How do I test if it works? +@item How fast is it? +@item Is there any additional documentation? +@end itemize + + +@menu +* Why use SMTP for a peer-to-peer transport?:: +* How does it work?:: +* How do I configure my peer?:: +* How do I test if it works?:: +* How fast is it?:: +@end menu + +@node Why use SMTP for a peer-to-peer transport? +@subsection Why use SMTP for a peer-to-peer transport? +@c %**end of header + +There are many reasons why one would not want to use SMTP: + +@itemize @bullet +@item SMTP is using more bandwidth than TCP, UDP or HTTP +@item SMTP has a much higher latency. +@item SMTP requires significantly more computation (encoding and decoding +time) for the peers. +@item SMTP is significantly more complicated to configure. +@item SMTP may be abused by tricking GNUnet into sending mail to@ +non-participating third parties. +@end itemize + +So why would anybody want to use SMTP? +@itemize @bullet +@item SMTP can be used to contact peers behind NAT boxes (in virtual +private networks). +@item SMTP can be used to circumvent policies that limit or prohibit +peer-to-peer traffic by masking as "legitimate" traffic. +@item SMTP uses E-mail addresses which are independent of a specific IP, +which can be useful to address peers that use dynamic IP addresses. +@item SMTP can be used to initiate a connection (e.g. initial address +exchange) and peers can then negotiate the use of a more efficient +protocol (e.g. TCP) for the actual communication. +@end itemize + +In summary, SMTP can for example be used to send a message to a peer +behind a NAT box that has a dynamic IP to tell the peer to establish a +TCP connection to a peer outside of the private network. Even an +extraordinary overhead for this first message would be irrelevant in this +type of situation. + +@node How does it work? +@subsection How does it work? +@c %**end of header + +When a GNUnet peer needs to send a message to another GNUnet peer that has +advertised (only) an SMTP transport address, GNUnet base64-encodes the +message and sends it in an E-mail to the advertised address. The +advertisement contains a filter which is placed in the E-mail header, +such that the receiving host can filter the tagged E-mails and forward it +to the GNUnet peer process. The filter can be specified individually by +each peer and be changed over time. This makes it impossible to censor +GNUnet E-mail messages by searching for a generic filter. + +@node How do I configure my peer? +@subsection How do I configure my peer? +@c %**end of header + +First, you need to configure @code{procmail} to filter your inbound E-mail +for GNUnet traffic. The GNUnet messages must be delivered into a pipe, for +example @code{/tmp/gnunet.smtp}. You also need to define a filter that is +used by @command{procmail} to detect GNUnet messages. You are free to +choose whichever filter you like, but you should make sure that it does +not occur in your other E-mail. In our example, we will use +@code{X-mailer: GNUnet}. The @code{~/.procmailrc} configuration file then +looks like this: + +@example +:0: +* ^X-mailer: GNUnet +/tmp/gnunet.smtp +# where do you want your other e-mail delivered to +# (default: /var/spool/mail/) +:0: /var/spool/mail/ +@end example + +After adding this file, first make sure that your regular E-mail still +works (e.g. by sending an E-mail to yourself). Then edit the GNUnet +configuration. In the section @code{SMTP} you need to specify your E-mail +address under @code{EMAIL}, your mail server (for outgoing mail) under +@code{SERVER}, the filter (X-mailer: GNUnet in the example) under +@code{FILTER} and the name of the pipe under @code{PIPE}.@ The completed +section could then look like this: + +@example +EMAIL = me@@mail.gnu.org MTU = 65000 SERVER = mail.gnu.org:25 FILTER = +"X-mailer: GNUnet" PIPE = /tmp/gnunet.smtp +@end example + +Finally, you need to add @code{smtp} to the list of @code{TRANSPORTS} in +the @code{GNUNETD} section. GNUnet peers will use the E-mail address that +you specified to contact your peer until the advertisement times out. +Thus, if you are not sure if everything works properly or if you are not +planning to be online for a long time, you may want to configure this +timeout to be short, e.g. just one hour. For this, set +@code{HELLOEXPIRES} to @code{1} in the @code{GNUNETD} section. + +This should be it, but you may probably want to test it first. + +@node How do I test if it works? +@subsection How do I test if it works? +@c %**end of header + +Any transport can be subjected to some rudimentary tests using the +@code{gnunet-transport-check} tool. The tool sends a message to the local +node via the transport and checks that a valid message is received. While +this test does not involve other peers and can not check if firewalls or +other network obstacles prohibit proper operation, this is a great +testcase for the SMTP transport since it tests pretty much nearly all of +the functionality. + +@code{gnunet-transport-check} should only be used without running +@code{gnunetd} at the same time. By default, @code{gnunet-transport-check} +tests all transports that are specified in the configuration file. But +you can specifically test SMTP by giving the option +@code{--transport=smtp}. + +Note that this test always checks if a transport can receive and send. +While you can configure most transports to only receive or only send +messages, this test will only work if you have configured the transport +to send and receive messages. + +@node How fast is it? +@subsection How fast is it? +@c %**end of header + +We have measured the performance of the UDP, TCP and SMTP transport layer +directly and when used from an application using the GNUnet core. +Measureing just the transport layer gives the better view of the actual +overhead of the protocol, whereas evaluating the transport from the +application puts the overhead into perspective from a practical point of +view. + +The loopback measurements of the SMTP transport were performed on three +different machines spanning a range of modern SMTP configurations. We +used a PIII-800 running RedHat 7.3 with the Purdue Computer Science +configuration which includes filters for spam. We also used a Xenon 2 GHZ +with a vanilla RedHat 8.0 sendmail configuration. Furthermore, we used +qmail on a PIII-1000 running Sorcerer GNU Linux (SGL). The numbers for +UDP and TCP are provided using the SGL configuration. The qmail benchmark +uses qmail's internal filtering whereas the sendmail benchmarks relies on +procmail to filter and deliver the mail. We used the transport layer to +send a message of b bytes (excluding transport protocol headers) directly +to the local machine. This way, network latency and packet loss on the +wire have no impact on the timings. n messages were sent sequentially over +the transport layer, sending message i+1 after the i-th message was +received. All messages were sent over the same connection and the time to +establish the connection was not taken into account since this overhead is +miniscule in practice --- as long as a connection is used for a +significant number of messages. + +@multitable @columnfractions .20 .15 .15 .15 .15 .15 +@headitem Transport @tab UDP @tab TCP @tab SMTP (Purdue sendmail) +@tab SMTP (RH 8.0) @tab SMTP (SGL qmail) +@item 11 bytes @tab 31 ms @tab 55 ms @tab 781 s @tab 77 s @tab 24 s +@item 407 bytes @tab 37 ms @tab 62 ms @tab 789 s @tab 78 s @tab 25 s +@item 1,221 bytes @tab 46 ms @tab 73 ms @tab 804 s @tab 78 s @tab 25 s +@end multitable + +The benchmarks show that UDP and TCP are, as expected, both significantly +faster compared with any of the SMTP services. Among the SMTP +implementations, there can be significant differences depending on the +SMTP configuration. Filtering with an external tool like procmail that +needs to re-parse its configuration for each mail can be very expensive. +Applying spam filters can also significantly impact the performance of +the underlying SMTP implementation. The microbenchmark shows that SMTP +can be a viable solution for initiating peer-to-peer sessions: a couple of +seconds to connect to a peer are probably not even going to be noticed by +users. The next benchmark measures the possible throughput for a +transport. Throughput can be measured by sending multiple messages in +parallel and measuring packet loss. Note that not only UDP but also the +TCP transport can actually loose messages since the TCP implementation +drops messages if the @code{write} to the socket would block. While the +SMTP protocol never drops messages itself, it is often so +slow that only a fraction of the messages can be sent and received in the +given time-bounds. For this benchmark we report the message loss after +allowing t time for sending m messages. If messages were not sent (or +received) after an overall timeout of t, they were considered lost. The +benchmark was performed using two Xeon 2 GHZ machines running RedHat 8.0 +with sendmail. The machines were connected with a direct 100 MBit ethernet +connection.@ Figures udp1200, tcp1200 and smtp-MTUs show that the +throughput for messages of size 1,200 octects is 2,343 kbps, 3,310 kbps +and 6 kbps for UDP, TCP and SMTP respectively. The high per-message +overhead of SMTP can be improved by increasing the MTU, for example, an +MTU of 12,000 octets improves the throughput to 13 kbps as figure +smtp-MTUs shows. Our research paper) has some more details on the +benchmarking results. + +@cindex Bluetooth plugin +@node Bluetooth plugin +@section Bluetooth plugin +@c %**end of header + +This page describes the new Bluetooth transport plugin for GNUnet. The +plugin is still in the testing stage so don't expect it to work +perfectly. If you have any questions or problems just post them here or +ask on the IRC channel. + +@itemize @bullet +@item What do I need to use the Bluetooth plugin transport? +@item BluetoothHow does it work? +@item What possible errors should I be aware of? +@item How do I configure my peer? +@item How can I test it? +@end itemize + +@menu +* What do I need to use the Bluetooth plugin transport?:: +* How does it work2?:: +* What possible errors should I be aware of?:: +* How do I configure my peer2?:: +* How can I test it?:: +* The implementation of the Bluetooth transport plugin:: +@end menu + +@node What do I need to use the Bluetooth plugin transport? +@subsection What do I need to use the Bluetooth plugin transport? +@c %**end of header + +If you are a GNU/Linux user and you want to use the Bluetooth +transport plugin you should install the +@command{BlueZ development libraries} (if they aren't already +installed). +For instructions about how to install the libraries you should +check out the BlueZ site +(@uref{http://www.bluez.org/, http://www.bluez.org}). If you don't know if +you have the necesarry libraries, don't worry, just run the GNUnet +configure script and you will be able to see a notification at the end +which will warn you if you don't have the necessary libraries. + +If you are a Windows user you should have installed the +@emph{MinGW}/@emph{MSys2} with the latest updates (especially the +@emph{ws2bth} header). If this is your first build of GNUnet on Windows +you should check out the SBuild repository. It will semi-automatically +assembles a @emph{MinGW}/@emph{MSys2} installation with a lot of extra +packages which are needed for the GNUnet build. So this will ease your +work!@ Finally you just have to be sure that you have the correct drivers +for your Bluetooth device installed and that your device is on and in a +discoverable mode. The Windows Bluetooth Stack supports only the RFCOMM +protocol so we cannot turn on your device programatically! + +@c FIXME: Change to unique title +@node How does it work2? +@subsection How does it work2? +@c %**end of header + +The Bluetooth transport plugin uses virtually the same code as the WLAN +plugin and only the helper binary is different. The helper takes a single +argument, which represents the interface name and is specified in the +configuration file. Here are the basic steps that are followed by the +helper binary used on GNU/Linux: + +@itemize @bullet +@item it verifies if the name corresponds to a Bluetooth interface name +@item it verifies if the iterface is up (if it is not, it tries to bring +it up) +@item it tries to enable the page and inquiry scan in order to make the +device discoverable and to accept incoming connection requests +@emph{The above operations require root access so you should start the +transport plugin with root privileges.} +@item it finds an available port number and registers a SDP service which +will be used to find out on which port number is the server listening on +and switch the socket in listening mode +@item it sends a HELLO message with its address +@item finally it forwards traffic from the reading sockets to the STDOUT +and from the STDIN to the writing socket +@end itemize + +Once in a while the device will make an inquiry scan to discover the +nearby devices and it will send them randomly HELLO messages for peer +discovery. + +@node What possible errors should I be aware of? +@subsection What possible errors should I be aware of? +@c %**end of header + +@emph{This section is dedicated for GNU/Linux users} + +Well there are many ways in which things could go wrong but I will try to +present some tools that you could use to debug and some scenarios. + +@itemize @bullet + +@item @code{bluetoothd -n -d} : use this command to enable logging in the +foreground and to print the logging messages + +@item @code{hciconfig}: can be used to configure the Bluetooth devices. +If you run it without any arguments it will print information about the +state of the interfaces. So if you receive an error that the device +couldn't be brought up you should try to bring it manually and to see if +it works (use @code{hciconfig -a hciX up}). If you can't and the +Bluetooth address has the form 00:00:00:00:00:00 it means that there is +something wrong with the D-Bus daemon or with the Bluetooth daemon. Use +@code{bluetoothd} tool to see the logs + +@item @code{sdptool} can be used to control and interogate SDP servers. +If you encounter problems regarding the SDP server (like the SDP server is +down) you should check out if the D-Bus daemon is running correctly and to +see if the Bluetooth daemon started correctly(use @code{bluetoothd} tool). +Also, sometimes the SDP service could work but somehow the device couldn't +register his service. Use @code{sdptool browse [dev-address]} to see if +the service is registered. There should be a service with the name of the +interface and GNUnet as provider. + +@item @code{hcitool} : another useful tool which can be used to configure +the device and to send some particular commands to it. + +@item @code{hcidump} : could be used for low level debugging +@end itemize + +@c FIXME: A more unique name +@node How do I configure my peer2? +@subsection How do I configure my peer2? +@c %**end of header + +On GNU/Linux, you just have to be sure that the interface name +corresponds to the one that you want to use. +Use the @code{hciconfig} tool to check that. +By default it is set to hci0 but you can change it. + +A basic configuration looks like this: + +@example +[transport-bluetooth] +# Name of the interface (typically hciX) +INTERFACE = hci0 +# Real hardware, no testing +TESTMODE = 0 TESTING_IGNORE_KEYS = ACCEPT_FROM; +@end example + +In order to use the Bluetooth transport plugin when the transport service +is started, you must add the plugin name to the default transport service +plugins list. For example: + +@example +[transport] ... PLUGINS = dns bluetooth ... +@end example + +If you want to use only the Bluetooth plugin set +@emph{PLUGINS = bluetooth} + +On Windows, you cannot specify which device to use. The only thing that +you should do is to add @emph{bluetooth} on the plugins list of the +transport service. + +@node How can I test it? +@subsection How can I test it? +@c %**end of header + +If you have two Bluetooth devices on the same machine and you are using +GNU/Linux you must: + +@itemize @bullet + +@item create two different file configuration (one which will use the +first interface (@emph{hci0}) and the other which will use the second +interface (@emph{hci1})). Let's name them @emph{peer1.conf} and +@emph{peer2.conf}. + +@item run @emph{gnunet-peerinfo -c peerX.conf -s} in order to generate the +peers private keys. The @strong{X} must be replace with 1 or 2. + +@item run @emph{gnunet-arm -c peerX.conf -s -i=transport} in order to +start the transport service. (Make sure that you have "bluetooth" on the +transport plugins list if the Bluetooth transport service doesn't start.) + +@item run @emph{gnunet-peerinfo -c peer1.conf -s} to get the first peer's +ID. If you already know your peer ID (you saved it from the first +command), this can be skipped. + +@item run @emph{gnunet-transport -c peer2.conf -p=PEER1_ID -s} to start +sending data for benchmarking to the other peer. + +@end itemize + + +This scenario will try to connect the second peer to the first one and +then start sending data for benchmarking. + +On Windows you cannot test the plugin functionality using two Bluetooth +devices from the same machine because after you install the drivers there +will occur some conflicts between the Bluetooth stacks. (At least that is +what happend on my machine : I wasn't able to use the Bluesoleil stack and +the WINDCOMM one in the same time). + +If you have two different machines and your configuration files are good +you can use the same scenario presented on the begining of this section. + +Another way to test the plugin functionality is to create your own +application which will use the GNUnet framework with the Bluetooth +transport service. + +@node The implementation of the Bluetooth transport plugin +@subsection The implementation of the Bluetooth transport plugin +@c %**end of header + +This page describes the implementation of the Bluetooth transport plugin. + +First I want to remind you that the Bluetooth transport plugin uses +virtually the same code as the WLAN plugin and only the helper binary is +different. Also the scope of the helper binary from the Bluetooth +transport plugin is the same as the one used for the wlan transport +plugin: it acceses the interface and then it forwards traffic in both +directions between the Bluetooth interface and stdin/stdout of the +process involved. + +The Bluetooth plugin transport could be used both on GNU/Linux and Windows +platforms. + +@itemize @bullet +@item Linux functionality +@item Windows functionality +@item Pending Features +@end itemize + + + +@menu +* Linux functionality:: +* THE INITIALIZATION:: +* THE LOOP:: +* Details about the broadcast implementation:: +* Windows functionality:: +* Pending features:: +@end menu + +@node Linux functionality +@subsubsection Linux functionality +@c %**end of header + +In order to implement the plugin functionality on GNU/Linux I +used the BlueZ stack. +For the communication with the other devices I used the RFCOMM +protocol. Also I used the HCI protocol to gain some control over the +device. The helper binary takes a single argument (the name of the +Bluetooth interface) and is separated in two stages: + +@c %** 'THE INITIALIZATION' should be in bigger letters or stand out, not +@c %** starting a new section? +@node THE INITIALIZATION +@subsubsection THE INITIALIZATION + +@itemize @bullet +@item first, it checks if we have root privilegies +(@emph{Remember that we need to have root privilegies in order to be able +to bring the interface up if it is down or to change its state.}). + +@item second, it verifies if the interface with the given name exists. + +@strong{If the interface with that name exists and it is a Bluetooth +interface:} + +@item it creates a RFCOMM socket which will be used for listening and call +the @emph{open_device} method + +On the @emph{open_device} method: +@itemize @bullet +@item creates a HCI socket used to send control events to the the device +@item searches for the device ID using the interface name +@item saves the device MAC address +@item checks if the interface is down and tries to bring it UP +@item checks if the interface is in discoverable mode and tries to make it +discoverable +@item closes the HCI socket and binds the RFCOMM one +@item switches the RFCOMM socket in listening mode +@item registers the SDP service (the service will be used by the other +devices to get the port on which this device is listening on) +@end itemize + +@item drops the root privilegies + +@strong{If the interface is not a Bluetooth interface the helper exits +with a suitable error} +@end itemize + +@c %** Same as for @node entry above +@node THE LOOP +@subsubsection THE LOOP + +The helper binary uses a list where it saves all the connected neighbour +devices (@emph{neighbours.devices}) and two buffers (@emph{write_pout} and +@emph{write_std}). The first message which is send is a control message +with the device's MAC address in order to announce the peer presence to +the neighbours. Here are a short description of what happens in the main +loop: + +@itemize @bullet +@item Every time when it receives something from the STDIN it processes +the data and saves the message in the first buffer (@emph{write_pout}). +When it has something in the buffer, it gets the destination address from +the buffer, searches the destination address in the list (if there is no +connection with that device, it creates a new one and saves it to the +list) and sends the message. +@item Every time when it receives something on the listening socket it +accepts the connection and saves the socket on a list with the reading +sockets. @item Every time when it receives something from a reading +socket it parses the message, verifies the CRC and saves it in the +@emph{write_std} buffer in order to be sent later to the STDOUT. +@end itemize + +So in the main loop we use the select function to wait until one of the +file descriptor saved in one of the two file descriptors sets used is +ready to use. The first set (@emph{rfds}) represents the reading set and +it could contain the list with the reading sockets, the STDIN file +descriptor or the listening socket. The second set (@emph{wfds}) is the +writing set and it could contain the sending socket or the STDOUT file +descriptor. After the select function returns, we check which file +descriptor is ready to use and we do what is supposed to do on that kind +of event. @emph{For example:} if it is the listening socket then we +accept a new connection and save the socket in the reading list; if it is +the STDOUT file descriptor, then we write to STDOUT the message from the +@emph{write_std} buffer. + +To find out on which port a device is listening on we connect to the local +SDP server and searche the registered service for that device. + +@emph{You should be aware of the fact that if the device fails to connect +to another one when trying to send a message it will attempt one more +time. If it fails again, then it skips the message.} +@emph{Also you should know that the transport Bluetooth plugin has +support for @strong{broadcast messages}.} + +@node Details about the broadcast implementation +@subsubsection Details about the broadcast implementation +@c %**end of header + +First I want to point out that the broadcast functionality for the CONTROL +messages is not implemented in a conventional way. Since the inquiry scan +time is too big and it will take some time to send a message to all the +discoverable devices I decided to tackle the problem in a different way. +Here is how I did it: + +@itemize @bullet +@item If it is the first time when I have to broadcast a message I make an +inquiry scan and save all the devices' addresses to a vector. +@item After the inquiry scan ends I take the first address from the list +and I try to connect to it. If it fails, I try to connect to the next one. +If it succeeds, I save the socket to a list and send the message to the +device. +@item When I have to broadcast another message, first I search on the list +for a new device which I'm not connected to. If there is no new device on +the list I go to the beginning of the list and send the message to the +old devices. After 5 cycles I make a new inquiry scan to check out if +there are new discoverable devices and save them to the list. If there +are no new discoverable devices I reset the cycling counter and go again +through the old list and send messages to the devices saved in it. +@end itemize + +@strong{Therefore}: + +@itemize @bullet +@item every time when I have a broadcast message I look up on the list +for a new device and send the message to it +@item if I reached the end of the list for 5 times and I'm connected to +all the devices from the list I make a new inquiry scan. +@emph{The number of the list's cycles after an inquiry scan could be +increased by redefining the MAX_LOOPS variable} +@item when there are no new devices I send messages to the old ones. +@end itemize + +Doing so, the broadcast control messages will reach the devices but with +delay. + +@emph{NOTICE:} When I have to send a message to a certain device first I +check on the broadcast list to see if we are connected to that device. If +not we try to connect to it and in case of success we save the address and +the socket on the list. If we are already connected to that device we +simply use the socket. + +@node Windows functionality +@subsubsection Windows functionality +@c %**end of header + +For Windows I decided to use the Microsoft Bluetooth stack which has the +advantage of coming standard from Windows XP SP2. The main disadvantage is +that it only supports the RFCOMM protocol so we will not be able to have +a low level control over the Bluetooth device. Therefore it is the user +responsability to check if the device is up and in the discoverable mode. +Also there are no tools which could be used for debugging in order to read +the data coming from and going to a Bluetooth device, which obviously +hindered my work. Another thing that slowed down the implementation of the +plugin (besides that I wasn't too accomodated with the win32 API) was that +there were some bugs on MinGW regarding the Bluetooth. Now they are solved +but you should keep in mind that you should have the latest updates +(especially the @emph{ws2bth} header). + +Besides the fact that it uses the Windows Sockets, the Windows +implemenation follows the same principles as the GNU/Linux one: + +@itemize @bullet +@item It has a initalization part where it initializes the +Windows Sockets, creates a RFCOMM socket which will be binded and switched +to the listening mode and registers a SDP service. In the Microsoft +Bluetooth API there are two ways to work with the SDP: +@itemize @bullet +@item an easy way which works with very simple service records +@item a hard way which is useful when you need to update or to delete the +record +@end itemize +@end itemize + +Since I only needed the SDP service to find out on which port the device +is listening on and that did not change, I decided to use the easy way. +In order to register the service I used the @emph{WSASetService} function +and I generated the @emph{Universally Unique Identifier} with the +@emph{guidgen.exe} Windows's tool. + +In the loop section the only difference from the GNU/Linux implementation +is that I used the @code{GNUNET_NETWORK} library for +functions like @emph{accept}, @emph{bind}, @emph{connect} or +@emph{select}. I decided to use the +@code{GNUNET_NETWORK} library because I also needed to interact +with the STDIN and STDOUT handles and on Windows +the select function is only defined for sockets, +and it will not work for arbitrary file handles. + +Another difference between GNU/Linux and Windows implementation is that in +GNU/Linux, the Bluetooth address is represented in 48 bits +while in Windows is represented in 64 bits. +Therefore I had to do some changes on @emph{plugin_transport_wlan} header. + +Also, currently on Windows the Bluetooth plugin doesn't have support for +broadcast messages. When it receives a broadcast message it will skip it. + +@node Pending features +@subsubsection Pending features +@c %**end of header + +@itemize @bullet +@item Implement the broadcast functionality on Windows @emph{(currently +working on)} +@item Implement a testcase for the helper :@ @emph{The testcase +consists of a program which emaluates the plugin and uses the helper. It +will simulate connections, disconnections and data transfers.} +@end itemize + +If you have a new idea about a feature of the plugin or suggestions about +how I could improve the implementation you are welcome to comment or to +contact me. + +@node WLAN plugin +@section WLAN plugin +@c %**end of header + +This section documents how the wlan transport plugin works. Parts which +are not implemented yet or could be better implemented are described at +the end. + +@cindex ATS Subsystem +@node ATS Subsystem +@section ATS Subsystem +@c %**end of header + +ATS stands for "automatic transport selection", and the function of ATS in +GNUnet is to decide on which address (and thus transport plugin) should +be used for two peers to communicate, and what bandwidth limits should be +imposed on such an individual connection. To help ATS make an informed +decision, higher-level services inform the ATS service about their +requirements and the quality of the service rendered. The ATS service +also interacts with the transport service to be appraised of working +addresses and to communicate its resource allocation decisions. Finally, +the ATS service's operation can be observed using a monitoring API. + +The main logic of the ATS service only collects the available addresses, +their performance characteristics and the applications requirements, but +does not make the actual allocation decision. This last critical step is +left to an ATS plugin, as we have implemented (currently three) different +allocation strategies which differ significantly in their performance and +maturity, and it is still unclear if any particular plugin is generally +superior. + +@cindex CORE Subsystem +@node CORE Subsystem +@section CORE Subsystem +@c %**end of header + +The CORE subsystem in GNUnet is responsible for securing link-layer +communications between nodes in the GNUnet overlay network. CORE builds +on the TRANSPORT subsystem which provides for the actual, insecure, +unreliable link-layer communication (for example, via UDP or WLAN), and +then adds fundamental security to the connections: + +@itemize @bullet +@item confidentiality with so-called perfect forward secrecy; we use +ECDHE@footnote{@uref{http://en.wikipedia.org/wiki/Elliptic_curve_Diffie%E2%80%93Hellman, Elliptic-curve Diffie---Hellman}} +powered by Curve25519 +@footnote{@uref{http://cr.yp.to/ecdh.html, Curve25519}} for the key +exchange and then use symmetric encryption, encrypting with both AES-256 +@footnote{@uref{http://en.wikipedia.org/wiki/Rijndael, AES-256}} and +Twofish @footnote{@uref{http://en.wikipedia.org/wiki/Twofish, Twofish}} +@item @uref{http://en.wikipedia.org/wiki/Authentication, authentication} +is achieved by signing the ephemeral keys using Ed25519 +@footnote{@uref{http://ed25519.cr.yp.to/, Ed25519}}, a deterministic +variant of ECDSA +@footnote{@uref{http://en.wikipedia.org/wiki/ECDSA, ECDSA}} +@item integrity protection (using SHA-512 +@footnote{@uref{http://en.wikipedia.org/wiki/SHA-2, SHA-512}} to do +encrypt-then-MAC +@footnote{@uref{http://en.wikipedia.org/wiki/Authenticated_encryption, encrypt-then-MAC}}) +@item Replay +@footnote{@uref{http://en.wikipedia.org/wiki/Replay_attack, replay}} +protection (using nonces, timestamps, challenge-response, +message counters and ephemeral keys) +@item liveness (keep-alive messages, timeout) +@end itemize + +@menu +* Limitations:: +* When is a peer "connected"?:: +* libgnunetcore:: +* The CORE Client-Service Protocol:: +* The CORE Peer-to-Peer Protocol:: +@end menu + +@cindex core subsystem limitations +@node Limitations +@subsection Limitations +@c %**end of header + +CORE does not perform +@uref{http://en.wikipedia.org/wiki/Routing, routing}; using CORE it is +only possible to communicate with peers that happen to already be +"directly" connected with each other. CORE also does not have an +API to allow applications to establish such "direct" connections --- for +this, applications can ask TRANSPORT, but TRANSPORT might not be able to +establish a "direct" connection. The TOPOLOGY subsystem is responsible for +trying to keep a few "direct" connections open at all times. Applications +that need to talk to particular peers should use the CADET subsystem, as +it can establish arbitrary "indirect" connections. + +Because CORE does not perform routing, CORE must only be used directly by +applications that either perform their own routing logic (such as +anonymous file-sharing) or that do not require routing, for example +because they are based on flooding the network. CORE communication is +unreliable and delivery is possibly out-of-order. Applications that +require reliable communication should use the CADET service. Each +application can only queue one message per target peer with the CORE +service at any time; messages cannot be larger than approximately +63 kilobytes. If messages are small, CORE may group multiple messages +(possibly from different applications) prior to encryption. If permitted +by the application (using the @uref{http://baus.net/on-tcp_cork/, cork} +option), CORE may delay transmissions to facilitate grouping of multiple +small messages. If cork is not enabled, CORE will transmit the message as +soon as TRANSPORT allows it (TRANSPORT is responsible for limiting +bandwidth and congestion control). CORE does not allow flow control; +applications are expected to process messages at line-speed. If flow +control is needed, applications should use the CADET service. + +@cindex when is a peer connected +@node When is a peer "connected"? +@subsection When is a peer "connected"? +@c %**end of header + +In addition to the security features mentioned above, CORE also provides +one additional key feature to applications using it, and that is a +limited form of protocol-compatibility checking. CORE distinguishes +between TRANSPORT-level connections (which enable communication with other +peers) and application-level connections. Applications using the CORE API +will (typically) learn about application-level connections from CORE, and +not about TRANSPORT-level connections. When a typical application uses +CORE, it will specify a set of message types +(from @code{gnunet_protocols.h}) that it understands. CORE will then +notify the application about connections it has with other peers if and +only if those applications registered an intersecting set of message +types with their CORE service. Thus, it is quite possible that CORE only +exposes a subset of the established direct connections to a particular +application --- and different applications running above CORE might see +different sets of connections at the same time. + +A special case are applications that do not register a handler for any +message type. +CORE assumes that these applications merely want to monitor connections +(or "all" messages via other callbacks) and will notify those applications +about all connections. This is used, for example, by the +@code{gnunet-core} command-line tool to display the active connections. +Note that it is also possible that the TRANSPORT service has more active +connections than the CORE service, as the CORE service first has to +perform a key exchange with connecting peers before exchanging information +about supported message types and notifying applications about the new +connection. + +@cindex libgnunetcore +@node libgnunetcore +@subsection libgnunetcore +@c %**end of header + +The CORE API (defined in @file{gnunet_core_service.h}) is the basic +messaging API used by P2P applications built using GNUnet. It provides +applications the ability to send and receive encrypted messages to the +peer's "directly" connected neighbours. + +As CORE connections are generally "direct" connections,@ applications must +not assume that they can connect to arbitrary peers this way, as "direct" +connections may not always be possible. Applications using CORE are +notified about which peers are connected. Creating new "direct" +connections must be done using the TRANSPORT API. + +The CORE API provides unreliable, out-of-order delivery. While the +implementation tries to ensure timely, in-order delivery, both message +losses and reordering are not detected and must be tolerated by the +application. Most important, the core will NOT perform retransmission if +messages could not be delivered. + +Note that CORE allows applications to queue one message per connected +peer. The rate at which each connection operates is influenced by the +preferences expressed by local application as well as restrictions +imposed by the other peer. Local applications can express their +preferences for particular connections using the "performance" API of the +ATS service. + +Applications that require more sophisticated transmission capabilities +such as TCP-like behavior, or if you intend to send messages to arbitrary +remote peers, should use the CADET API. + +The typical use of the CORE API is to connect to the CORE service using +@code{GNUNET_CORE_connect}, process events from the CORE service (such as +peers connecting, peers disconnecting and incoming messages) and send +messages to connected peers using +@code{GNUNET_CORE_notify_transmit_ready}. Note that applications must +cancel pending transmission requests if they receive a disconnect event +for a peer that had a transmission pending; furthermore, queueing more +than one transmission request per peer per application using the +service is not permitted. + +The CORE API also allows applications to monitor all communications of the +peer prior to encryption (for outgoing messages) or after decryption (for +incoming messages). This can be useful for debugging, diagnostics or to +establish the presence of cover traffic (for anonymity). As monitoring +applications are often not interested in the payload, the monitoring +callbacks can be configured to only provide the message headers (including +the message type and size) instead of copying the full data stream to the +monitoring client. + +The init callback of the @code{GNUNET_CORE_connect} function is called +with the hash of the public key of the peer. This public key is used to +identify the peer globally in the GNUnet network. Applications are +encouraged to check that the provided hash matches the hash that they are +using (as theoretically the application may be using a different +configuration file with a different private key, which would result in +hard to find bugs). + +As with most service APIs, the CORE API isolates applications from crashes +of the CORE service. If the CORE service crashes, the application will see +disconnect events for all existing connections. Once the connections are +re-established, the applications will be receive matching connect events. + +@cindex core clinet-service protocol +@node The CORE Client-Service Protocol +@subsection The CORE Client-Service Protocol +@c %**end of header + +This section describes the protocol between an application using the CORE +service (the client) and the CORE service process itself. + + +@menu +* Setup2:: +* Notifications:: +* Sending:: +@end menu + +@node Setup2 +@subsubsection Setup2 +@c %**end of header + +When a client connects to the CORE service, it first sends a +@code{InitMessage} which specifies options for the connection and a set of +message type values which are supported by the application. The options +bitmask specifies which events the client would like to be notified about. +The options include: + +@table @asis +@item GNUNET_CORE_OPTION_NOTHING No notifications +@item GNUNET_CORE_OPTION_STATUS_CHANGE Peers connecting and disconnecting +@item GNUNET_CORE_OPTION_FULL_INBOUND All inbound messages (after +decryption) with full payload +@item GNUNET_CORE_OPTION_HDR_INBOUND Just the @code{MessageHeader} +of all inbound messages +@item GNUNET_CORE_OPTION_FULL_OUTBOUND All outbound +messages (prior to encryption) with full payload +@item GNUNET_CORE_OPTION_HDR_OUTBOUND Just the @code{MessageHeader} of all +outbound messages +@end table + +Typical applications will only monitor for connection status changes. + +The CORE service responds to the @code{InitMessage} with an +@code{InitReplyMessage} which contains the peer's identity. Afterwards, +both CORE and the client can send messages. + +@node Notifications +@subsubsection Notifications +@c %**end of header + +The CORE will send @code{ConnectNotifyMessage}s and +@code{DisconnectNotifyMessage}s whenever peers connect or disconnect from +the CORE (assuming their type maps overlap with the message types +registered by the client). When the CORE receives a message that matches +the set of message types specified during the @code{InitMessage} (or if +monitoring is enabled in for inbound messages in the options), it sends a +@code{NotifyTrafficMessage} with the peer identity of the sender and the +decrypted payload. The same message format (except with +@code{GNUNET_MESSAGE_TYPE_CORE_NOTIFY_OUTBOUND} for the message type) is +used to notify clients monitoring outbound messages; here, the peer +identity given is that of the receiver. + +@node Sending +@subsubsection Sending +@c %**end of header + +When a client wants to transmit a message, it first requests a +transmission slot by sending a @code{SendMessageRequest} which specifies +the priority, deadline and size of the message. Note that these values +may be ignored by CORE. When CORE is ready for the message, it answers +with a @code{SendMessageReady} response. The client can then transmit the +payload with a @code{SendMessage} message. Note that the actual message +size in the @code{SendMessage} is allowed to be smaller than the size in +the original request. A client may at any time send a fresh +@code{SendMessageRequest}, which then superceeds the previous +@code{SendMessageRequest}, which is then no longer valid. The client can +tell which @code{SendMessageRequest} the CORE service's +@code{SendMessageReady} message is for as all of these messages contain a +"unique" request ID (based on a counter incremented by the client +for each request). + +@cindex CORE Peer-to-Peer Protocol +@node The CORE Peer-to-Peer Protocol +@subsection The CORE Peer-to-Peer Protocol +@c %**end of header + + +@menu +* Creating the EphemeralKeyMessage:: +* Establishing a connection:: +* Encryption and Decryption:: +* Type maps:: +@end menu + +@cindex EphemeralKeyMessage creation +@node Creating the EphemeralKeyMessage +@subsubsection Creating the EphemeralKeyMessage +@c %**end of header + +When the CORE service starts, each peer creates a fresh ephemeral (ECC) +public-private key pair and signs the corresponding +@code{EphemeralKeyMessage} with its long-term key (which we usually call +the peer's identity; the hash of the public long term key is what results +in a @code{struct GNUNET_PeerIdentity} in all GNUnet APIs. The ephemeral +key is ONLY used for an ECDHE@footnote{@uref{http://en.wikipedia.org/wiki/Elliptic_curve_Diffie%E2%80%93Hellman, Elliptic-curve Diffie---Hellman}} +exchange by the CORE service to establish symmetric session keys. A peer +will use the same @code{EphemeralKeyMessage} for all peers for +@code{REKEY_FREQUENCY}, which is usually 12 hours. After that time, it +will create a fresh ephemeral key (forgetting the old one) and broadcast +the new @code{EphemeralKeyMessage} to all connected peers, resulting in +fresh symmetric session keys. Note that peers independently decide on +when to discard ephemeral keys; it is not a protocol violation to discard +keys more often. Ephemeral keys are also never stored to disk; restarting +a peer will thus always create a fresh ephemeral key. The use of ephemeral +keys is what provides @uref{http://en.wikipedia.org/wiki/Forward_secrecy, forward secrecy}. + +Just before transmission, the @code{EphemeralKeyMessage} is patched to +reflect the current sender_status, which specifies the current state of +the connection from the point of view of the sender. The possible values +are: + +@itemize @bullet +@item @code{KX_STATE_DOWN} Initial value, never used on the network +@item @code{KX_STATE_KEY_SENT} We sent our ephemeral key, do not know the +key of the other peer +@item @code{KX_STATE_KEY_RECEIVED} This peer has received a valid +ephemeral key of the other peer, but we are waiting for the other peer to +confirm it's authenticity (ability to decode) via challenge-response. +@item @code{KX_STATE_UP} The connection is fully up from the point of +view of the sender (now performing keep-alives) +@item @code{KX_STATE_REKEY_SENT} The sender has initiated a rekeying +operation; the other peer has so far failed to confirm a working +connection using the new ephemeral key +@end itemize + +@node Establishing a connection +@subsubsection Establishing a connection +@c %**end of header + +Peers begin their interaction by sending a @code{EphemeralKeyMessage} to +the other peer once the TRANSPORT service notifies the CORE service about +the connection. +A peer receiving an @code{EphemeralKeyMessage} with a status +indicating that the sender does not have the receiver's ephemeral key, the +receiver's @code{EphemeralKeyMessage} is sent in response. +Additionally, if the receiver has not yet confirmed the authenticity of +the sender, it also sends an (encrypted)@code{PingMessage} with a +challenge (and the identity of the target) to the other peer. Peers +receiving a @code{PingMessage} respond with an (encrypted) +@code{PongMessage} which includes the challenge. Peers receiving a +@code{PongMessage} check the challenge, and if it matches set the +connection to @code{KX_STATE_UP}. + +@node Encryption and Decryption +@subsubsection Encryption and Decryption +@c %**end of header + +All functions related to the key exchange and encryption/decryption of +messages can be found in @file{gnunet-service-core_kx.c} (except for the +cryptographic primitives, which are in @file{util/crypto*.c}). +Given the key material from ECDHE, a Key derivation function +@footnote{@uref{https://en.wikipedia.org/wiki/Key_derivation_function, Key derivation function}} +is used to derive two pairs of encryption and decryption keys for AES-256 +and TwoFish, as well as initialization vectors and authentication keys +(for HMAC@footnote{@uref{https://en.wikipedia.org/wiki/HMAC, HMAC}}). +The HMAC is computed over the encrypted payload. +Encrypted messages include an iv_seed and the HMAC in the header. + +Each encrypted message in the CORE service includes a sequence number and +a timestamp in the encrypted payload. The CORE service remembers the +largest observed sequence number and a bit-mask which represents which of +the previous 32 sequence numbers were already used. +Messages with sequence numbers lower than the largest observed sequence +number minus 32 are discarded. Messages with a timestamp that is less +than @code{REKEY_TOLERANCE} off (5 minutes) are also discarded. This of +course means that system clocks need to be reasonably synchronized for +peers to be able to communicate. Additionally, as the ephemeral key +changes every 12 hours, a peer would not even be able to decrypt messages +older than 12 hours. + +@node Type maps +@subsubsection Type maps +@c %**end of header + +Once an encrypted connection has been established, peers begin to exchange +type maps. Type maps are used to allow the CORE service to determine which +(encrypted) connections should be shown to which applications. A type map +is an array of 65536 bits representing the different types of messages +understood by applications using the CORE service. Each CORE service +maintains this map, simply by setting the respective bit for each message +type supported by any of the applications using the CORE service. Note +that bits for message types embedded in higher-level protocols (such as +MESH) will not be included in these type maps. + +Typically, the type map of a peer will be sparse. Thus, the CORE service +attempts to compress its type map using @code{gzip}-style compression +("deflate") prior to transmission. However, if the compression fails to +compact the map, the map may also be transmitted without compression +(resulting in @code{GNUNET_MESSAGE_TYPE_CORE_COMPRESSED_TYPE_MAP} or +@code{GNUNET_MESSAGE_TYPE_CORE_BINARY_TYPE_MAP} messages respectively). +Upon receiving a type map, the respective CORE service notifies +applications about the connection to the other peer if they support any +message type indicated in the type map (or no message type at all). +If the CORE service experience a connect or disconnect event from an +application, it updates its type map (setting or unsetting the respective +bits) and notifies its neighbours about the change. +The CORE services of the neighbours then in turn generate connect and +disconnect events for the peer that sent the type map for their respective +applications. As CORE messages may be lost, the CORE service confirms +receiving a type map by sending back a +@code{GNUNET_MESSAGE_TYPE_CORE_CONFIRM_TYPE_MAP}. If such a confirmation +(with the correct hash of the type map) is not received, the sender will +retransmit the type map (with exponential back-off). + +@cindex CADET Subsystem +@node CADET Subsystem +@section CADET Subsystem + +The CADET subsystem in GNUnet is responsible for secure end-to-end +communications between nodes in the GNUnet overlay network. CADET builds +on the CORE subsystem which provides for the link-layer communication and +then adds routing, forwarding and additional security to the connections. +CADET offers the same cryptographic services as CORE, but on an +end-to-end level. This is done so peers retransmitting traffic on behalf +of other peers cannot access the payload data. + +@itemize @bullet +@item CADET provides confidentiality with so-called perfect forward +secrecy; we use ECDHE powered by Curve25519 for the key exchange and then +use symmetric encryption, encrypting with both AES-256 and Twofish +@item authentication is achieved by signing the ephemeral keys using +Ed25519, a deterministic variant of ECDSA +@item integrity protection (using SHA-512 to do encrypt-then-MAC, although +only 256 bits are sent to reduce overhead) +@item replay protection (using nonces, timestamps, challenge-response, +message counters and ephemeral keys) +@item liveness (keep-alive messages, timeout) +@end itemize + +Additional to the CORE-like security benefits, CADET offers other +properties that make it a more universal service than CORE. + +@itemize @bullet +@item CADET can establish channels to arbitrary peers in GNUnet. If a +peer is not immediately reachable, CADET will find a path through the +network and ask other peers to retransmit the traffic on its behalf. +@item CADET offers (optional) reliability mechanisms. In a reliable +channel traffic is guaranteed to arrive complete, unchanged and in-order. +@item CADET takes care of flow and congestion control mechanisms, not +allowing the sender to send more traffic than the receiver or the network +are able to process. +@end itemize + +@menu +* libgnunetcadet:: +@end menu + +@cindex libgnunetcadet +@node libgnunetcadet +@subsection libgnunetcadet + + +The CADET API (defined in @file{gnunet_cadet_service.h}) is the +messaging API used by P2P applications built using GNUnet. +It provides applications the ability to send and receive encrypted +messages to any peer participating in GNUnet. +The API is heavily base on the CORE API. + +CADET delivers messages to other peers in "channels". +A channel is a permanent connection defined by a destination peer +(identified by its public key) and a port number. +Internally, CADET tunnels all channels towards a destiantion peer +using one session key and relays the data on multiple "connections", +independent from the channels. + +Each channel has optional paramenters, the most important being the +reliability flag. +Should a message get lost on TRANSPORT/CORE level, if a channel is +created with as reliable, CADET will retransmit the lost message and +deliver it in order to the destination application. + +To communicate with other peers using CADET, it is necessary to first +connect to the service using @code{GNUNET_CADET_connect}. +This function takes several parameters in form of callbacks, to allow the +client to react to various events, like incoming channels or channels that +terminate, as well as specify a list of ports the client wishes to listen +to (at the moment it is not possible to start listening on further ports +once connected, but nothing prevents a client to connect several times to +CADET, even do one connection per listening port). +The function returns a handle which has to be used for any further +interaction with the service. + +To connect to a remote peer a client has to call the +@code{GNUNET_CADET_channel_create} function. The most important parameters +given are the remote peer's identity (it public key) and a port, which +specifies which application on the remote peer to connect to, similar to +TCP/UDP ports. CADET will then find the peer in the GNUnet network and +establish the proper low-level connections and do the necessary key +exchanges to assure and authenticated, secure and verified communication. +Similar to @code{GNUNET_CADET_connect},@code{GNUNET_CADET_create_channel} +returns a handle to interact with the created channel. + +For every message the client wants to send to the remote application, +@code{GNUNET_CADET_notify_transmit_ready} must be called, indicating the +channel on which the message should be sent and the size of the message +(but not the message itself!). Once CADET is ready to send the message, +the provided callback will fire, and the message contents are provided to +this callback. + +Please note the CADET does not provide an explicit notification of when a +channel is connected. In loosely connected networks, like big wireless +mesh networks, this can take several seconds, even minutes in the worst +case. To be alerted when a channel is online, a client can call +@code{GNUNET_CADET_notify_transmit_ready} immediately after +@code{GNUNET_CADET_create_channel}. When the callback is activated, it +means that the channel is online. The callback can give 0 bytes to CADET +if no message is to be sent, this is ok. + +If a transmission was requested but before the callback fires it is no +longer needed, it can be cancelled with +@code{GNUNET_CADET_notify_transmit_ready_cancel}, which uses the handle +given back by @code{GNUNET_CADET_notify_transmit_ready}. +As in the case of CORE, only one message can be requested at a time: a +client must not call @code{GNUNET_CADET_notify_transmit_ready} again until +the callback is called or the request is cancelled. + +When a channel is no longer needed, a client can call +@code{GNUNET_CADET_channel_destroy} to get rid of it. +Note that CADET will try to transmit all pending traffic before notifying +the remote peer of the destruction of the channel, including +retransmitting lost messages if the channel was reliable. + +Incoming channels, channels being closed by the remote peer, and traffic +on any incoming or outgoing channels are given to the client when CADET +executes the callbacks given to it at the time of +@code{GNUNET_CADET_connect}. + +Finally, when an application no longer wants to use CADET, it should call +@code{GNUNET_CADET_disconnect}, but first all channels and pending +transmissions must be closed (otherwise CADET will complain). + +@cindex NSE Subsystem +@node NSE Subsystem +@section NSE Subsystem + + +NSE stands for @dfn{Network Size Estimation}. The NSE subsystem provides +other subsystems and users with a rough estimate of the number of peers +currently participating in the GNUnet overlay. +The computed value is not a precise number as producing a precise number +in a decentralized, efficient and secure way is impossible. +While NSE's estimate is inherently imprecise, NSE also gives the expected +range. For a peer that has been running in a stable network for a +while, the real network size will typically (99.7% of the time) be in the +range of [2/3 estimate, 3/2 estimate]. We will now give an overview of the +algorithm used to calculate the estimate; +all of the details can be found in this technical report. + +@c FIXME: link to the report. + +@menu +* Motivation:: +* Principle:: +* libgnunetnse:: +* The NSE Client-Service Protocol:: +* The NSE Peer-to-Peer Protocol:: +@end menu + +@node Motivation +@subsection Motivation + + +Some subsytems, like DHT, need to know the size of the GNUnet network to +optimize some parameters of their own protocol. The decentralized nature +of GNUnet makes efficient and securely counting the exact number of peers +infeasable. Although there are several decentralized algorithms to count +the number of peers in a system, so far there is none to do so securely. +Other protocols may allow any malicious peer to manipulate the final +result or to take advantage of the system to perform +@dfn{Denial of Service} (DoS) attacks against the network. +GNUnet's NSE protocol avoids these drawbacks. + + + +@menu +* Security:: +@end menu + +@cindex NSE security +@cindex nse security +@node Security +@subsubsection Security + + +The NSE subsystem is designed to be resilient against these attacks. +It uses @uref{http://en.wikipedia.org/wiki/Proof-of-work_system, proofs of work} +to prevent one peer from impersonating a large number of participants, +which would otherwise allow an adversary to artifically inflate the +estimate. +The DoS protection comes from the time-based nature of the protocol: +the estimates are calculated periodically and out-of-time traffic is +either ignored or stored for later retransmission by benign peers. +In particular, peers cannot trigger global network communication at will. + +@cindex NSE principle +@cindex nse principle +@node Principle +@subsection Principle + + +The algorithm calculates the estimate by finding the globally closest +peer ID to a random, time-based value. + +The idea is that the closer the ID is to the random value, the more +"densely packed" the ID space is, and therefore, more peers are in the +network. + + + +@menu +* Example:: +* Algorithm:: +* Target value:: +* Timing:: +* Controlled Flooding:: +* Calculating the estimate:: +@end menu + +@node Example +@subsubsection Example + + +Suppose all peers have IDs between 0 and 100 (our ID space), and the +random value is 42. +If the closest peer has the ID 70 we can imagine that the average +"distance" between peers is around 30 and therefore the are around 3 +peers in the whole ID space. On the other hand, if the closest peer has +the ID 44, we can imagine that the space is rather packed with peers, +maybe as much as 50 of them. +Naturally, we could have been rather unlucky, and there is only one peer +and happens to have the ID 44. Thus, the current estimate is calculated +as the average over multiple rounds, and not just a single sample. + +@node Algorithm +@subsubsection Algorithm + + +Given that example, one can imagine that the job of the subsystem is to +efficiently communicate the ID of the closest peer to the target value +to all the other peers, who will calculate the estimate from it. + +@node Target value +@subsubsection Target value + +@c %**end of header + +The target value itself is generated by hashing the current time, rounded +down to an agreed value. If the rounding amount is 1h (default) and the +time is 12:34:56, the time to hash would be 12:00:00. The process is +repeated each rouning amount (in this example would be every hour). +Every repetition is called a round. + +@node Timing +@subsubsection Timing +@c %**end of header + +The NSE subsystem has some timing control to avoid everybody broadcasting +its ID all at one. Once each peer has the target random value, it +compares its own ID to the target and calculates the hypothetical size of +the network if that peer were to be the closest. +Then it compares the hypothetical size with the estimate from the previous +rounds. For each value there is an assiciated point in the period, +let's call it "broadcast time". If its own hypothetical estimate +is the same as the previous global estimate, its "broadcast time" will be +in the middle of the round. If its bigger it will be earlier and if its +smaller (the most likely case) it will be later. This ensures that the +peers closests to the target value start broadcasting their ID the first. + +@node Controlled Flooding +@subsubsection Controlled Flooding + +@c %**end of header + +When a peer receives a value, first it verifies that it is closer than the +closest value it had so far, otherwise it answers the incoming message +with a message containing the better value. Then it checks a proof of +work that must be included in the incoming message, to ensure that the +other peer's ID is not made up (otherwise a malicious peer could claim to +have an ID of exactly the target value every round). Once validated, it +compares the brodcast time of the received value with the current time +and if it's not too early, sends the received value to its neighbors. +Otherwise it stores the value until the correct broadcast time comes. +This prevents unnecessary traffic of sub-optimal values, since a better +value can come before the broadcast time, rendering the previous one +obsolete and saving the traffic that would have been used to broadcast it +to the neighbors. + +@node Calculating the estimate +@subsubsection Calculating the estimate + +@c %**end of header + +Once the closest ID has been spread across the network each peer gets the +exact distance betweed this ID and the target value of the round and +calculates the estimate with a mathematical formula described in the tech +report. The estimate generated with this method for a single round is not +very precise. Remember the case of the example, where the only peer is the +ID 44 and we happen to generate the target value 42, thinking there are +50 peers in the network. Therefore, the NSE subsystem remembers the last +64 estimates and calculates an average over them, giving a result of which +usually has one bit of uncertainty (the real size could be half of the +estimate or twice as much). Note that the actual network size is +calculated in powers of two of the raw input, thus one bit of uncertainty +means a factor of two in the size estimate. + +@cindex libgnunetnse +@node libgnunetnse +@subsection libgnunetnse + +@c %**end of header + +The NSE subsystem has the simplest API of all services, with only two +calls: @code{GNUNET_NSE_connect} and @code{GNUNET_NSE_disconnect}. + +The connect call gets a callback function as a parameter and this function +is called each time the network agrees on an estimate. This usually is +once per round, with some exceptions: if the closest peer has a late +local clock and starts spreading his ID after everyone else agreed on a +value, the callback might be activated twice in a round, the second value +being always bigger than the first. The default round time is set to +1 hour. + +The disconnect call disconnects from the NSE subsystem and the callback +is no longer called with new estimates. + + + +@menu +* Results:: +* libgnunetnse - Examples:: +@end menu + +@node Results +@subsubsection Results + +@c %**end of header + +The callback provides two values: the average and the +@uref{http://en.wikipedia.org/wiki/Standard_deviation, standard deviation} +of the last 64 rounds. The values provided by the callback function are +logarithmic, this means that the real estimate numbers can be obtained by +calculating 2 to the power of the given value (2average). From a +statistics point of view this means that: + +@itemize @bullet +@item 68% of the time the real size is included in the interval +[(2average-stddev), 2] +@item 95% of the time the real size is included in the interval +[(2average-2*stddev, 2^average+2*stddev] +@item 99.7% of the time the real size is included in the interval +[(2average-3*stddev, 2average+3*stddev] +@end itemize + +The expected standard variation for 64 rounds in a network of stable size +is 0.2. Thus, we can say that normally: + +@itemize @bullet +@item 68% of the time the real size is in the range [-13%, +15%] +@item 95% of the time the real size is in the range [-24%, +32%] +@item 99.7% of the time the real size is in the range [-34%, +52%] +@end itemize + +As said in the introduction, we can be quite sure that usually the real +size is between one third and three times the estimate. This can of +course vary with network conditions. +Thus, applications may want to also consider the provided standard +deviation value, not only the average (in particular, if the standard +veriation is very high, the average maybe meaningless: the network size is +changing rapidly). + +@node libgnunetnse - Examples +@subsubsection libgnunetnse -Examples + +@c %**end of header + +Let's close with a couple examples. + +@table @asis + +@item Average: 10, std dev: 1 Here the estimate would be +2^10 = 1024 peers. @footnote{The range in which we can be 95% sure is: +[2^8, 2^12] = [256, 4096]. We can be very (>99.7%) sure that the network +is not a hundred peers and absolutely sure that it is not a million peers, +but somewhere around a thousand.} + +@item Average 22, std dev: 0.2 Here the estimate would be +2^22 = 4 Million peers. @footnote{The range in which we can be 99.7% sure +is: [2^21.4, 2^22.6] = [2.8M, 6.3M]. We can be sure that the network size +is around four million, with absolutely way of it being 1 million.} + +@end table + +To put this in perspective, if someone remembers the LHC Higgs boson +results, were announced with "5 sigma" and "6 sigma" certainties. In this +case a 5 sigma minimum would be 2 million and a 6 sigma minimum, +1.8 million. + +@node The NSE Client-Service Protocol +@subsection The NSE Client-Service Protocol + +@c %**end of header + +As with the API, the client-service protocol is very simple, only has 2 +different messages, defined in @code{src/nse/nse.h}: + +@itemize @bullet +@item @code{GNUNET_MESSAGE_TYPE_NSE_START}@ This message has no parameters +and is sent from the client to the service upon connection. +@item @code{GNUNET_MESSAGE_TYPE_NSE_ESTIMATE}@ This message is sent from +the service to the client for every new estimate and upon connection. +Contains a timestamp for the estimate, the average and the standard +deviation for the respective round. +@end itemize + +When the @code{GNUNET_NSE_disconnect} API call is executed, the client +simply disconnects from the service, with no message involved. + +@cindex NSE Peer-to-Peer Protocol +@node The NSE Peer-to-Peer Protocol +@subsection The NSE Peer-to-Peer Protocol + +@c %**end of header + +The NSE subsystem only has one message in the P2P protocol, the +@code{GNUNET_MESSAGE_TYPE_NSE_P2P_FLOOD} message. + +This message key contents are the timestamp to identify the round +(differences in system clocks may cause some peers to send messages way +too early or way too late, so the timestamp allows other peers to +identify such messages easily), the +@uref{http://en.wikipedia.org/wiki/Proof-of-work_system, proof of work} +used to make it difficult to mount a +@uref{http://en.wikipedia.org/wiki/Sybil_attack, Sybil attack}, and the +public key, which is used to verify the signature on the message. + +Every peer stores a message for the previous, current and next round. The +messages for the previous and current round are given to peers that +connect to us. The message for the next round is simply stored until our +system clock advances to the next round. The message for the current round +is what we are flooding the network with right now. +At the beginning of each round the peer does the following: + +@itemize @bullet +@item calculates his own distance to the target value +@item creates, signs and stores the message for the current round (unless +it has a better message in the "next round" slot which came early in the +previous round) +@item calculates, based on the stored round message (own or received) when +to stard flooding it to its neighbors +@end itemize + +Upon receiving a message the peer checks the validity of the message +(round, proof of work, signature). The next action depends on the +contents of the incoming message: + +@itemize @bullet +@item if the message is worse than the current stored message, the peer +sends the current message back immediately, to stop the other peer from +spreading suboptimal results +@item if the message is better than the current stored message, the peer +stores the new message and calculates the new target time to start +spreading it to its neighbors (excluding the one the message came from) +@item if the message is for the previous round, it is compared to the +message stored in the "previous round slot", which may then be updated +@item if the message is for the next round, it is compared to the message +stored in the "next round slot", which again may then be updated +@end itemize + +Finally, when it comes to send the stored message for the current round to +the neighbors there is a random delay added for each neighbor, to avoid +traffic spikes and minimize cross-messages. + +@cindex HOSTLIST Subsystem +@node HOSTLIST Subsystem +@section HOSTLIST Subsystem + +@c %**end of header + +Peers in the GNUnet overlay network need address information so that they +can connect with other peers. GNUnet uses so called HELLO messages to +store and exchange peer addresses. +GNUnet provides several methods for peers to obtain this information: + +@itemize @bullet +@item out-of-band exchange of HELLO messages (manually, using for example +gnunet-peerinfo) +@item HELLO messages shipped with GNUnet (automatic with distribution) +@item UDP neighbor discovery in LAN (IPv4 broadcast, IPv6 multicast) +@item topology gossiping (learning from other peers we already connected +to), and +@item the HOSTLIST daemon covered in this section, which is particularly +relevant for bootstrapping new peers. +@end itemize + +New peers have no existing connections (and thus cannot learn from gossip +among peers), may not have other peers in their LAN and might be started +with an outdated set of HELLO messages from the distribution. +In this case, getting new peers to connect to the network requires either +manual effort or the use of a HOSTLIST to obtain HELLOs. + +@menu +* HELLOs:: +* Overview for the HOSTLIST subsystem:: +* Interacting with the HOSTLIST daemon:: +* Hostlist security address validation:: +* The HOSTLIST daemon:: +* The HOSTLIST server:: +* The HOSTLIST client:: +* Usage:: +@end menu + +@node HELLOs +@subsection HELLOs + +@c %**end of header + +The basic information peers require to connect to other peers are +contained in so called HELLO messages you can think of as a business card. +Besides the identity of the peer (based on the cryptographic public key) a +HELLO message may contain address information that specifies ways to +contact a peer. By obtaining HELLO messages, a peer can learn how to +contact other peers. + +@node Overview for the HOSTLIST subsystem +@subsection Overview for the HOSTLIST subsystem + +@c %**end of header + +The HOSTLIST subsystem provides a way to distribute and obtain contact +information to connect to other peers using a simple HTTP GET request. +It's implementation is split in three parts, the main file for the daemon +itself (@file{gnunet-daemon-hostlist.c}), the HTTP client used to download +peer information (@file{hostlist-client.c}) and the server component used +to provide this information to other peers (@file{hostlist-server.c}). +The server is basically a small HTTP web server (based on GNU +libmicrohttpd) which provides a list of HELLOs known to the local peer for +download. The client component is basically a HTTP client +(based on libcurl) which can download hostlists from one or more websites. +The hostlist format is a binary blob containing a sequence of HELLO +messages. Note that any HTTP server can theoretically serve a hostlist, +the build-in hostlist server makes it simply convenient to offer this +service. + + +@menu +* Features:: +* HOSTLIST - Limitations:: +@end menu + +@node Features +@subsubsection Features + +@c %**end of header + +The HOSTLIST daemon can: + +@itemize @bullet +@item provide HELLO messages with validated addresses obtained from +PEERINFO to download for other peers +@item download HELLO messages and forward these message to the TRANSPORT +subsystem for validation +@item advertises the URL of this peer's hostlist address to other peers +via gossip +@item automatically learn about hostlist servers from the gossip of other +peers +@end itemize + +@node HOSTLIST - Limitations +@subsubsection HOSTLIST - Limitations + +@c %**end of header + +The HOSTLIST daemon does not: + +@itemize @bullet +@item verify the cryptographic information in the HELLO messages +@item verify the address information in the HELLO messages +@end itemize + +@node Interacting with the HOSTLIST daemon +@subsection Interacting with the HOSTLIST daemon + +@c %**end of header + +The HOSTLIST subsystem is currently implemented as a daemon, so there is +no need for the user to interact with it and therefore there is no +command line tool and no API to communicate with the daemon. In the +future, we can envision changing this to allow users to manually trigger +the download of a hostlist. + +Since there is no command line interface to interact with HOSTLIST, the +only way to interact with the hostlist is to use STATISTICS to obtain or +modify information about the status of HOSTLIST: + +@example +$ gnunet-statistics -s hostlist +@end example + +@noindent +In particular, HOSTLIST includes a @strong{persistent} value in statistics +that specifies when the hostlist server might be queried next. As this +value is exponentially increasing during runtime, developers may want to +reset or manually adjust it. Note that HOSTLIST (but not STATISTICS) needs +to be shutdown if changes to this value are to have any effect on the +daemon (as HOSTLIST does not monitor STATISTICS for changes to the +download frequency). + +@node Hostlist security address validation +@subsection Hostlist security address validation + +@c %**end of header + +Since information obtained from other parties cannot be trusted without +validation, we have to distinguish between @emph{validated} and +@emph{not validated} addresses. Before using (and so trusting) +information from other parties, this information has to be double-checked +(validated). Address validation is not done by HOSTLIST but by the +TRANSPORT service. + +The HOSTLIST component is functionally located between the PEERINFO and +the TRANSPORT subsystem. When acting as a server, the daemon obtains valid +(@emph{validated}) peer information (HELLO messages) from the PEERINFO +service and provides it to other peers. When acting as a client, it +contacts the HOSTLIST servers specified in the configuration, downloads +the (unvalidated) list of HELLO messages and forwards these information +to the TRANSPORT server to validate the addresses. + +@cindex HOSTLIST daemon +@node The HOSTLIST daemon +@subsection The HOSTLIST daemon + +@c %**end of header + +The hostlist daemon is the main component of the HOSTLIST subsystem. It is +started by the ARM service and (if configured) starts the HOSTLIST client +and server components. + +If the daemon provides a hostlist itself it can advertise it's own +hostlist to other peers. To do so it sends a +@code{GNUNET_MESSAGE_TYPE_HOSTLIST_ADVERTISEMENT} message to other peers +when they connect to this peer on the CORE level. This hostlist +advertisement message contains the URL to access the HOSTLIST HTTP +server of the sender. The daemon may also subscribe to this type of +message from CORE service, and then forward these kind of message to the +HOSTLIST client. The client then uses all available URLs to download peer +information when necessary. + +When starting, the HOSTLIST daemon first connects to the CORE subsystem +and if hostlist learning is enabled, registers a CORE handler to receive +this kind of messages. Next it starts (if configured) the client and +server. It passes pointers to CORE connect and disconnect and receive +handlers where the client and server store their functions, so the daemon +can notify them about CORE events. + +To clean up on shutdown, the daemon has a cleaning task, shutting down all +subsystems and disconnecting from CORE. + +@cindex HOSTLIST server +@node The HOSTLIST server +@subsection The HOSTLIST server + +@c %**end of header + +The server provides a way for other peers to obtain HELLOs. Basically it +is a small web server other peers can connect to and download a list of +HELLOs using standard HTTP; it may also advertise the URL of the hostlist +to other peers connecting on CORE level. + + +@menu +* The HTTP Server:: +* Advertising the URL:: +@end menu + +@node The HTTP Server +@subsubsection The HTTP Server + +@c %**end of header + +During startup, the server starts a web server listening on the port +specified with the HTTPPORT value (default 8080). In addition it connects +to the PEERINFO service to obtain peer information. The HOSTLIST server +uses the GNUNET_PEERINFO_iterate function to request HELLO information for +all peers and adds their information to a new hostlist if they are +suitable (expired addresses and HELLOs without addresses are both not +suitable) and the maximum size for a hostlist is not exceeded +(MAX_BYTES_PER_HOSTLISTS = 500000). +When PEERINFO finishes (with a last NULL callback), the server destroys +the previous hostlist response available for download on the web server +and replaces it with the updated hostlist. The hostlist format is +basically a sequence of HELLO messages (as obtained from PEERINFO) without +any special tokenization. Since each HELLO message contains a size field, +the response can easily be split into separate HELLO messages by the +client. + +A HOSTLIST client connecting to the HOSTLIST server will receive the +hostlist as a HTTP response and the the server will terminate the +connection with the result code @code{HTTP 200 OK}. +The connection will be closed immediately if no hostlist is available. + +@node Advertising the URL +@subsubsection Advertising the URL + +@c %**end of header + +The server also advertises the URL to download the hostlist to other peers +if hostlist advertisement is enabled. +When a new peer connects and has hostlist learning enabled, the server +sends a @code{GNUNET_MESSAGE_TYPE_HOSTLIST_ADVERTISEMENT} message to this +peer using the CORE service. + +@cindex HOSTLIST client +@node The HOSTLIST client +@subsection The HOSTLIST client + +@c %**end of header + +The client provides the functionality to download the list of HELLOs from +a set of URLs. +It performs a standard HTTP request to the URLs configured and learned +from advertisement messages received from other peers. When a HELLO is +downloaded, the HOSTLIST client forwards the HELLO to the TRANSPORT +service for validation. + +The client supports two modes of operation: + +@itemize @bullet +@item download of HELLOs (bootstrapping) +@item learning of URLs +@end itemize + +@menu +* Bootstrapping:: +* Learning:: +@end menu + +@node Bootstrapping +@subsubsection Bootstrapping + +@c %**end of header + +For bootstrapping, it schedules a task to download the hostlist from the +set of known URLs. +The downloads are only performed if the number of current +connections is smaller than a minimum number of connections +(at the moment 4). +The interval between downloads increases exponentially; however, the +exponential growth is limited if it becomes longer than an hour. +At that point, the frequency growth is capped at +(#number of connections * 1h). + +Once the decision has been taken to download HELLOs, the daemon chooses a +random URL from the list of known URLs. URLs can be configured in the +configuration or be learned from advertisement messages. +The client uses a HTTP client library (libcurl) to initiate the download +using the libcurl multi interface. +Libcurl passes the data to the callback_download function which +stores the data in a buffer if space is available and the maximum size for +a hostlist download is not exceeded (MAX_BYTES_PER_HOSTLISTS = 500000). +When a full HELLO was downloaded, the HOSTLIST client offers this +HELLO message to the TRANSPORT service for validation. +When the download is finished or failed, statistical information about the +quality of this URL is updated. + +@cindex HOSTLIST learning +@node Learning +@subsubsection Learning + +@c %**end of header + +The client also manages hostlist advertisements from other peers. The +HOSTLIST daemon forwards @code{GNUNET_MESSAGE_TYPE_HOSTLIST_ADVERTISEMENT} +messages to the client subsystem, which extracts the URL from the message. +Next, a test of the newly obtained URL is performed by triggering a +download from the new URL. If the URL works correctly, it is added to the +list of working URLs. + +The size of the list of URLs is restricted, so if an additional server is +added and the list is full, the URL with the worst quality ranking +(determined through successful downloads and number of HELLOs e.g.) is +discarded. During shutdown the list of URLs is saved to a file for +persistance and loaded on startup. URLs from the configuration file are +never discarded. + +@node Usage +@subsection Usage + +@c %**end of header + +To start HOSTLIST by default, it has to be added to the DEFAULTSERVICES +section for the ARM services. This is done in the default configuration. + +For more information on how to configure the HOSTLIST subsystem see the +installation handbook:@ +Configuring the hostlist to bootstrap@ +Configuring your peer to provide a hostlist + +@cindex IDENTITY Subsystem +@node IDENTITY Subsystem +@section IDENTITY Subsystem + +@c %**end of header + +Identities of "users" in GNUnet are called egos. +Egos can be used as pseudonyms ("fake names") or be tied to an +organization (for example, "GNU") or even the actual identity of a human. +GNUnet users are expected to have many egos. They might have one tied to +their real identity, some for organizations they manage, and more for +different domains where they want to operate under a pseudonym. + +The IDENTITY service allows users to manage their egos. The identity +service manages the private keys egos of the local user; it does not +manage identities of other users (public keys). Public keys for other +users need names to become manageable. GNUnet uses the +@dfn{GNU Name System} (GNS) to give names to other users and manage their +public keys securely. This chapter is about the IDENTITY service, +which is about the management of private keys. + +On the network, an ego corresponds to an ECDSA key (over Curve25519, +using RFC 6979, as required by GNS). Thus, users can perform actions +under a particular ego by using (signing with) a particular private key. +Other users can then confirm that the action was really performed by that +ego by checking the signature against the respective public key. + +The IDENTITY service allows users to associate a human-readable name with +each ego. This way, users can use names that will remind them of the +purpose of a particular ego. +The IDENTITY service will store the respective private keys and +allows applications to access key information by name. +Users can change the name that is locally (!) associated with an ego. +Egos can also be deleted, which means that the private key will be removed +and it thus will not be possible to perform actions with that ego in the +future. + +Additionally, the IDENTITY subsystem can associate service functions with +egos. +For example, GNS requires the ego that should be used for the shorten +zone. GNS will ask IDENTITY for an ego for the "gns-short" service. +The IDENTITY service has a mapping of such service strings to the name of +the ego that the user wants to use for this service, for example +"my-short-zone-ego". + +Finally, the IDENTITY API provides access to a special ego, the +anonymous ego. The anonymous ego is special in that its private key is not +really private, but fixed and known to everyone. +Thus, anyone can perform actions as anonymous. This can be useful as with +this trick, code does not have to contain a special case to distinguish +between anonymous and pseudonymous egos. + +@menu +* libgnunetidentity:: +* The IDENTITY Client-Service Protocol:: +@end menu + +@cindex libgnunetidentity +@node libgnunetidentity +@subsection libgnunetidentity +@c %**end of header + + +@menu +* Connecting to the service:: +* Operations on Egos:: +* The anonymous Ego:: +* Convenience API to lookup a single ego:: +* Associating egos with service functions:: +@end menu + +@node Connecting to the service +@subsubsection Connecting to the service + +@c %**end of header + +First, typical clients connect to the identity service using +@code{GNUNET_IDENTITY_connect}. This function takes a callback as a +parameter. +If the given callback parameter is non-null, it will be invoked to notify +the application about the current state of the identities in the system. + +@itemize @bullet +@item First, it will be invoked on all known egos at the time of the +connection. For each ego, a handle to the ego and the user's name for the +ego will be passed to the callback. Furthermore, a @code{void **} context +argument will be provided which gives the client the opportunity to +associate some state with the ego. +@item Second, the callback will be invoked with NULL for the ego, the name +and the context. This signals that the (initial) iteration over all egos +has completed. +@item Then, the callback will be invoked whenever something changes about +an ego. +If an ego is renamed, the callback is invoked with the ego handle of the +ego that was renamed, and the new name. If an ego is deleted, the callback +is invoked with the ego handle and a name of NULL. In the deletion case, +the application should also release resources stored in the context. +@item When the application destroys the connection to the identity service +using @code{GNUNET_IDENTITY_disconnect}, the callback is again invoked +with the ego and a name of NULL (equivalent to deletion of the egos). +This should again be used to clean up the per-ego context. +@end itemize + +The ego handle passed to the callback remains valid until the callback is +invoked with a name of NULL, so it is safe to store a reference to the +ego's handle. + +@node Operations on Egos +@subsubsection Operations on Egos + +@c %**end of header + +Given an ego handle, the main operations are to get its associated private +key using @code{GNUNET_IDENTITY_ego_get_private_key} or its associated +public key using @code{GNUNET_IDENTITY_ego_get_public_key}. + +The other operations on egos are pretty straightforward. +Using @code{GNUNET_IDENTITY_create}, an application can request the +creation of an ego by specifying the desired name. +The operation will fail if that name is +already in use. Using @code{GNUNET_IDENTITY_rename} the name of an +existing ego can be changed. Finally, egos can be deleted using +@code{GNUNET_IDENTITY_delete}. All of these operations will trigger +updates to the callback given to the @code{GNUNET_IDENTITY_connect} +function of all applications that are connected with the identity service +at the time. @code{GNUNET_IDENTITY_cancel} can be used to cancel the +operations before the respective continuations would be called. +It is not guaranteed that the operation will not be completed anyway, +only the continuation will no longer be called. + +@node The anonymous Ego +@subsubsection The anonymous Ego + +@c %**end of header + +A special way to obtain an ego handle is to call +@code{GNUNET_IDENTITY_ego_get_anonymous}, which returns an ego for the +"anonymous" user --- anyone knows and can get the private key for this +user, so it is suitable for operations that are supposed to be anonymous +but require signatures (for example, to avoid a special path in the code). +The anonymous ego is always valid and accessing it does not require a +connection to the identity service. + +@node Convenience API to lookup a single ego +@subsubsection Convenience API to lookup a single ego + + +As applications commonly simply have to lookup a single ego, there is a +convenience API to do just that. Use @code{GNUNET_IDENTITY_ego_lookup} to +lookup a single ego by name. Note that this is the user's name for the +ego, not the service function. The resulting ego will be returned via a +callback and will only be valid during that callback. The operation can +be cancelled via @code{GNUNET_IDENTITY_ego_lookup_cancel} +(cancellation is only legal before the callback is invoked). + +@node Associating egos with service functions +@subsubsection Associating egos with service functions + + +The @code{GNUNET_IDENTITY_set} function is used to associate a particular +ego with a service function. The name used by the service and the ego are +given as arguments. +Afterwards, the service can use its name to lookup the associated ego +using @code{GNUNET_IDENTITY_get}. + +@node The IDENTITY Client-Service Protocol +@subsection The IDENTITY Client-Service Protocol + +@c %**end of header + +A client connecting to the identity service first sends a message with +type +@code{GNUNET_MESSAGE_TYPE_IDENTITY_START} to the service. After that, the +client will receive information about changes to the egos by receiving +messages of type @code{GNUNET_MESSAGE_TYPE_IDENTITY_UPDATE}. +Those messages contain the private key of the ego and the user's name of +the ego (or zero bytes for the name to indicate that the ego was deleted). +A special bit @code{end_of_list} is used to indicate the end of the +initial iteration over the identity service's egos. + +The client can trigger changes to the egos by sending @code{CREATE}, +@code{RENAME} or @code{DELETE} messages. +The CREATE message contains the private key and the desired name.@ +The RENAME message contains the old name and the new name.@ +The DELETE message only needs to include the name of the ego to delete.@ +The service responds to each of these messages with a @code{RESULT_CODE} +message which indicates success or error of the operation, and possibly +a human-readable error message. + +Finally, the client can bind the name of a service function to an ego by +sending a @code{SET_DEFAULT} message with the name of the service function +and the private key of the ego. +Such bindings can then be resolved using a @code{GET_DEFAULT} message, +which includes the name of the service function. The identity service +will respond to a GET_DEFAULT request with a SET_DEFAULT message +containing the respective information, or with a RESULT_CODE to +indicate an error. + +@cindex NAMESTORE Subsystem +@node NAMESTORE Subsystem +@section NAMESTORE Subsystem + +The NAMESTORE subsystem provides persistent storage for local GNS zone +information. All local GNS zone information are managed by NAMESTORE. It +provides both the functionality to administer local GNS information (e.g. +delete and add records) as well as to retrieve GNS information (e.g to +list name information in a client). +NAMESTORE does only manage the persistent storage of zone information +belonging to the user running the service: GNS information from other +users obtained from the DHT are stored by the NAMECACHE subsystem. + +NAMESTORE uses a plugin-based database backend to store GNS information +with good performance. Here sqlite, MySQL and PostgreSQL are supported +database backends. +NAMESTORE clients interact with the IDENTITY subsystem to obtain +cryptographic information about zones based on egos as described with the +IDENTITY subsystem, but internally NAMESTORE refers to zones using the +ECDSA private key. +In addition, it collaborates with the NAMECACHE subsystem and +stores zone information when local information are modified in the +GNS cache to increase look-up performance for local information. + +NAMESTORE provides functionality to look-up and store records, to iterate +over a specific or all zones and to monitor zones for changes. NAMESTORE +functionality can be accessed using the NAMESTORE api or the NAMESTORE +command line tool. + +@menu +* libgnunetnamestore:: +@end menu + +@cindex libgnunetnamestore +@node libgnunetnamestore +@subsection libgnunetnamestore + +To interact with NAMESTORE clients first connect to the NAMESTORE service +using the @code{GNUNET_NAMESTORE_connect} passing a configuration handle. +As a result they obtain a NAMESTORE handle, they can use for operations, +or NULL is returned if the connection failed. + +To disconnect from NAMESTORE, clients use +@code{GNUNET_NAMESTORE_disconnect} and specify the handle to disconnect. + +NAMESTORE internally uses the ECDSA private key to refer to zones. These +private keys can be obtained from the IDENTITY subsytem. +Here @emph{egos} @emph{can be used to refer to zones or the default ego +assigned to the GNS subsystem can be used to obtained the master zone's +private key.} + + +@menu +* Editing Zone Information:: +* Iterating Zone Information:: +* Monitoring Zone Information:: +@end menu + +@node Editing Zone Information +@subsubsection Editing Zone Information + +@c %**end of header + +NAMESTORE provides functions to lookup records stored under a label in a +zone and to store records under a label in a zone. + +To store (and delete) records, the client uses the +@code{GNUNET_NAMESTORE_records_store} function and has to provide +namestore handle to use, the private key of the zone, the label to store +the records under, the records and number of records plus an callback +function. +After the operation is performed NAMESTORE will call the provided +callback function with the result GNUNET_SYSERR on failure +(including timeout/queue drop/failure to validate), GNUNET_NO if content +was already there or not found GNUNET_YES (or other positive value) on +success plus an additional error message. + +Records are deleted by using the store command with 0 records to store. +It is important to note, that records are not merged when records exist +with the label. +So a client has first to retrieve records, merge with existing records +and then store the result. + +To perform a lookup operation, the client uses the +@code{GNUNET_NAMESTORE_records_store} function. Here he has to pass the +namestore handle, the private key of the zone and the label. He also has +to provide a callback function which will be called with the result of +the lookup operation: +the zone for the records, the label, and the records including the +number of records included. + +A special operation is used to set the preferred nickname for a zone. +This nickname is stored with the zone and is automatically merged with +all labels and records stored in a zone. Here the client uses the +@code{GNUNET_NAMESTORE_set_nick} function and passes the private key of +the zone, the nickname as string plus a the callback with the result of +the operation. + +@node Iterating Zone Information +@subsubsection Iterating Zone Information + +@c %**end of header + +A client can iterate over all information in a zone or all zones managed +by NAMESTORE. +Here a client uses the @code{GNUNET_NAMESTORE_zone_iteration_start} +function and passes the namestore handle, the zone to iterate over and a +callback function to call with the result. +If the client wants to iterate over all the, he passes NULL for the zone. +A @code{GNUNET_NAMESTORE_ZoneIterator} handle is returned to be used to +continue iteration. + +NAMESTORE calls the callback for every result and expects the client to +call @code{GNUNET_NAMESTORE_zone_iterator_next} to continue to iterate or +@code{GNUNET_NAMESTORE_zone_iterator_stop} to interrupt the iteration. +When NAMESTORE reached the last item it will call the callback with a +NULL value to indicate. + +@node Monitoring Zone Information +@subsubsection Monitoring Zone Information + +@c %**end of header + +Clients can also monitor zones to be notified about changes. Here the +clients uses the @code{GNUNET_NAMESTORE_zone_monitor_start} function and +passes the private key of the zone and and a callback function to call +with updates for a zone. +The client can specify to obtain zone information first by iterating over +the zone and specify a synchronization callback to be called when the +client and the namestore are synced. + +On an update, NAMESTORE will call the callback with the private key of the +zone, the label and the records and their number. + +To stop monitoring, the client calls +@code{GNUNET_NAMESTORE_zone_monitor_stop} and passes the handle obtained +from the function to start the monitoring. + +@cindex PEERINFO Subsystem +@node PEERINFO Subsystem +@section PEERINFO Subsystem + +@c %**end of header + +The PEERINFO subsystem is used to store verified (validated) information +about known peers in a persistent way. It obtains these addresses for +example from TRANSPORT service which is in charge of address validation. +Validation means that the information in the HELLO message are checked by +connecting to the addresses and performing a cryptographic handshake to +authenticate the peer instance stating to be reachable with these +addresses. +Peerinfo does not validate the HELLO messages itself but only stores them +and gives them to interested clients. + +As future work, we think about moving from storing just HELLO messages to +providing a generic persistent per-peer information store. +More and more subsystems tend to need to store per-peer information in +persistent way. +To not duplicate this functionality we plan to provide a PEERSTORE +service providing this functionality. + +@menu +* PEERINFO - Features:: +* PEERINFO - Limitations:: +* DeveloperPeer Information:: +* Startup:: +* Managing Information:: +* Obtaining Information:: +* The PEERINFO Client-Service Protocol:: +* libgnunetpeerinfo:: +@end menu + +@node PEERINFO - Features +@subsection PEERINFO - Features + +@c %**end of header + +@itemize @bullet +@item Persistent storage +@item Client notification mechanism on update +@item Periodic clean up for expired information +@item Differentiation between public and friend-only HELLO +@end itemize + +@node PEERINFO - Limitations +@subsection PEERINFO - Limitations + + +@itemize @bullet +@item Does not perform HELLO validation +@end itemize + +@node DeveloperPeer Information +@subsection DeveloperPeer Information + +@c %**end of header + +The PEERINFO subsystem stores these information in the form of HELLO +messages you can think of as business cards. +These HELLO messages contain the public key of a peer and the addresses +a peer can be reached under. +The addresses include an expiration date describing how long they are +valid. This information is updated regularly by the TRANSPORT service by +revalidating the address. +If an address is expired and not renewed, it can be removed from the +HELLO message. + +Some peer do not want to have their HELLO messages distributed to other +peers, especially when GNUnet's friend-to-friend modus is enabled. +To prevent this undesired distribution. PEERINFO distinguishes between +@emph{public} and @emph{friend-only} HELLO messages. +Public HELLO messages can be freely distributed to other (possibly +unknown) peers (for example using the hostlist, gossiping, broadcasting), +whereas friend-only HELLO messages may not be distributed to other peers. +Friend-only HELLO messages have an additional flag @code{friend_only} set +internally. For public HELLO message this flag is not set. +PEERINFO does and cannot not check if a client is allowed to obtain a +specific HELLO type. + +The HELLO messages can be managed using the GNUnet HELLO library. +Other GNUnet systems can obtain these information from PEERINFO and use +it for their purposes. +Clients are for example the HOSTLIST component providing these +information to other peers in form of a hostlist or the TRANSPORT +subsystem using these information to maintain connections to other peers. + +@node Startup +@subsection Startup + +@c %**end of header + +During startup the PEERINFO services loads persistent HELLOs from disk. +First PEERINFO parses the directory configured in the HOSTS value of the +@code{PEERINFO} configuration section to store PEERINFO information. +For all files found in this directory valid HELLO messages are extracted. +In addition it loads HELLO messages shipped with the GNUnet distribution. +These HELLOs are used to simplify network bootstrapping by providing +valid peer information with the distribution. +The use of these HELLOs can be prevented by setting the +@code{USE_INCLUDED_HELLOS} in the @code{PEERINFO} configuration section to +@code{NO}. Files containing invalid information are removed. + +@node Managing Information +@subsection Managing Information + +@c %**end of header + +The PEERINFO services stores information about known PEERS and a single +HELLO message for every peer. +A peer does not need to have a HELLO if no information are available. +HELLO information from different sources, for example a HELLO obtained +from a remote HOSTLIST and a second HELLO stored on disk, are combined +and merged into one single HELLO message per peer which will be given to +clients. During this merge process the HELLO is immediately written to +disk to ensure persistence. + +PEERINFO in addition periodically scans the directory where information +are stored for empty HELLO messages with expired TRANSPORT addresses. +This periodic task scans all files in the directory and recreates the +HELLO messages it finds. +Expired TRANSPORT addresses are removed from the HELLO and if the +HELLO does not contain any valid addresses, it is discarded and removed +from the disk. + +@node Obtaining Information +@subsection Obtaining Information + +@c %**end of header + +When a client requests information from PEERINFO, PEERINFO performs a +lookup for the respective peer or all peers if desired and transmits this +information to the client. +The client can specify if friend-only HELLOs have to be included or not +and PEERINFO filters the respective HELLO messages before transmitting +information. + +To notify clients about changes to PEERINFO information, PEERINFO +maintains a list of clients interested in this notifications. +Such a notification occurs if a HELLO for a peer was updated (due to a +merge for example) or a new peer was added. + +@node The PEERINFO Client-Service Protocol +@subsection The PEERINFO Client-Service Protocol + +@c %**end of header + +To connect and disconnect to and from the PEERINFO Service PEERINFO +utilizes the util client/server infrastructure, so no special messages +types are used here. + +To add information for a peer, the plain HELLO message is transmitted to +the service without any wrapping. All pieces of information required are +stored within the HELLO message. +The PEERINFO service provides a message handler accepting and processing +these HELLO messages. + +When obtaining PEERINFO information using the iterate functionality +specific messages are used. To obtain information for all peers, a +@code{struct ListAllPeersMessage} with message type +@code{GNUNET_MESSAGE_TYPE_PEERINFO_GET_ALL} and a flag +include_friend_only to indicate if friend-only HELLO messages should be +included are transmitted. If information for a specific peer is required +a @code{struct ListAllPeersMessage} with +@code{GNUNET_MESSAGE_TYPE_PEERINFO_GET} containing the peer identity is +used. + +For both variants the PEERINFO service replies for each HELLO message it +wants to transmit with a @code{struct ListAllPeersMessage} with type +@code{GNUNET_MESSAGE_TYPE_PEERINFO_INFO} containing the plain HELLO. +The final message is @code{struct GNUNET_MessageHeader} with type +@code{GNUNET_MESSAGE_TYPE_PEERINFO_INFO}. If the client receives this +message, it can proceed with the next request if any is pending. + +@node libgnunetpeerinfo +@subsection libgnunetpeerinfo + +@c %**end of header + +The PEERINFO API consists mainly of three different functionalities: + +@itemize @bullet +@item maintaining a connection to the service +@item adding new information to the PEERINFO service +@item retrieving information from the PEERINFO service +@end itemize + +@menu +* Connecting to the PEERINFO Service:: +* Adding Information to the PEERINFO Service:: +* Obtaining Information from the PEERINFO Service:: +@end menu + +@node Connecting to the PEERINFO Service +@subsubsection Connecting to the PEERINFO Service + +@c %**end of header + +To connect to the PEERINFO service the function +@code{GNUNET_PEERINFO_connect} is used, taking a configuration handle as +an argument, and to disconnect from PEERINFO the function +@code{GNUNET_PEERINFO_disconnect}, taking the PEERINFO +handle returned from the connect function has to be called. + +@node Adding Information to the PEERINFO Service +@subsubsection Adding Information to the PEERINFO Service + +@c %**end of header + +@code{GNUNET_PEERINFO_add_peer} adds a new peer to the PEERINFO subsystem +storage. This function takes the PEERINFO handle as an argument, the HELLO +message to store and a continuation with a closure to be called with the +result of the operation. +The @code{GNUNET_PEERINFO_add_peer} returns a handle to this operation +allowing to cancel the operation with the respective cancel function +@code{GNUNET_PEERINFO_add_peer_cancel}. To retrieve information from +PEERINFO you can iterate over all information stored with PEERINFO or you +can tell PEERINFO to notify if new peer information are available. + +@node Obtaining Information from the PEERINFO Service +@subsubsection Obtaining Information from the PEERINFO Service + +@c %**end of header + +To iterate over information in PEERINFO you use +@code{GNUNET_PEERINFO_iterate}. +This function expects the PEERINFO handle, a flag if HELLO messages +intended for friend only mode should be included, a timeout how long the +operation should take and a callback with a callback closure to be called +for the results. +If you want to obtain information for a specific peer, you can specify +the peer identity, if this identity is NULL, information for all peers are +returned. The function returns a handle to allow to cancel the operation +using @code{GNUNET_PEERINFO_iterate_cancel}. + +To get notified when peer information changes, you can use +@code{GNUNET_PEERINFO_notify}. +This function expects a configuration handle and a flag if friend-only +HELLO messages should be included. The PEERINFO service will notify you +about every change and the callback function will be called to notify you +about changes. The function returns a handle to cancel notifications +with @code{GNUNET_PEERINFO_notify_cancel}. + +@cindex PEERSTORE Subsystem +@node PEERSTORE Subsystem +@section PEERSTORE Subsystem + +@c %**end of header + +GNUnet's PEERSTORE subsystem offers persistent per-peer storage for other +GNUnet subsystems. GNUnet subsystems can use PEERSTORE to persistently +store and retrieve arbitrary data. +Each data record stored with PEERSTORE contains the following fields: + +@itemize @bullet +@item subsystem: Name of the subsystem responsible for the record. +@item peerid: Identity of the peer this record is related to. +@item key: a key string identifying the record. +@item value: binary record value. +@item expiry: record expiry date. +@end itemize + +@menu +* Functionality:: +* Architecture:: +* libgnunetpeerstore:: +@end menu + +@node Functionality +@subsection Functionality + +@c %**end of header + +Subsystems can store any type of value under a (subsystem, peerid, key) +combination. A "replace" flag set during store operations forces the +PEERSTORE to replace any old values stored under the same +(subsystem, peerid, key) combination with the new value. +Additionally, an expiry date is set after which the record is *possibly* +deleted by PEERSTORE. + +Subsystems can iterate over all values stored under any of the following +combination of fields: + +@itemize @bullet +@item (subsystem) +@item (subsystem, peerid) +@item (subsystem, key) +@item (subsystem, peerid, key) +@end itemize + +Subsystems can also request to be notified about any new values stored +under a (subsystem, peerid, key) combination by sending a "watch" +request to PEERSTORE. + +@node Architecture +@subsection Architecture + +@c %**end of header + +PEERSTORE implements the following components: + +@itemize @bullet +@item PEERSTORE service: Handles store, iterate and watch operations. +@item PEERSTORE API: API to be used by other subsystems to communicate and +issue commands to the PEERSTORE service. +@item PEERSTORE plugins: Handles the persistent storage. At the moment, +only an "sqlite" plugin is implemented. +@end itemize + +@cindex libgnunetpeerstore +@node libgnunetpeerstore +@subsection libgnunetpeerstore + +@c %**end of header + +libgnunetpeerstore is the library containing the PEERSTORE API. Subsystems +wishing to communicate with the PEERSTORE service use this API to open a +connection to PEERSTORE. This is done by calling +@code{GNUNET_PEERSTORE_connect} which returns a handle to the newly +created connection. +This handle has to be used with any further calls to the API. + +To store a new record, the function @code{GNUNET_PEERSTORE_store} is to +be used which requires the record fields and a continuation function that +will be called by the API after the STORE request is sent to the +PEERSTORE service. +Note that calling the continuation function does not mean that the record +is successfully stored, only that the STORE request has been successfully +sent to the PEERSTORE service. +@code{GNUNET_PEERSTORE_store_cancel} can be called to cancel the STORE +request only before the continuation function has been called. + +To iterate over stored records, the function +@code{GNUNET_PEERSTORE_iterate} is +to be used. @emph{peerid} and @emph{key} can be set to NULL. An iterator +callback function will be called with each matching record found and a +NULL record at the end to signal the end of result set. +@code{GNUNET_PEERSTORE_iterate_cancel} can be used to cancel the ITERATE +request before the iterator callback is called with a NULL record. + +To be notified with new values stored under a (subsystem, peerid, key) +combination, the function @code{GNUNET_PEERSTORE_watch} is to be used. +This will register the watcher with the PEERSTORE service, any new +records matching the given combination will trigger the callback +function passed to @code{GNUNET_PEERSTORE_watch}. This continues until +@code{GNUNET_PEERSTORE_watch_cancel} is called or the connection to the +service is destroyed. + +After the connection is no longer needed, the function +@code{GNUNET_PEERSTORE_disconnect} can be called to disconnect from the +PEERSTORE service. +Any pending ITERATE or WATCH requests will be destroyed. +If the @code{sync_first} flag is set to @code{GNUNET_YES}, the API will +delay the disconnection until all pending STORE requests are sent to +the PEERSTORE service, otherwise, the pending STORE requests will be +destroyed as well. + +@cindex SET Subsystem +@node SET Subsystem +@section SET Subsystem + +@c %**end of header + +The SET service implements efficient set operations between two peers +over a mesh tunnel. +Currently, set union and set intersection are the only supported +operations. Elements of a set consist of an @emph{element type} and +arbitrary binary @emph{data}. +The size of an element's data is limited to around 62 KB. + +@menu +* Local Sets:: +* Set Modifications:: +* Set Operations:: +* Result Elements:: +* libgnunetset:: +* The SET Client-Service Protocol:: +* The SET Intersection Peer-to-Peer Protocol:: +* The SET Union Peer-to-Peer Protocol:: +@end menu + +@node Local Sets +@subsection Local Sets + +@c %**end of header + +Sets created by a local client can be modified and reused for multiple +operations. As each set operation requires potentially expensive special +auxilliary data to be computed for each element of a set, a set can only +participate in one type of set operation (i.e. union or intersection). +The type of a set is determined upon its creation. +If a the elements of a set are needed for an operation of a different +type, all of the set's element must be copied to a new set of appropriate +type. + +@node Set Modifications +@subsection Set Modifications + +@c %**end of header + +Even when set operations are active, one can add to and remove elements +from a set. +However, these changes will only be visible to operations that have been +created after the changes have taken place. That is, every set operation +only sees a snapshot of the set from the time the operation was started. +This mechanism is @emph{not} implemented by copying the whole set, but by +attaching @emph{generation information} to each element and operation. + +@node Set Operations +@subsection Set Operations + +@c %**end of header + +Set operations can be started in two ways: Either by accepting an +operation request from a remote peer, or by requesting a set operation +from a remote peer. +Set operations are uniquely identified by the involved @emph{peers}, an +@emph{application id} and the @emph{operation type}. + +The client is notified of incoming set operations by @emph{set listeners}. +A set listener listens for incoming operations of a specific operation +type and application id. +Once notified of an incoming set request, the client can accept the set +request (providing a local set for the operation) or reject it. + +@node Result Elements +@subsection Result Elements + +@c %**end of header + +The SET service has three @emph{result modes} that determine how an +operation's result set is delivered to the client: + +@itemize @bullet +@item @strong{Full Result Set.} All elements of set resulting from the set +operation are returned to the client. +@item @strong{Added Elements.} Only elements that result from the +operation and are not already in the local peer's set are returned. +Note that for some operations (like set intersection) this result mode +will never return any elements. +This can be useful if only the remove peer is actually interested in +the result of the set operation. +@item @strong{Removed Elements.} Only elements that are in the local +peer's initial set but not in the operation's result set are returned. +Note that for some operations (like set union) this result mode will +never return any elements. This can be useful if only the remove peer is +actually interested in the result of the set operation. +@end itemize + +@cindex libgnunetset +@node libgnunetset +@subsection libgnunetset + +@c %**end of header + +@menu +* Sets:: +* Listeners:: +* Operations:: +* Supplying a Set:: +* The Result Callback:: +@end menu + +@node Sets +@subsubsection Sets + +@c %**end of header + +New sets are created with @code{GNUNET_SET_create}. Both the local peer's +configuration (as each set has its own client connection) and the +operation type must be specified. +The set exists until either the client calls @code{GNUNET_SET_destroy} or +the client's connection to the service is disrupted. +In the latter case, the client is notified by the return value of +functions dealing with sets. This return value must always be checked. + +Elements are added and removed with @code{GNUNET_SET_add_element} and +@code{GNUNET_SET_remove_element}. + +@node Listeners +@subsubsection Listeners + +@c %**end of header + +Listeners are created with @code{GNUNET_SET_listen}. Each time time a +remote peer suggests a set operation with an application id and operation +type matching a listener, the listener's callback is invoked. +The client then must synchronously call either @code{GNUNET_SET_accept} +or @code{GNUNET_SET_reject}. Note that the operation will not be started +until the client calls @code{GNUNET_SET_commit} +(see Section "Supplying a Set"). + +@node Operations +@subsubsection Operations + +@c %**end of header + +Operations to be initiated by the local peer are created with +@code{GNUNET_SET_prepare}. Note that the operation will not be started +until the client calls @code{GNUNET_SET_commit} +(see Section "Supplying a Set"). + +@node Supplying a Set +@subsubsection Supplying a Set + +@c %**end of header + +To create symmetry between the two ways of starting a set operation +(accepting and nitiating it), the operation handles returned by +@code{GNUNET_SET_accept} and @code{GNUNET_SET_prepare} do not yet have a +set to operate on, thus they can not do any work yet. + +The client must call @code{GNUNET_SET_commit} to specify a set to use for +an operation. @code{GNUNET_SET_commit} may only be called once per set +operation. + +@node The Result Callback +@subsubsection The Result Callback + +@c %**end of header + +Clients must specify both a result mode and a result callback with +@code{GNUNET_SET_accept} and @code{GNUNET_SET_prepare}. The result +callback with a status indicating either that an element was received, or +the operation failed or succeeded. +The interpretation of the received element depends on the result mode. +The callback needs to know which result mode it is used in, as the +arguments do not indicate if an element is part of the full result set, +or if it is in the difference between the original set and the final set. + +@node The SET Client-Service Protocol +@subsection The SET Client-Service Protocol + +@c %**end of header + +@menu +* Creating Sets:: +* Listeners2:: +* Initiating Operations:: +* Modifying Sets:: +* Results and Operation Status:: +* Iterating Sets:: +@end menu + +@node Creating Sets +@subsubsection Creating Sets + +@c %**end of header + +For each set of a client, there exists a client connection to the service. +Sets are created by sending the @code{GNUNET_SERVICE_SET_CREATE} message +over a new client connection. Multiple operations for one set are +multiplexed over one client connection, using a request id supplied by +the client. + +@node Listeners2 +@subsubsection Listeners2 + +@c %**end of header + +Each listener also requires a seperate client connection. By sending the +@code{GNUNET_SERVICE_SET_LISTEN} message, the client notifies the service +of the application id and operation type it is interested in. A client +rejects an incoming request by sending @code{GNUNET_SERVICE_SET_REJECT} +on the listener's client connection. +In contrast, when accepting an incoming request, a +@code{GNUNET_SERVICE_SET_ACCEPT} message must be sent over the@ set that +is supplied for the set operation. + +@node Initiating Operations +@subsubsection Initiating Operations + +@c %**end of header + +Operations with remote peers are initiated by sending a +@code{GNUNET_SERVICE_SET_EVALUATE} message to the service. The@ client +connection that this message is sent by determines the set to use. + +@node Modifying Sets +@subsubsection Modifying Sets + +@c %**end of header + +Sets are modified with the @code{GNUNET_SERVICE_SET_ADD} and +@code{GNUNET_SERVICE_SET_REMOVE} messages. + + +@c %@menu +@c %* Results and Operation Status:: +@c %* Iterating Sets:: +@c %@end menu + +@node Results and Operation Status +@subsubsection Results and Operation Status +@c %**end of header + +The service notifies the client of result elements and success/failure of +a set operation with the @code{GNUNET_SERVICE_SET_RESULT} message. + +@node Iterating Sets +@subsubsection Iterating Sets + +@c %**end of header + +All elements of a set can be requested by sending +@code{GNUNET_SERVICE_SET_ITER_REQUEST}. The server responds with +@code{GNUNET_SERVICE_SET_ITER_ELEMENT} and eventually terminates the +iteration with @code{GNUNET_SERVICE_SET_ITER_DONE}. +After each received element, the client +must send @code{GNUNET_SERVICE_SET_ITER_ACK}. Note that only one set +iteration may be active for a set at any given time. + +@node The SET Intersection Peer-to-Peer Protocol +@subsection The SET Intersection Peer-to-Peer Protocol + +@c %**end of header + +The intersection protocol operates over CADET and starts with a +GNUNET_MESSAGE_TYPE_SET_P2P_OPERATION_REQUEST being sent by the peer +initiating the operation to the peer listening for inbound requests. +It includes the number of elements of the initiating peer, which is used +to decide which side will send a Bloom filter first. + +The listening peer checks if the operation type and application +identifier are acceptable for its current state. +If not, it responds with a GNUNET_MESSAGE_TYPE_SET_RESULT and a status of +GNUNET_SET_STATUS_FAILURE (and terminates the CADET channel). + +If the application accepts the request, the listener sends back a +@code{GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_ELEMENT_INFO} if it has +more elements in the set than the client. +Otherwise, it immediately starts with the Bloom filter exchange. +If the initiator receives a +@code{GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_ELEMENT_INFO} response, +it beings the Bloom filter exchange, unless the set size is indicated to +be zero, in which case the intersection is considered finished after +just the initial handshake. + + +@menu +* The Bloom filter exchange:: +* Salt:: +@end menu + +@node The Bloom filter exchange +@subsubsection The Bloom filter exchange + +@c %**end of header + +In this phase, each peer transmits a Bloom filter over the remaining +keys of the local set to the other peer using a +@code{GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_BF} message. This +message additionally includes the number of elements left in the sender's +set, as well as the XOR over all of the keys in that set. + +The number of bits 'k' set per element in the Bloom filter is calculated +based on the relative size of the two sets. +Furthermore, the size of the Bloom filter is calculated based on 'k' and +the number of elements in the set to maximize the amount of data filtered +per byte transmitted on the wire (while avoiding an excessively high +number of iterations). + +The receiver of the message removes all elements from its local set that +do not pass the Bloom filter test. +It then checks if the set size of the sender and the XOR over the keys +match what is left of his own set. If they do, he sends a +@code{GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_DONE} back to indicate +that the latest set is the final result. +Otherwise, the receiver starts another Bloom fitler exchange, except +this time as the sender. + +@node Salt +@subsubsection Salt + +@c %**end of header + +Bloomfilter operations are probablistic: With some non-zero probability +the test may incorrectly say an element is in the set, even though it is +not. + +To mitigate this problem, the intersection protocol iterates exchanging +Bloom filters using a different random 32-bit salt in each iteration (the +salt is also included in the message). +With different salts, set operations may fail for different elements. +Merging the results from the executions, the probability of failure drops +to zero. + +The iterations terminate once both peers have established that they have +sets of the same size, and where the XOR over all keys computes the same +512-bit value (leaving a failure probability of 2-511). + +@node The SET Union Peer-to-Peer Protocol +@subsection The SET Union Peer-to-Peer Protocol + +@c %**end of header + +The SET union protocol is based on Eppstein's efficient set reconciliation +without prior context. You should read this paper first if you want to +understand the protocol. + +The union protocol operates over CADET and starts with a +GNUNET_MESSAGE_TYPE_SET_P2P_OPERATION_REQUEST being sent by the peer +initiating the operation to the peer listening for inbound requests. +It includes the number of elements of the initiating peer, which is +currently not used. + +The listening peer checks if the operation type and application +identifier are acceptable for its current state. If not, it responds with +a @code{GNUNET_MESSAGE_TYPE_SET_RESULT} and a status of +@code{GNUNET_SET_STATUS_FAILURE} (and terminates the CADET channel). + +If the application accepts the request, it sends back a strata estimator +using a message of type GNUNET_MESSAGE_TYPE_SET_UNION_P2P_SE. The +initiator evaluates the strata estimator and initiates the exchange of +invertible Bloom filters, sending a GNUNET_MESSAGE_TYPE_SET_UNION_P2P_IBF. + +During the IBF exchange, if the receiver cannot invert the Bloom filter or +detects a cycle, it sends a larger IBF in response (up to a defined +maximum limit; if that limit is reached, the operation fails). +Elements decoded while processing the IBF are transmitted to the other +peer using GNUNET_MESSAGE_TYPE_SET_P2P_ELEMENTS, or requested from the +other peer using GNUNET_MESSAGE_TYPE_SET_P2P_ELEMENT_REQUESTS messages, +depending on the sign observed during decoding of the IBF. +Peers respond to a GNUNET_MESSAGE_TYPE_SET_P2P_ELEMENT_REQUESTS message +with the respective element in a GNUNET_MESSAGE_TYPE_SET_P2P_ELEMENTS +message. If the IBF fully decodes, the peer responds with a +GNUNET_MESSAGE_TYPE_SET_UNION_P2P_DONE message instead of another +GNUNET_MESSAGE_TYPE_SET_UNION_P2P_IBF. + +All Bloom filter operations use a salt to mingle keys before hasing them +into buckets, such that future iterations have a fresh chance of +succeeding if they failed due to collisions before. + +@cindex STATISTICS Subsystem +@node STATISTICS Subsystem +@section STATISTICS Subsystem + +@c %**end of header + +In GNUnet, the STATISTICS subsystem offers a central place for all +subsystems to publish unsigned 64-bit integer run-time statistics. +Keeping this information centrally means that there is a unified way for +the user to obtain data on all subsystems, and individual subsystems do +not have to always include a custom data export method for performance +metrics and other statistics. For example, the TRANSPORT system uses +STATISTICS to update information about the number of directly connected +peers and the bandwidth that has been consumed by the various plugins. +This information is valuable for diagnosing connectivity and performance +issues. + +Following the GNUnet service architecture, the STATISTICS subsystem is +divided into an API which is exposed through the header +@strong{gnunet_statistics_service.h} and the STATISTICS service +@strong{gnunet-service-statistics}. The @strong{gnunet-statistics} +command-line tool can be used to obtain (and change) information about +the values stored by the STATISTICS service. The STATISTICS service does +not communicate with other peers. + +Data is stored in the STATISTICS service in the form of tuples +@strong{(subsystem, name, value, persistence)}. The subsystem determines +to which other GNUnet's subsystem the data belongs. name is the name +through which value is associated. It uniquely identifies the record +from among other records belonging to the same subsystem. +In some parts of the code, the pair @strong{(subsystem, name)} is called +a @strong{statistic} as it identifies the values stored in the STATISTCS +service.The persistence flag determines if the record has to be preserved +across service restarts. A record is said to be persistent if this flag +is set for it; if not, the record is treated as a non-persistent record +and it is lost after service restart. Persistent records are written to +and read from the file @strong{statistics.data} before shutdown +and upon startup. The file is located in the HOME directory of the peer. + +An anomaly of the STATISTICS service is that it does not terminate +immediately upon receiving a shutdown signal if it has any clients +connected to it. It waits for all the clients that are not monitors to +close their connections before terminating itself. +This is to prevent the loss of data during peer shutdown --- delaying the +STATISTICS service shutdown helps other services to store important data +to STATISTICS during shutdown. + +@menu +* libgnunetstatistics:: +* The STATISTICS Client-Service Protocol:: +@end menu + +@cindex libgnunetstatistics +@node libgnunetstatistics +@subsection libgnunetstatistics + +@c %**end of header + +@strong{libgnunetstatistics} is the library containing the API for the +STATISTICS subsystem. Any process requiring to use STATISTICS should use +this API by to open a connection to the STATISTICS service. +This is done by calling the function @code{GNUNET_STATISTICS_create()}. +This function takes the subsystem's name which is trying to use STATISTICS +and a configuration. +All values written to STATISTICS with this connection will be placed in +the section corresponding to the given subsystem's name. +The connection to STATISTICS can be destroyed with the function +@code{GNUNET_STATISTICS_destroy()}. This function allows for the +connection to be destroyed immediately or upon transferring all +pending write requests to the service. + +Note: STATISTICS subsystem can be disabled by setting @code{DISABLE = YES} +under the @code{[STATISTICS]} section in the configuration. With such a +configuration all calls to @code{GNUNET_STATISTICS_create()} return +@code{NULL} as the STATISTICS subsystem is unavailable and no other +functions from the API can be used. + + +@menu +* Statistics retrieval:: +* Setting statistics and updating them:: +* Watches:: +@end menu + +@node Statistics retrieval +@subsubsection Statistics retrieval + +@c %**end of header + +Once a connection to the statistics service is obtained, information +about any other system which uses statistics can be retrieved with the +function GNUNET_STATISTICS_get(). +This function takes the connection handle, the name of the subsystem +whose information we are interested in (a @code{NULL} value will +retrieve information of all available subsystems using STATISTICS), the +name of the statistic we are interested in (a @code{NULL} value will +retrieve all available statistics), a continuation callback which is +called when all of requested information is retrieved, an iterator +callback which is called for each parameter in the retrieved information +and a closure for the aforementioned callbacks. The library then invokes +the iterator callback for each value matching the request. + +Call to @code{GNUNET_STATISTICS_get()} is asynchronous and can be +canceled with the function @code{GNUNET_STATISTICS_get_cancel()}. +This is helpful when retrieving statistics takes too long and especially +when we want to shutdown and cleanup everything. + +@node Setting statistics and updating them +@subsubsection Setting statistics and updating them + +@c %**end of header + +So far we have seen how to retrieve statistics, here we will learn how we +can set statistics and update them so that other subsystems can retrieve +them. + +A new statistic can be set using the function +@code{GNUNET_STATISTICS_set()}. +This function takes the name of the statistic and its value and a flag to +make the statistic persistent. +The value of the statistic should be of the type @code{uint64_t}. +The function does not take the name of the subsystem; it is determined +from the previous @code{GNUNET_STATISTICS_create()} invocation. If +the given statistic is already present, its value is overwritten. + +An existing statistics can be updated, i.e its value can be increased or +decreased by an amount with the function +@code{GNUNET_STATISTICS_update()}. +The parameters to this function are similar to +@code{GNUNET_STATISTICS_set()}, except that it takes the amount to be +changed as a type @code{int64_t} instead of the value. + +The library will combine multiple set or update operations into one +message if the client performs requests at a rate that is faster than the +available IPC with the STATISTICS service. Thus, the client does not have +to worry about sending requests too quickly. + +@node Watches +@subsubsection Watches + +@c %**end of header + +As interesting feature of STATISTICS lies in serving notifications +whenever a statistic of our interest is modified. +This is achieved by registering a watch through the function +@code{GNUNET_STATISTICS_watch()}. +The parameters of this function are similar to those of +@code{GNUNET_STATISTICS_get()}. +Changes to the respective statistic's value will then cause the given +iterator callback to be called. +Note: A watch can only be registered for a specific statistic. Hence +the subsystem name and the parameter name cannot be @code{NULL} in a +call to @code{GNUNET_STATISTICS_watch()}. + +A registered watch will keep notifying any value changes until +@code{GNUNET_STATISTICS_watch_cancel()} is called with the same +parameters that are used for registering the watch. + +@node The STATISTICS Client-Service Protocol +@subsection The STATISTICS Client-Service Protocol +@c %**end of header + + +@menu +* Statistics retrieval2:: +* Setting and updating statistics:: +* Watching for updates:: +@end menu + +@node Statistics retrieval2 +@subsubsection Statistics retrieval2 + +@c %**end of header + +To retrieve statistics, the client transmits a message of type +@code{GNUNET_MESSAGE_TYPE_STATISTICS_GET} containing the given subsystem +name and statistic parameter to the STATISTICS service. +The service responds with a message of type +@code{GNUNET_MESSAGE_TYPE_STATISTICS_VALUE} for each of the statistics +parameters that match the client request for the client. The end of +information retrieved is signaled by the service by sending a message of +type @code{GNUNET_MESSAGE_TYPE_STATISTICS_END}. + +@node Setting and updating statistics +@subsubsection Setting and updating statistics + +@c %**end of header + +The subsystem name, parameter name, its value and the persistence flag are +communicated to the service through the message +@code{GNUNET_MESSAGE_TYPE_STATISTICS_SET}. + +When the service receives a message of type +@code{GNUNET_MESSAGE_TYPE_STATISTICS_SET}, it retrieves the subsystem +name and checks for a statistic parameter with matching the name given in +the message. +If a statistic parameter is found, the value is overwritten by the new +value from the message; if not found then a new statistic parameter is +created with the given name and value. + +In addition to just setting an absolute value, it is possible to perform a +relative update by sending a message of type +@code{GNUNET_MESSAGE_TYPE_STATISTICS_SET} with an update flag +(@code{GNUNET_STATISTICS_SETFLAG_RELATIVE}) signifying that the value in +the message should be treated as an update value. + +@node Watching for updates +@subsubsection Watching for updates + +@c %**end of header + +The function registers the watch at the service by sending a message of +type @code{GNUNET_MESSAGE_TYPE_STATISTICS_WATCH}. The service then sends +notifications through messages of type +@code{GNUNET_MESSAGE_TYPE_STATISTICS_WATCH_VALUE} whenever the statistic +parameter's value is changed. + +@cindex DHT +@cindex Distributed Hash Table +@node Distributed Hash Table (DHT) +@section Distributed Hash Table (DHT) + +@c %**end of header + +GNUnet includes a generic distributed hash table that can be used by +developers building P2P applications in the framework. +This section documents high-level features and how developers are +expected to use the DHT. +We have a research paper detailing how the DHT works. +Also, Nate's thesis includes a detailed description and performance +analysis (in chapter 6). + +Key features of GNUnet's DHT include: + +@itemize @bullet +@item stores key-value pairs with values up to (approximately) 63k in size +@item works with many underlay network topologies (small-world, random +graph), underlay does not need to be a full mesh / clique +@item support for extended queries (more than just a simple 'key'), +filtering duplicate replies within the network (bloomfilter) and content +validation (for details, please read the subsection on the block library) +@item can (optionally) return paths taken by the PUT and GET operations +to the application +@item provides content replication to handle churn +@end itemize + +GNUnet's DHT is randomized and unreliable. Unreliable means that there is +no strict guarantee that a value stored in the DHT is always +found --- values are only found with high probability. +While this is somewhat true in all P2P DHTs, GNUnet developers should be +particularly wary of this fact (this will help you write secure, +fault-tolerant code). Thus, when writing any application using the DHT, +you should always consider the possibility that a value stored in the +DHT by you or some other peer might simply not be returned, or returned +with a significant delay. +Your application logic must be written to tolerate this (naturally, some +loss of performance or quality of service is expected in this case). + +@menu +* Block library and plugins:: +* libgnunetdht:: +* The DHT Client-Service Protocol:: +* The DHT Peer-to-Peer Protocol:: +@end menu + +@node Block library and plugins +@subsection Block library and plugins + +@c %**end of header + +@menu +* What is a Block?:: +* The API of libgnunetblock:: +* Queries:: +* Sample Code:: +* Conclusion2:: +@end menu + +@node What is a Block? +@subsubsection What is a Block? + +@c %**end of header + +Blocks are small (< 63k) pieces of data stored under a key (struct +GNUNET_HashCode). Blocks have a type (enum GNUNET_BlockType) which defines +their data format. Blocks are used in GNUnet as units of static data +exchanged between peers and stored (or cached) locally. +Uses of blocks include file-sharing (the files are broken up into blocks), +the VPN (DNS information is stored in blocks) and the DHT (all +information in the DHT and meta-information for the maintenance of the +DHT are both stored using blocks). +The block subsystem provides a few common functions that must be +available for any type of block. + +@cindex libgnunetblock API +@node The API of libgnunetblock +@subsubsection The API of libgnunetblock + +@c %**end of header + +The block library requires for each (family of) block type(s) a block +plugin (implementing @file{gnunet_block_plugin.h}) that provides basic +functions that are needed by the DHT (and possibly other subsystems) to +manage the block. +These block plugins are typically implemented within their respective +subsystems. +The main block library is then used to locate, load and query the +appropriate block plugin. +Which plugin is appropriate is determined by the block type (which is +just a 32-bit integer). Block plugins contain code that specifies which +block types are supported by a given plugin. The block library loads all +block plugins that are installed at the local peer and forwards the +application request to the respective plugin. + +The central functions of the block APIs (plugin and main library) are to +allow the mapping of blocks to their respective key (if possible) and the +ability to check that a block is well-formed and matches a given +request (again, if possible). +This way, GNUnet can avoid storing invalid blocks, storing blocks under +the wrong key and forwarding blocks in response to a query that they do +not answer. + +One key function of block plugins is that it allows GNUnet to detect +duplicate replies (via the Bloom filter). All plugins MUST support +detecting duplicate replies (by adding the current response to the +Bloom filter and rejecting it if it is encountered again). +If a plugin fails to do this, responses may loop in the network. + +@node Queries +@subsubsection Queries +@c %**end of header + +The query format for any block in GNUnet consists of four main components. +First, the type of the desired block must be specified. Second, the query +must contain a hash code. The hash code is used for lookups in hash +tables and databases and must not be unique for the block (however, if +possible a unique hash should be used as this would be best for +performance). +Third, an optional Bloom filter can be specified to exclude known results; +replies that hash to the bits set in the Bloom filter are considered +invalid. False-positives can be eliminated by sending the same query +again with a different Bloom filter mutator value, which parameterizes +the hash function that is used. +Finally, an optional application-specific "eXtended query" (xquery) can +be specified to further constrain the results. It is entirely up to +the type-specific plugin to determine whether or not a given block +matches a query (type, hash, Bloom filter, and xquery). +Naturally, not all xquery's are valid and some types of blocks may not +support Bloom filters either, so the plugin also needs to check if the +query is valid in the first place. + +Depending on the results from the plugin, the DHT will then discard the +(invalid) query, forward the query, discard the (invalid) reply, cache the +(valid) reply, and/or forward the (valid and non-duplicate) reply. + +@node Sample Code +@subsubsection Sample Code + +@c %**end of header + +The source code in @strong{plugin_block_test.c} is a good starting point +for new block plugins --- it does the minimal work by implementing a +plugin that performs no validation at all. +The respective @strong{Makefile.am} shows how to build and install a +block plugin. + +@node Conclusion2 +@subsubsection Conclusion2 + +@c %**end of header + +In conclusion, GNUnet subsystems that want to use the DHT need to define a +block format and write a plugin to match queries and replies. For testing, +the @code{GNUNET_BLOCK_TYPE_TEST} block type can be used; it accepts +any query as valid and any reply as matching any query. +This type is also used for the DHT command line tools. +However, it should NOT be used for normal applications due to the lack +of error checking that results from this primitive implementation. + +@cindex libgnunetdht +@node libgnunetdht +@subsection libgnunetdht + +@c %**end of header + +The DHT API itself is pretty simple and offers the usual GET and PUT +functions that work as expected. The specified block type refers to the +block library which allows the DHT to run application-specific logic for +data stored in the network. + + +@menu +* GET:: +* PUT:: +* MONITOR:: +* DHT Routing Options:: +@end menu + +@node GET +@subsubsection GET + +@c %**end of header + +When using GET, the main consideration for developers (other than the +block library) should be that after issuing a GET, the DHT will +continuously cause (small amounts of) network traffic until the operation +is explicitly canceled. +So GET does not simply send out a single network request once; instead, +the DHT will continue to search for data. This is needed to achieve good +success rates and also handles the case where the respective PUT +operation happens after the GET operation was started. +Developers should not cancel an existing GET operation and then +explicitly re-start it to trigger a new round of network requests; +this is simply inefficient, especially as the internal automated version +can be more efficient, for example by filtering results in the network +that have already been returned. + +If an application that performs a GET request has a set of replies that it +already knows and would like to filter, it can call@ +@code{GNUNET_DHT_get_filter_known_results} with an array of hashes over +the respective blocks to tell the DHT that these results are not +desired (any more). +This way, the DHT will filter the respective blocks using the block +library in the network, which may result in a significant reduction in +bandwidth consumption. + +@node PUT +@subsubsection PUT + +@c %**end of header + +In contrast to GET operations, developers @strong{must} manually re-run +PUT operations periodically (if they intend the content to continue to be +available). Content stored in the DHT expires or might be lost due to +churn. +Furthermore, GNUnet's DHT typically requires multiple rounds of PUT +operations before a key-value pair is consistently available to all +peers (the DHT randomizes paths and thus storage locations, and only +after multiple rounds of PUTs there will be a sufficient number of +replicas in large DHTs). An explicit PUT operation using the DHT API will +only cause network traffic once, so in order to ensure basic availability +and resistance to churn (and adversaries), PUTs must be repeated. +While the exact frequency depends on the application, a rule of thumb is +that there should be at least a dozen PUT operations within the content +lifetime. Content in the DHT typically expires after one day, so +DHT PUT operations should be repeated at least every 1-2 hours. + +@node MONITOR +@subsubsection MONITOR + +@c %**end of header + +The DHT API also allows applications to monitor messages crossing the +local DHT service. +The types of messages used by the DHT are GET, PUT and RESULT messages. +Using the monitoring API, applications can choose to monitor these +requests, possibly limiting themselves to requests for a particular block +type. + +The monitoring API is not only usefu only for diagnostics, it can also be +used to trigger application operations based on PUT operations. +For example, an application may use PUTs to distribute work requests to +other peers. +The workers would then monitor for PUTs that give them work, instead of +looking for work using GET operations. +This can be beneficial, especially if the workers have no good way to +guess the keys under which work would be stored. +Naturally, additional protocols might be needed to ensure that the desired +number of workers will process the distributed workload. + +@node DHT Routing Options +@subsubsection DHT Routing Options + +@c %**end of header + +There are two important options for GET and PUT requests: + +@table @asis +@item GNUNET_DHT_RO_DEMULITPLEX_EVERYWHERE This option means that all +peers should process the request, even if their peer ID is not closest to +the key. For a PUT request, this means that all peers that a request +traverses may make a copy of the data. +Similarly for a GET request, all peers will check their local database +for a result. Setting this option can thus significantly improve caching +and reduce bandwidth consumption --- at the expense of a larger DHT +database. If in doubt, we recommend that this option should be used. +@item GNUNET_DHT_RO_RECORD_ROUTE This option instructs the DHT to record +the path that a GET or a PUT request is taking through the overlay +network. The resulting paths are then returned to the application with +the respective result. This allows the receiver of a result to construct +a path to the originator of the data, which might then be used for +routing. Naturally, setting this option requires additional bandwidth +and disk space, so applications should only set this if the paths are +needed by the application logic. +@item GNUNET_DHT_RO_FIND_PEER This option is an internal option used by +the DHT's peer discovery mechanism and should not be used by applications. +@item GNUNET_DHT_RO_BART This option is currently not implemented. It may +in the future offer performance improvements for clique topologies. +@end table + +@node The DHT Client-Service Protocol +@subsection The DHT Client-Service Protocol + +@c %**end of header + +@menu +* PUTting data into the DHT:: +* GETting data from the DHT:: +* Monitoring the DHT:: +@end menu + +@node PUTting data into the DHT +@subsubsection PUTting data into the DHT + +@c %**end of header + +To store (PUT) data into the DHT, the client sends a +@code{struct GNUNET_DHT_ClientPutMessage} to the service. +This message specifies the block type, routing options, the desired +replication level, the expiration time, key, +value and a 64-bit unique ID for the operation. The service responds with +a @code{struct GNUNET_DHT_ClientPutConfirmationMessage} with the same +64-bit unique ID. Note that the service sends the confirmation as soon as +it has locally processed the PUT request. The PUT may still be +propagating through the network at this time. + +In the future, we may want to change this to provide (limited) feedback +to the client, for example if we detect that the PUT operation had no +effect because the same key-value pair was already stored in the DHT. +However, changing this would also require additional state and messages +in the P2P interaction. + +@node GETting data from the DHT +@subsubsection GETting data from the DHT + +@c %**end of header + +To retrieve (GET) data from the DHT, the client sends a +@code{struct GNUNET_DHT_ClientGetMessage} to the service. The message +specifies routing options, a replication level (for replicating the GET, +not the content), the desired block type, the key, the (optional) +extended query and unique 64-bit request ID. + +Additionally, the client may send any number of +@code{struct GNUNET_DHT_ClientGetResultSeenMessage}s to notify the +service about results that the client is already aware of. +These messages consist of the key, the unique 64-bit ID of the request, +and an arbitrary number of hash codes over the blocks that the client is +already aware of. As messages are restricted to 64k, a client that +already knows more than about a thousand blocks may need to send +several of these messages. Naturally, the client should transmit these +messages as quickly as possible after the original GET request such that +the DHT can filter those results in the network early on. Naturally, as +these messages are send after the original request, it is conceivalbe +that the DHT service may return blocks that match those already known +to the client anyway. + +In response to a GET request, the service will send @code{struct +GNUNET_DHT_ClientResultMessage}s to the client. These messages contain the +block type, expiration, key, unique ID of the request and of course the +value (a block). Depending on the options set for the respective +operations, the replies may also contain the path the GET and/or the PUT +took through the network. + +A client can stop receiving replies either by disconnecting or by sending +a @code{struct GNUNET_DHT_ClientGetStopMessage} which must contain the +key and the 64-bit unique ID of the original request. Using an +explicit "stop" message is more common as this allows a client to run +many concurrent GET operations over the same connection with the DHT +service --- and to stop them individually. + +@node Monitoring the DHT +@subsubsection Monitoring the DHT + +@c %**end of header + +To begin monitoring, the client sends a +@code{struct GNUNET_DHT_MonitorStartStop} message to the DHT service. +In this message, flags can be set to enable (or disable) monitoring of +GET, PUT and RESULT messages that pass through a peer. The message can +also restrict monitoring to a particular block type or a particular key. +Once monitoring is enabled, the DHT service will notify the client about +any matching event using @code{struct GNUNET_DHT_MonitorGetMessage}s for +GET events, @code{struct GNUNET_DHT_MonitorPutMessage} for PUT events +and @code{struct GNUNET_DHT_MonitorGetRespMessage} for RESULTs. Each of +these messages contains all of the information about the event. + +@node The DHT Peer-to-Peer Protocol +@subsection The DHT Peer-to-Peer Protocol +@c %**end of header + + +@menu +* Routing GETs or PUTs:: +* PUTting data into the DHT2:: +* GETting data from the DHT2:: +@end menu + +@node Routing GETs or PUTs +@subsubsection Routing GETs or PUTs + +@c %**end of header + +When routing GETs or PUTs, the DHT service selects a suitable subset of +neighbours for forwarding. The exact number of neighbours can be zero or +more and depends on the hop counter of the query (initially zero) in +relation to the (log of) the network size estimate, the desired +replication level and the peer's connectivity. +Depending on the hop counter and our network size estimate, the selection +of the peers maybe randomized or by proximity to the key. +Furthermore, requests include a set of peers that a request has already +traversed; those peers are also excluded from the selection. + +@node PUTting data into the DHT2 +@subsubsection PUTting data into the DHT2 + +@c %**end of header + +To PUT data into the DHT, the service sends a @code{struct PeerPutMessage} +of type @code{GNUNET_MESSAGE_TYPE_DHT_P2P_PUT} to the respective +neighbour. +In addition to the usual information about the content (type, routing +options, desired replication level for the content, expiration time, key +and value), the message contains a fixed-size Bloom filter with +information about which peers (may) have already seen this request. +This Bloom filter is used to ensure that DHT messages never loop back to +a peer that has already processed the request. +Additionally, the message includes the current hop counter and, depending +on the routing options, the message may include the full path that the +message has taken so far. +The Bloom filter should already contain the identity of the previous hop; +however, the path should not include the identity of the previous hop and +the receiver should append the identity of the sender to the path, not +its own identity (this is done to reduce bandwidth). + +@node GETting data from the DHT2 +@subsubsection GETting data from the DHT2 + +@c %**end of header + +A peer can search the DHT by sending @code{struct PeerGetMessage}s of type +@code{GNUNET_MESSAGE_TYPE_DHT_P2P_GET} to other peers. In addition to the +usual information about the request (type, routing options, desired +replication level for the request, the key and the extended query), a GET +request also again contains a hop counter, a Bloom filter over the peers +that have processed the request already and depending on the routing +options the full path traversed by the GET. +Finally, a GET request includes a variable-size second Bloom filter and a +so-called Bloom filter mutator value which together indicate which +replies the sender has already seen. During the lookup, each block that +matches they block type, key and extended query is additionally subjected +to a test against this Bloom filter. +The block plugin is expected to take the hash of the block and combine it +with the mutator value and check if the result is not yet in the Bloom +filter. The originator of the query will from time to time modify the +mutator to (eventually) allow false-positives filtered by the Bloom filter +to be returned. + +Peers that receive a GET request perform a local lookup (depending on +their proximity to the key and the query options) and forward the request +to other peers. +They then remember the request (including the Bloom filter for blocking +duplicate results) and when they obtain a matching, non-filtered response +a @code{struct PeerResultMessage} of type +@code{GNUNET_MESSAGE_TYPE_DHT_P2P_RESULT} is forwarded to the previous +hop. +Whenver a result is forwarded, the block plugin is used to update the +Bloom filter accordingly, to ensure that the same result is never +forwarded more than once. +The DHT service may also cache forwarded results locally if the +"CACHE_RESULTS" option is set to "YES" in the configuration. + +@cindex GNS +@cindex GNU Name System +@node GNU Name System (GNS) +@section GNU Name System (GNS) + +@c %**end of header + +The GNU Name System (GNS) is a decentralized database that enables users +to securely resolve names to values. +Names can be used to identify other users (for example, in social +networking), or network services (for example, VPN services running at a +peer in GNUnet, or purely IP-based services on the Internet). +Users interact with GNS by typing in a hostname that ends in ".gnu" +or ".zkey". + +Videos giving an overview of most of the GNS and the motivations behind +it is available here and here. +The remainder of this chapter targets developers that are familiar with +high level concepts of GNS as presented in these talks. +@c TODO: Add links to here and here and to these. + +GNS-aware applications should use the GNS resolver to obtain the +respective records that are stored under that name in GNS. +Each record consists of a type, value, expiration time and flags. + +The type specifies the format of the value. Types below 65536 correspond +to DNS record types, larger values are used for GNS-specific records. +Applications can define new GNS record types by reserving a number and +implementing a plugin (which mostly needs to convert the binary value +representation to a human-readable text format and vice-versa). +The expiration time specifies how long the record is to be valid. +The GNS API ensures that applications are only given non-expired values. +The flags are typically irrelevant for applications, as GNS uses them +internally to control visibility and validity of records. + +Records are stored along with a signature. +The signature is generated using the private key of the authoritative +zone. This allows any GNS resolver to verify the correctness of a +name-value mapping. + +Internally, GNS uses the NAMECACHE to cache information obtained from +other users, the NAMESTORE to store information specific to the local +users, and the DHT to exchange data between users. +A plugin API is used to enable applications to define new GNS +record types. + +@menu +* libgnunetgns:: +* libgnunetgnsrecord:: +* GNS plugins:: +* The GNS Client-Service Protocol:: +* Hijacking the DNS-Traffic using gnunet-service-dns:: +* Serving DNS lookups via GNS on W32:: +@end menu + +@node libgnunetgns +@subsection libgnunetgns + +@c %**end of header + +The GNS API itself is extremely simple. Clients first connec to the +GNS service using @code{GNUNET_GNS_connect}. +They can then perform lookups using @code{GNUNET_GNS_lookup} or cancel +pending lookups using @code{GNUNET_GNS_lookup_cancel}. +Once finished, clients disconnect using @code{GNUNET_GNS_disconnect}. + +@menu +* Looking up records:: +* Accessing the records:: +* Creating records:: +* Future work:: +@end menu + +@node Looking up records +@subsubsection Looking up records + +@c %**end of header + +@code{GNUNET_GNS_lookup} takes a number of arguments: + +@table @asis +@item handle This is simply the GNS connection handle from +@code{GNUNET_GNS_connect}. +@item name The client needs to specify the name to +be resolved. This can be any valid DNS or GNS hostname. +@item zone The client +needs to specify the public key of the GNS zone against which the +resolution should be done (the ".gnu" zone). +Note that a key must be provided, even if the name ends in ".zkey". +This should typically be the public key of the master-zone of the user. +@item type This is the desired GNS or DNS record type +to look for. While all records for the given name will be returned, this +can be important if the client wants to resolve record types that +themselves delegate resolution, such as CNAME, PKEY or GNS2DNS. +Resolving a record of any of these types will only work if the respective +record type is specified in the request, as the GNS resolver will +otherwise follow the delegation and return the records from the +respective destination, instead of the delegating record. +@item only_cached This argument should typically be set to +@code{GNUNET_NO}. Setting it to @code{GNUNET_YES} disables resolution via +the overlay network. +@item shorten_zone_key If GNS encounters new names during resolution, +their respective zones can automatically be learned and added to the +"shorten zone". If this is desired, clients must pass the private key of +the shorten zone. If NULL is passed, shortening is disabled. +@item proc This argument identifies +the function to call with the result. It is given proc_cls, the number of +records found (possilby zero) and the array of the records as arguments. +proc will only be called once. After proc,> has been called, the lookup +must no longer be cancelled. +@item proc_cls The closure for proc. +@end table + +@node Accessing the records +@subsubsection Accessing the records + +@c %**end of header + +The @code{libgnunetgnsrecord} library provides an API to manipulate the +GNS record array that is given to proc. In particular, it offers +functions such as converting record values to human-readable +strings (and back). However, most @code{libgnunetgnsrecord} functions are +not interesting to GNS client applications. + +For DNS records, the @code{libgnunetdnsparser} library provides +functions for parsing (and serializing) common types of DNS records. + +@node Creating records +@subsubsection Creating records + +@c %**end of header + +Creating GNS records is typically done by building the respective record +information (possibly with the help of @code{libgnunetgnsrecord} and +@code{libgnunetdnsparser}) and then using the @code{libgnunetnamestore} to +publish the information. The GNS API is not involved in this +operation. + +@node Future work +@subsubsection Future work + +@c %**end of header + +In the future, we want to expand @code{libgnunetgns} to allow +applications to observe shortening operations performed during GNS +resolution, for example so that users can receive visual feedback when +this happens. + +@node libgnunetgnsrecord +@subsection libgnunetgnsrecord + +@c %**end of header + +The @code{libgnunetgnsrecord} library is used to manipulate GNS +records (in plaintext or in their encrypted format). +Applications mostly interact with @code{libgnunetgnsrecord} by using the +functions to convert GNS record values to strings or vice-versa, or to +lookup a GNS record type number by name (or vice-versa). +The library also provides various other functions that are mostly +used internally within GNS, such as converting keys to names, checking for +expiration, encrypting GNS records to GNS blocks, verifying GNS block +signatures and decrypting GNS records from GNS blocks. + +We will now discuss the four commonly used functions of the API.@ +@code{libgnunetgnsrecord} does not perform these operations itself, +but instead uses plugins to perform the operation. +GNUnet includes plugins to support common DNS record types as well as +standard GNS record types. + +@menu +* Value handling:: +* Type handling:: +@end menu + +@node Value handling +@subsubsection Value handling + +@c %**end of header + +@code{GNUNET_GNSRECORD_value_to_string} can be used to convert +the (binary) representation of a GNS record value to a human readable, +0-terminated UTF-8 string. +NULL is returned if the specified record type is not supported by any +available plugin. + +@code{GNUNET_GNSRECORD_string_to_value} can be used to try to convert a +human readable string to the respective (binary) representation of +a GNS record value. + +@node Type handling +@subsubsection Type handling + +@c %**end of header + +@code{GNUNET_GNSRECORD_typename_to_number} can be used to obtain the +numeric value associated with a given typename. For example, given the +typename "A" (for DNS A reocrds), the function will return the number 1. +A list of common DNS record types is +@uref{http://en.wikipedia.org/wiki/List_of_DNS_record_types, here}. +Note that not all DNS record types are supported by GNUnet GNSRECORD +plugins at this time. + +@code{GNUNET_GNSRECORD_number_to_typename} can be used to obtain the +typename associated with a given numeric value. +For example, given the type number 1, the function will return the +typename "A". + +@node GNS plugins +@subsection GNS plugins + +@c %**end of header + +Adding a new GNS record type typically involves writing (or extending) a +GNSRECORD plugin. The plugin needs to implement the +@code{gnunet_gnsrecord_plugin.h} API which provides basic functions that +are needed by GNSRECORD to convert typenames and values of the respective +record type to strings (and back). +These gnsrecord plugins are typically implemented within their respective +subsystems. +Examples for such plugins can be found in the GNSRECORD, GNS and +CONVERSATION subsystems. + +The @code{libgnunetgnsrecord} library is then used to locate, load and +query the appropriate gnsrecord plugin. +Which plugin is appropriate is determined by the record type (which is +just a 32-bit integer). The @code{libgnunetgnsrecord} library loads all +block plugins that are installed at the local peer and forwards the +application request to the plugins. If the record type is not +supported by the plugin, it should simply return an error code. + +The central functions of the block APIs (plugin and main library) are the +same four functions for converting between values and strings, and +typenames and numbers documented in the previous subsection. + +@node The GNS Client-Service Protocol +@subsection The GNS Client-Service Protocol +@c %**end of header + +The GNS client-service protocol consists of two simple messages, the +@code{LOOKUP} message and the @code{LOOKUP_RESULT}. Each @code{LOOKUP} +message contains a unique 32-bit identifier, which will be included in the +corresponding response. Thus, clients can send many lookup requests in +parallel and receive responses out-of-order. +A @code{LOOKUP} request also includes the public key of the GNS zone, +the desired record type and fields specifying whether shortening is +enabled or networking is disabled. Finally, the @code{LOOKUP} message +includes the name to be resolved. + +The response includes the number of records and the records themselves +in the format created by @code{GNUNET_GNSRECORD_records_serialize}. +They can thus be deserialized using +@code{GNUNET_GNSRECORD_records_deserialize}. + +@node Hijacking the DNS-Traffic using gnunet-service-dns +@subsection Hijacking the DNS-Traffic using gnunet-service-dns + +@c %**end of header + +This section documents how the gnunet-service-dns (and the +gnunet-helper-dns) intercepts DNS queries from the local system. +This is merely one method for how we can obtain GNS queries. +It is also possible to change @code{resolv.conf} to point to a machine +running @code{gnunet-dns2gns} or to modify libc's name system switch +(NSS) configuration to include a GNS resolution plugin. +The method described in this chaper is more of a last-ditch catch-all +approach. + +@code{gnunet-service-dns} enables intercepting DNS traffic using policy +based routing. +We MARK every outgoing DNS-packet if it was not sent by our application. +Using a second routing table in the Linux kernel these marked packets are +then routed through our virtual network interface and can thus be +captured unchanged. + +Our application then reads the query and decides how to handle it: A +query to an address ending in ".gnu" or ".zkey" is hijacked by +@code{gnunet-service-gns} and resolved internally using GNS. +In the future, a reverse query for an address of the configured virtual +network could be answered with records kept about previous forward +queries. +Queries that are not hijacked by some application using the DNS service +will be sent to the original recipient. +The answer to the query will always be sent back through the virtual +interface with the original nameserver as source address. + + +@menu +* Network Setup Details:: +@end menu + +@node Network Setup Details +@subsubsection Network Setup Details + +@c %**end of header + +The DNS interceptor adds the following rules to the Linux kernel: +@example +iptables -t mangle -I OUTPUT 1 -p udp --sport $LOCALPORT --dport 53 \ +-j ACCEPT iptables -t mangle -I OUTPUT 2 -p udp --dport 53 -j MARK \ +--set-mark 3 ip rule add fwmark 3 table2 ip route add default via \ +$VIRTUALDNS table2 +@end example + +@c FIXME: Rewrite to reflect display which is no longer content by line +@c FIXME: due to the < 74 characters limit. +Line 1 makes sure that all packets coming from a port our application +opened beforehand (@code{$LOCALPORT}) will be routed normally. +Line 2 marks every other packet to a DNS-Server with mark 3 (chosen +arbitrarily). The third line adds a routing policy based on this mark +3 via the routing table. + +@node Serving DNS lookups via GNS on W32 +@subsection Serving DNS lookups via GNS on W32 + +@c %**end of header + +This section documents how the libw32nsp (and +gnunet-gns-helper-service-w32) do DNS resolutions of DNS queries on the +local system. This only applies to GNUnet running on W32. + +W32 has a concept of "Namespaces" and "Namespace providers". +These are used to present various name systems to applications in a +generic way. +Namespaces include DNS, mDNS, NLA and others. For each namespace any +number of providers could be registered, and they are queried in an order +of priority (which is adjustable). + +Applications can resolve names by using WSALookupService*() family of +functions. + +However, these are WSA-only facilities. Common BSD socket functions for +namespace resolutions are gethostbyname and getaddrinfo (among others). +These functions are implemented internally (by default - by mswsock, +which also implements the default DNS provider) as wrappers around +WSALookupService*() functions (see "Sample Code for a Service Provider" +on MSDN). + +On W32 GNUnet builds a libw32nsp - a namespace provider, which can then be +installed into the system by using w32nsp-install (and uninstalled by +w32nsp-uninstall), as described in "Installation Handbook". + +libw32nsp is very simple and has almost no dependencies. As a response to +NSPLookupServiceBegin(), it only checks that the provider GUID passed to +it by the caller matches GNUnet DNS Provider GUID, checks that name being +resolved ends in ".gnu" or ".zkey", then connects to +gnunet-gns-helper-service-w32 at 127.0.0.1:5353 (hardcoded) and sends the +name resolution request there, returning the connected socket to the +caller. + +When the caller invokes NSPLookupServiceNext(), libw32nsp reads a +completely formed reply from that socket, unmarshalls it, then gives +it back to the caller. + +At the moment gnunet-gns-helper-service-w32 is implemented to ever give +only one reply, and subsequent calls to NSPLookupServiceNext() will fail +with WSA_NODATA (first call to NSPLookupServiceNext() might also fail if +GNS failed to find the name, or there was an error connecting to it). + +gnunet-gns-helper-service-w32 does most of the processing: + +@itemize @bullet +@item Maintains a connection to GNS. +@item Reads GNS config and loads appropriate keys. +@item Checks service GUID and decides on the type of record to look up, +refusing to make a lookup outright when unsupported service GUID is +passed. +@item Launches the lookup +@end itemize + +When lookup result arrives, gnunet-gns-helper-service-w32 forms a complete +reply (including filling a WSAQUERYSETW structure and, possibly, a binary +blob with a hostent structure for gethostbyname() client), marshalls it, +and sends it back to libw32nsp. If no records were found, it sends an +empty header. + +This works for most normal applications that use gethostbyname() or +getaddrinfo() to resolve names, but fails to do anything with +applications that use alternative means of resolving names (such as +sending queries to a DNS server directly by themselves). +This includes some of well known utilities, like "ping" and "nslookup". + +@cindex GNS Namecache +@node GNS Namecache +@section GNS Namecache + +@c %**end of header + +The NAMECACHE subsystem is responsible for caching (encrypted) resolution +results of the GNU Name System (GNS). GNS makes zone information available +to other users via the DHT. However, as accessing the DHT for every +lookup is expensive (and as the DHT's local cache is lost whenever the +peer is restarted), GNS uses the NAMECACHE as a more persistent cache for +DHT lookups. +Thus, instead of always looking up every name in the DHT, GNS first +checks if the result is already available locally in the NAMECACHE. +Only if there is no result in the NAMECACHE, GNS queries the DHT. +The NAMECACHE stores data in the same (encrypted) format as the DHT. +It thus makes no sense to iterate over all items in the +NAMECACHE --- the NAMECACHE does not have a way to provide the keys +required to decrypt the entries. + +Blocks in the NAMECACHE share the same expiration mechanism as blocks in +the DHT --- the block expires wheneever any of the records in +the (encrypted) block expires. +The expiration time of the block is the only information stored in +plaintext. The NAMECACHE service internally performs all of the required +work to expire blocks, clients do not have to worry about this. +Also, given that NAMECACHE stores only GNS blocks that local users +requested, there is no configuration option to limit the size of the +NAMECACHE. It is assumed to be always small enough (a few MB) to fit on +the drive. + +The NAMECACHE supports the use of different database backends via a +plugin API. + +@menu +* libgnunetnamecache:: +* The NAMECACHE Client-Service Protocol:: +* The NAMECACHE Plugin API:: +@end menu + +@node libgnunetnamecache +@subsection libgnunetnamecache + +@c %**end of header + +The NAMECACHE API consists of five simple functions. First, there is +@code{GNUNET_NAMECACHE_connect} to connect to the NAMECACHE service. +This returns the handle required for all other operations on the +NAMECACHE. Using @code{GNUNET_NAMECACHE_block_cache} clients can insert a +block into the cache. +@code{GNUNET_NAMECACHE_lookup_block} can be used to lookup blocks that +were stored in the NAMECACHE. Both operations can be cancelled using +@code{GNUNET_NAMECACHE_cancel}. Note that cancelling a +@code{GNUNET_NAMECACHE_block_cache} operation can result in the block +being stored in the NAMECACHE --- or not. Cancellation primarily ensures +that the continuation function with the result of the operation will no +longer be invoked. +Finally, @code{GNUNET_NAMECACHE_disconnect} closes the connection to the +NAMECACHE. + +The maximum size of a block that can be stored in the NAMECACHE is +@code{GNUNET_NAMECACHE_MAX_VALUE_SIZE}, which is defined to be 63 kB. + +@node The NAMECACHE Client-Service Protocol +@subsection The NAMECACHE Client-Service Protocol + +@c %**end of header + +All messages in the NAMECACHE IPC protocol start with the +@code{struct GNUNET_NAMECACHE_Header} which adds a request +ID (32-bit integer) to the standard message header. +The request ID is used to match requests with the +respective responses from the NAMECACHE, as they are allowed to happen +out-of-order. + + +@menu +* Lookup:: +* Store:: +@end menu + +@node Lookup +@subsubsection Lookup + +@c %**end of header + +The @code{struct LookupBlockMessage} is used to lookup a block stored in +the cache. +It contains the query hash. The NAMECACHE always responds with a +@code{struct LookupBlockResponseMessage}. If the NAMECACHE has no +response, it sets the expiration time in the response to zero. +Otherwise, the response is expected to contain the expiration time, the +ECDSA signature, the derived key and the (variable-size) encrypted data +of the block. + +@node Store +@subsubsection Store + +@c %**end of header + +The @code{struct BlockCacheMessage} is used to cache a block in the +NAMECACHE. +It has the same structure as the @code{struct LookupBlockResponseMessage}. +The service responds with a @code{struct BlockCacheResponseMessage} which +contains the result of the operation (success or failure). +In the future, we might want to make it possible to provide an error +message as well. + +@node The NAMECACHE Plugin API +@subsection The NAMECACHE Plugin API +@c %**end of header + +The NAMECACHE plugin API consists of two functions, @code{cache_block} to +store a block in the database, and @code{lookup_block} to lookup a block +in the database. + + +@menu +* Lookup2:: +* Store2:: +@end menu + +@node Lookup2 +@subsubsection Lookup2 + +@c %**end of header + +The @code{lookup_block} function is expected to return at most one block +to the iterator, and return @code{GNUNET_NO} if there were no non-expired +results. +If there are multiple non-expired results in the cache, the lookup is +supposed to return the result with the largest expiration time. + +@node Store2 +@subsubsection Store2 + +@c %**end of header + +The @code{cache_block} function is expected to try to store the block in +the database, and return @code{GNUNET_SYSERR} if this was not possible +for any reason. +Furthermore, @code{cache_block} is expected to implicitly perform cache +maintenance and purge blocks from the cache that have expired. Note that +@code{cache_block} might encounter the case where the database already has +another block stored under the same key. In this case, the plugin must +ensure that the block with the larger expiration time is preserved. +Obviously, this can done either by simply adding new blocks and selecting +for the most recent expiration time during lookup, or by checking which +block is more recent during the store operation. + +@cindex REVOCATION Subsystem +@node REVOCATION Subsystem +@section REVOCATION Subsystem +@c %**end of header + +The REVOCATION subsystem is responsible for key revocation of Egos. +If a user learns that theis private key has been compromised or has lost +it, they can use the REVOCATION system to inform all of the other users +that their private key is no longer valid. +The subsystem thus includes ways to query for the validity of keys and to +propagate revocation messages. + +@menu +* Dissemination:: +* Revocation Message Design Requirements:: +* libgnunetrevocation:: +* The REVOCATION Client-Service Protocol:: +* The REVOCATION Peer-to-Peer Protocol:: +@end menu + +@node Dissemination +@subsection Dissemination + +@c %**end of header + +When a revocation is performed, the revocation is first of all +disseminated by flooding the overlay network. +The goal is to reach every peer, so that when a peer needs to check if a +key has been revoked, this will be purely a local operation where the +peer looks at his local revocation list. Flooding the network is also the +most robust form of key revocation --- an adversary would have to control +a separator of the overlay graph to restrict the propagation of the +revocation message. Flooding is also very easy to implement --- peers that +receive a revocation message for a key that they have never seen before +simply pass the message to all of their neighbours. + +Flooding can only distribute the revocation message to peers that are +online. +In order to notify peers that join the network later, the revocation +service performs efficient set reconciliation over the sets of known +revocation messages whenever two peers (that both support REVOCATION +dissemination) connect. +The SET service is used to perform this operation efficiently. + +@node Revocation Message Design Requirements +@subsection Revocation Message Design Requirements + +@c %**end of header + +However, flooding is also quite costly, creating O(|E|) messages on a +network with |E| edges. +Thus, revocation messages are required to contain a proof-of-work, the +result of an expensive computation (which, however, is cheap to verify). +Only peers that have expended the CPU time necessary to provide +this proof will be able to flood the network with the revocation message. +This ensures that an attacker cannot simply flood the network with +millions of revocation messages. The proof-of-work required by GNUnet is +set to take days on a typical PC to compute; if the ability to quickly +revoke a key is needed, users have the option to pre-compute revocation +messages to store off-line and use instantly after their key has expired. + +Revocation messages must also be signed by the private key that is being +revoked. Thus, they can only be created while the private key is in the +possession of the respective user. This is another reason to create a +revocation message ahead of time and store it in a secure location. + +@node libgnunetrevocation +@subsection libgnunetrevocation + +@c %**end of header + +The REVOCATION API consists of two parts, to query and to issue +revocations. + + +@menu +* Querying for revoked keys:: +* Preparing revocations:: +* Issuing revocations:: +@end menu + +@node Querying for revoked keys +@subsubsection Querying for revoked keys + +@c %**end of header + +@code{GNUNET_REVOCATION_query} is used to check if a given ECDSA public +key has been revoked. +The given callback will be invoked with the result of the check. +The query can be cancelled using @code{GNUNET_REVOCATION_query_cancel} on +the return value. + +@node Preparing revocations +@subsubsection Preparing revocations + +@c %**end of header + +It is often desirable to create a revocation record ahead-of-time and +store it in an off-line location to be used later in an emergency. +This is particularly true for GNUnet revocations, where performing the +revocation operation itself is computationally expensive and thus is +likely to take some time. +Thus, if users want the ability to perform revocations quickly in an +emergency, they must pre-compute the revocation message. +The revocation API enables this with two functions that are used to +compute the revocation message, but not trigger the actual revocation +operation. + +@code{GNUNET_REVOCATION_check_pow} should be used to calculate the +proof-of-work required in the revocation message. This function takes the +public key, the required number of bits for the proof of work (which in +GNUnet is a network-wide constant) and finally a proof-of-work number as +arguments. +The function then checks if the given proof-of-work number is a valid +proof of work for the given public key. Clients preparing a revocation +are expected to call this function repeatedly (typically with a +monotonically increasing sequence of numbers of the proof-of-work number) +until a given number satisfies the check. +That number should then be saved for later use in the revocation +operation. + +@code{GNUNET_REVOCATION_sign_revocation} is used to generate the +signature that is required in a revocation message. +It takes the private key that (possibly in the future) is to be revoked +and returns the signature. +The signature can again be saved to disk for later use, which will then +allow performing a revocation even without access to the private key. + +@node Issuing revocations +@subsubsection Issuing revocations + + +Given a ECDSA public key, the signature from @code{GNUNET_REVOCATION_sign} +and the proof-of-work, +@code{GNUNET_REVOCATION_revoke} can be used to perform the +actual revocation. The given callback is called upon completion of the +operation. @code{GNUNET_REVOCATION_revoke_cancel} can be used to stop the +library from calling the continuation; however, in that case it is +undefined whether or not the revocation operation will be executed. + +@node The REVOCATION Client-Service Protocol +@subsection The REVOCATION Client-Service Protocol + + +The REVOCATION protocol consists of four simple messages. + +A @code{QueryMessage} containing a public ECDSA key is used to check if a +particular key has been revoked. The service responds with a +@code{QueryResponseMessage} which simply contains a bit that says if the +given public key is still valid, or if it has been revoked. + +The second possible interaction is for a client to revoke a key by +passing a @code{RevokeMessage} to the service. The @code{RevokeMessage} +contains the ECDSA public key to be revoked, a signature by the +corresponding private key and the proof-of-work, The service responds +with a @code{RevocationResponseMessage} which can be used to indicate +that the @code{RevokeMessage} was invalid (i.e. proof of work incorrect), +or otherwise indicates that the revocation has been processed +successfully. + +@node The REVOCATION Peer-to-Peer Protocol +@subsection The REVOCATION Peer-to-Peer Protocol + +@c %**end of header + +Revocation uses two disjoint ways to spread revocation information among +peers. +First of all, P2P gossip exchanged via CORE-level neighbours is used to +quickly spread revocations to all connected peers. +Second, whenever two peers (that both support revocations) connect, +the SET service is used to compute the union of the respective revocation +sets. + +In both cases, the exchanged messages are @code{RevokeMessage}s which +contain the public key that is being revoked, a matching ECDSA signature, +and a proof-of-work. +Whenever a peer learns about a new revocation this way, it first +validates the signature and the proof-of-work, then stores it to disk +(typically to a file $GNUNET_DATA_HOME/revocation.dat) and finally +spreads the information to all directly connected neighbours. + +For computing the union using the SET service, the peer with the smaller +hashed peer identity will connect (as a "client" in the two-party set +protocol) to the other peer after one second (to reduce traffic spikes +on connect) and initiate the computation of the set union. +All revocation services use a common hash to identify the SET operation +over revocation sets. + +The current implementation accepts revocation set union operations from +all peers at any time; however, well-behaved peers should only initiate +this operation once after establishing a connection to a peer with a +larger hashed peer identity. + +@cindex FS +@cindex FS Subsystem +@node File-sharing (FS) Subsystem +@section File-sharing (FS) Subsystem + +@c %**end of header + +This chapter describes the details of how the file-sharing service works. +As with all services, it is split into an API (libgnunetfs), the service +process (gnunet-service-fs) and user interface(s). +The file-sharing service uses the datastore service to store blocks and +the DHT (and indirectly datacache) for lookups for non-anonymous +file-sharing. +Furthermore, the file-sharing service uses the block library (and the +block fs plugin) for validation of DHT operations. + +In contrast to many other services, libgnunetfs is rather complex since +the client library includes a large number of high-level abstractions; +this is necessary since the Fs service itself largely only operates on +the block level. +The FS library is responsible for providing a file-based abstraction to +applications, including directories, meta data, keyword search, +verification, and so on. + +The method used by GNUnet to break large files into blocks and to use +keyword search is called the +"Encoding for Censorship Resistant Sharing" (ECRS). +ECRS is largely implemented in the fs library; block validation is also +reflected in the block FS plugin and the FS service. +ECRS on-demand encoding is implemented in the FS service. + +NOTE: The documentation in this chapter is quite incomplete. + +@menu +* Encoding for Censorship-Resistant Sharing (ECRS):: +* File-sharing persistence directory structure:: +@end menu + +@cindex ECRS +@cindex Encoding for Censorship-Resistant Sharing +@node Encoding for Censorship-Resistant Sharing (ECRS) +@subsection Encoding for Censorship-Resistant Sharing (ECRS) + +@c %**end of header + +When GNUnet shares files, it uses a content encoding that is called ECRS, +the Encoding for Censorship-Resistant Sharing. +Most of ECRS is described in the (so far unpublished) research paper +attached to this page. ECRS obsoletes the previous ESED and ESED II +encodings which were used in GNUnet before version 0.7.0. +The rest of this page assumes that the reader is familiar with the +attached paper. What follows is a description of some minor extensions +that GNUnet makes over what is described in the paper. +The reason why these extensions are not in the paper is that we felt +that they were obvious or trivial extensions to the original scheme and +thus did not warrant space in the research report. + +@menu +* Namespace Advertisements:: +* KSBlocks:: +@end menu + +@node Namespace Advertisements +@subsubsection Namespace Advertisements + +@c %**end of header +@c %**FIXME: all zeroses -> ? + +An @code{SBlock} with identifier all zeros is a signed +advertisement for a namespace. This special @code{SBlock} contains +metadata describing the content of the namespace. +Instead of the name of the identifier for a potential update, it contains +the identifier for the root of the namespace. +The URI should always be empty. The @code{SBlock} is signed with the +content provder's RSA private key (just like any other SBlock). Peers +can search for @code{SBlock}s in order to find out more about a namespace. + +@node KSBlocks +@subsubsection KSBlocks + +@c %**end of header + +GNUnet implements @code{KSBlocks} which are @code{KBlocks} that, instead +of encrypting a CHK and metadata, encrypt an @code{SBlock} instead. +In other words, @code{KSBlocks} enable GNUnet to find @code{SBlocks} +using the global keyword search. +Usually the encrypted @code{SBlock} is a namespace advertisement. +The rationale behind @code{KSBlock}s and @code{SBlock}s is to enable +peers to discover namespaces via keyword searches, and, to associate +useful information with namespaces. When GNUnet finds @code{KSBlocks} +during a normal keyword search, it adds the information to an internal +list of discovered namespaces. Users looking for interesting namespaces +can then inspect this list, reducing the need for out-of-band discovery +of namespaces. +Naturally, namespaces (or more specifically, namespace advertisements) can +also be referenced from directories, but @code{KSBlock}s should make it +easier to advertise namespaces for the owner of the pseudonym since they +eliminate the need to first create a directory. + +Collections are also advertised using @code{KSBlock}s. + +@table @asis +@item Attachment Size +@item ecrs.pdf 270.68 KB +@item https://gnunet.org/sites/default/files/ecrs.pdf +@end table + +@node File-sharing persistence directory structure +@subsection File-sharing persistence directory structure + +@c %**end of header + +This section documents how the file-sharing library implements +persistence of file-sharing operations and specifically the resulting +directory structure. +This code is only active if the @code{GNUNET_FS_FLAGS_PERSISTENCE} flag +was set when calling @code{GNUNET_FS_start}. +In this case, the file-sharing library will try hard to ensure that all +major operations (searching, downloading, publishing, unindexing) are +persistent, that is, can live longer than the process itself. +More specifically, an operation is supposed to live until it is +explicitly stopped. + +If @code{GNUNET_FS_stop} is called before an operation has been stopped, a +@code{SUSPEND} event is generated and then when the process calls +@code{GNUNET_FS_start} next time, a @code{RESUME} event is generated. +Additionally, even if an application crashes (segfault, SIGKILL, system +crash) and hence @code{GNUNET_FS_stop} is never called and no +@code{SUSPEND} events are generated, operations are still resumed (with +@code{RESUME} events). +This is implemented by constantly writing the current state of the +file-sharing operations to disk. +Specifically, the current state is always written to disk whenever +anything significant changes (the exception are block-wise progress in +publishing and unindexing, since those operations would be slowed down +significantly and can be resumed cheaply even without detailed +accounting). +Note that if the process crashes (or is killed) during a serialization +operation, FS does not guarantee that this specific operation is +recoverable (no strict transactional semantics, again for performance +reasons). However, all other unrelated operations should resume nicely. + +Since we need to serialize the state continuously and want to recover as +much as possible even after crashing during a serialization operation, +we do not use one large file for serialization. +Instead, several directories are used for the various operations. +When @code{GNUNET_FS_start} executes, the master directories are scanned +for files describing operations to resume. +Sometimes, these operations can refer to related operations in child +directories which may also be resumed at this point. +Note that corrupted files are cleaned up automatically. +However, dangling files in child directories (those that are not +referenced by files from the master directories) are not automatically +removed. + +Persistence data is kept in a directory that begins with the "STATE_DIR" +prefix from the configuration file +(by default, "$SERVICEHOME/persistence/") followed by the name of the +client as given to @code{GNUNET_FS_start} (for example, "gnunet-gtk") +followed by the actual name of the master or child directory. + +The names for the master directories follow the names of the operations: + +@itemize @bullet +@item "search" +@item "download" +@item "publish" +@item "unindex" +@end itemize + +Each of the master directories contains names (chosen at random) for each +active top-level (master) operation. +Note that a download that is associated with a search result is not a +top-level operation. + +In contrast to the master directories, the child directories are only +consulted when another operation refers to them. +For each search, a subdirectory (named after the master search +synchronization file) contains the search results. +Search results can have an associated download, which is then stored in +the general "download-child" directory. +Downloads can be recursive, in which case children are stored in +subdirectories mirroring the structure of the recursive download +(either starting in the master "download" directory or in the +"download-child" directory depending on how the download was initiated). +For publishing operations, the "publish-file" directory contains +information about the individual files and directories that are part of +the publication. +However, this directory structure is flat and does not mirror the +structure of the publishing operation. +Note that unindex operations cannot have associated child operations. + +@cindex REGEX subsystem +@node REGEX Subsystem +@section REGEX Subsystem + +@c %**end of header + +Using the REGEX subsystem, you can discover peers that offer a particular +service using regular expressions. +The peers that offer a service specify it using a regular expressions. +Peers that want to patronize a service search using a string. +The REGEX subsystem will then use the DHT to return a set of matching +offerers to the patrons. + +For the technical details, we have Max's defense talk and Max's Master's +thesis. + +@c An additional publication is under preparation and available to +@c team members (in Git). +@c FIXME: Where is the file? Point to it. Assuming that it's szengel2012ms + +@menu +* How to run the regex profiler:: +@end menu + +@node How to run the regex profiler +@subsection How to run the regex profiler + +@c %**end of header + +The gnunet-regex-profiler can be used to profile the usage of mesh/regex +for a given set of regular expressions and strings. +Mesh/regex allows you to announce your peer ID under a certain regex and +search for peers matching a particular regex using a string. +See @uref{https://gnunet.org/szengel2012ms, szengel2012ms} for a full +introduction. + +First of all, the regex profiler uses GNUnet testbed, thus all the +implications for testbed also apply to the regex profiler +(for example you need password-less ssh login to the machines listed in +your hosts file). + +@strong{Configuration} + +Moreover, an appropriate configuration file is needed. +Generally you can refer to the +@file{contrib/regex_profiler_infiniband.conf} file in the sourcecode +of GNUnet for an example configuration. +In the following paragraph the important details are highlighted. + +Announcing of the regular expressions is done by the +gnunet-daemon-regexprofiler, therefore you have to make sure it is +started, by adding it to the AUTOSTART set of ARM: + +@example +[regexprofiler] +AUTOSTART = YES +@end example + +@noindent +Furthermore you have to specify the location of the binary: + +@example +[regexprofiler] +# Location of the gnunet-daemon-regexprofiler binary. +BINARY = /home/szengel/gnunet/src/mesh/.libs/gnunet-daemon-regexprofiler +# Regex prefix that will be applied to all regular expressions and +# search string. +REGEX_PREFIX = "GNVPN-0001-PAD" +@end example + +@noindent +When running the profiler with a large scale deployment, you probably +want to reduce the workload of each peer. +Use the following options to do this. + +@example +[dht] +# Force network size estimation +FORCE_NSE = 1 + +[dhtcache] +DATABASE = heap +# Disable RC-file for Bloom filter? (for benchmarking with limited IO +# availability) +DISABLE_BF_RC = YES +# Disable Bloom filter entirely +DISABLE_BF = YES + +[nse] +# Minimize proof-of-work CPU consumption by NSE +WORKBITS = 1 +@end example + +@noindent +@strong{Options} + +To finally run the profiler some options and the input data need to be +specified on the command line. + +@example +gnunet-regex-profiler -c config-file -d log-file -n num-links \ +-p path-compression-length -s search-delay -t matching-timeout \ +-a num-search-strings hosts-file policy-dir search-strings-file +@end example + +@noindent +Where... + +@itemize @bullet +@item ... @code{config-file} means the configuration file created earlier. +@item ... @code{log-file} is the file where to write statistics output. +@item ... @code{num-links} indicates the number of random links between +started peers. +@item ... @code{path-compression-length} is the maximum path compression +length in the DFA. +@item ... @code{search-delay} time to wait between peers finished linking +and starting to match strings. +@item ... @code{matching-timeout} timeout after which to cancel the +searching. +@item ... @code{num-search-strings} number of strings in the +search-strings-file. +@item ... the @code{hosts-file} should contain a list of hosts for the +testbed, one per line in the following format: + +@itemize @bullet +@item @code{user@@host_ip:port} +@end itemize +@item ... the @code{policy-dir} is a folder containing text files +containing one or more regular expressions. A peer is started for each +file in that folder and the regular expressions in the corresponding file +are announced by this peer. +@item ... the @code{search-strings-file} is a text file containing search +strings, one in each line. +@end itemize + +@noindent +You can create regular expressions and search strings for every AS in the +Internet using the attached scripts. You need one of the +@uref{http://data.caida.org/datasets/routing/routeviews-prefix2as/, CAIDA routeviews prefix2as} +data files for this. Run + +@example +create_regex.py <filename> <output path> +@end example + +@noindent +to create the regular expressions and + +@example +create_strings.py <input path> <outfile> +@end example + +@noindent +to create a search strings file from the previously created +regular expressions. diff --git a/doc/documentation/chapters/installation.texi b/doc/documentation/chapters/installation.texi new file mode 100644 index 0000000000..7be1e9833c --- /dev/null +++ b/doc/documentation/chapters/installation.texi @@ -0,0 +1,4086 @@ +@node GNUnet Installation Handbook +@chapter GNUnet Installation Handbook + +This handbook describes how to install (build, setup, compile) and +setup (configure, start) GNUnet @value{VERSION}. After following these +instructions you should be able to install and then start user-interfaces +to interact with the network. + +Note: This manual is far from complete, and we welcome informed +contributions, be it in the form of new chapters or insightful comments. + +@menu +* Dependencies:: +* Pre-installation notes:: +* Generic installation instructions:: +* Build instructions for Ubuntu 12.04 using Git:: +* Build instructions for software builds from source:: +* Build Instructions for Microsoft Windows Platforms:: +* Build instructions for Debian 7.5:: +* Installing GNUnet from Git on Ubuntu 14.4:: +* Build instructions for Debian 8:: +* Outdated build instructions for previous revisions:: +@c * Portable GNUnet:: +* The graphical configuration interface:: +* How to start and stop a GNUnet peer:: +@end menu + +@node Dependencies +@section Dependencies +@c %**end of header + +This section lists the various known dependencies for +GNUnet @value{EDITION}. +Suggestions for missing dependencies or wrong version numbers are welcome. + +@menu +* External dependencies:: +* Optional dependencies:: +* Internal dependencies:: +@end menu + +@node External dependencies +@subsection External dependencies +@c %**end of header + +These packages must be installed before a typical GNUnet installation +can be performed: + +@itemize @bullet +@item autoconf +@item automake +@item pkg-config +@item libltdl +@item gstreamer +@item gst-plugins-base +@item perl +@item python (only 2.7 supported)@footnote{tests and gnunet-qr} +@item jansson +@item nss +@item glib +@item gmp +@item bluez +@item miniupnpc +@item gettext +@item which +@item texinfo @geq{} 5.2 +@item GNU libmicrohttpd @geq{} 0.9.30 @footnote{We recommend to build it +with a GnuTLS version that was configured with libunbound} +@item GNU libextractor @geq{} 1.0 +@item GNU libtool @geq{} 2.2 +@item GNU libunistring @geq{} 0.9.1.1 +@item GNU libidn @geq{} 1.0.0 +@item @uref{https://gnupg.org/software/libgcrypt/, GNU libgcrypt} @geq{} +@uref{https://gnupg.org/ftp/gcrypt/libgcrypt/, 1.6.0} +@item @uref{https://gnutls.org/, GnuTLS} @geq{} 3.2.7 +@footnote{We recommend to compile with libunbound for DANE support; +GnuTLS also requires GNU nettle 2.7 (update: GnuTLS 3.2.7 appears NOT +to work against GNU nettle > 2.7, due to some API updatings done by +nettle. Thus it should be compiled against nettle 2.7 +and, in case you get some error on the reference to `rpl_strerror' being +undefined, follow the instructions on +@uref{http://lists.gnupg.org/pipermail/gnutls-devel/2013-November/006588.html, this} +post (and the link inside it)).} +@item @uref{https://gnunet.org/gnurl, gnURL} libgnurl @geq{} 7.34.0 +@footnote{must be compiled after @code{GnuTLS}} +@item libglpk @geq{} 4.45 +@item @uref{http://www.openssl.org/, OpenSSL} @geq{} 1.0 +@item TeX Live @geq{} 2012, optional (for gnunet-bcd) +@item Texinfo @geq{} 5.2 (for documentation) +@item libsqlite @geq{} 3.8.0 @footnote{(note that the code will +compile and often work with lower version numbers, but you may get subtle +bugs with respect to quota management in certain rare cases); +alternatively, MySQL or Postgres can also be installed, but those +databases will require more complex configurations (not +recommended for first-time users)} +@item zlib +@end itemize + +@node Optional dependencies +@subsection Optional dependencies + +These applications must be installed for various experimental or otherwise +optional features such as @command{gnunet-conversation}, +and @command{gnunet-gtk} (most of these features are only build if you +configure GNUnet with @command{--enable-experimental}): + +@itemize @bullet +@item libpulse @geq{} 2.0, +optional (for @command{gnunet-conversation}) +@item libopus @geq{} 1.0.1, +optional (for @command{gnunet-conversation}) +@item libogg @geq{} 1.3.0, +optional (for @command{gnunet-conversation}) +@item libnss contained @command{certool} binary, +optional for convenient installation of +the GNS proxy. +@item python-zbar @geq{} 0.10, +optional (for @command{gnunet-qr}) +@item Gtk+ @geq{} 3.0, +optional (for @command{gnunet-gtk}) +@item libgladeui (must match Gtk+ version), +optional (for @command{gnunet-gtk}) +@item libqrencode @geq{} 3.0, +optional (for @command{gnunet-namestore-gtk}) +@end itemize + +@node Internal dependencies +@subsection Internal dependencies + +This section tries to give an overview of what processes a typical GNUnet +peer running a particular application would consist of. All of the +processes listed here should be automatically started by +@command{gnunet-arm -s}. +The list is given as a rough first guide to users for failure diagnostics. +Ideally, end-users should never have to worry about these internal +dependencies. + +In terms of internal dependencies, a minimum file-sharing system consists +of the following GNUnet processes (in order of dependency): + +@itemize @bullet +@item gnunet-service-arm +@item gnunet-service-resolver (required by all) +@item gnunet-service-statistics (required by all) +@item gnunet-service-peerinfo +@item gnunet-service-transport (requires peerinfo) +@item gnunet-service-core (requires transport) +@item gnunet-daemon-hostlist (requires core) +@item gnunet-daemon-topology (requires hostlist, peerinfo) +@item gnunet-service-datastore +@item gnunet-service-dht (requires core) +@item gnunet-service-identity +@item gnunet-service-fs (requires identity, mesh, dht, datastore, core) +@end itemize + +@noindent +A minimum VPN system consists of the following GNUnet processes (in +order of dependency): + +@itemize @bullet +@item gnunet-service-arm +@item gnunet-service-resolver (required by all) +@item gnunet-service-statistics (required by all) +@item gnunet-service-peerinfo +@item gnunet-service-transport (requires peerinfo) +@item gnunet-service-core (requires transport) +@item gnunet-daemon-hostlist (requires core) +@item gnunet-service-dht (requires core) +@item gnunet-service-mesh (requires dht, core) +@item gnunet-service-dns (requires dht) +@item gnunet-service-regex (requires dht) +@item gnunet-service-vpn (requires regex, dns, mesh, dht) +@end itemize + +@noindent +A minimum GNS system consists of the following GNUnet processes (in +order of dependency): + +@itemize @bullet +@item gnunet-service-arm +@item gnunet-service-resolver (required by all) +@item gnunet-service-statistics (required by all) +@item gnunet-service-peerinfo +@item gnunet-service-transport (requires peerinfo) +@item gnunet-service-core (requires transport) +@item gnunet-daemon-hostlist (requires core) +@item gnunet-service-dht (requires core) +@item gnunet-service-mesh (requires dht, core) +@item gnunet-service-dns (requires dht) +@item gnunet-service-regex (requires dht) +@item gnunet-service-vpn (requires regex, dns, mesh, dht) +@item gnunet-service-identity +@item gnunet-service-namestore (requires identity) +@item gnunet-service-gns (requires vpn, dns, dht, namestore, identity) +@end itemize + +@node Pre-installation notes +@section Pre-installation notes + +Please note that in the code instructions for the installation, +@emph{#} indicates commands run as privileged root user and +@emph{$} shows commands run as unprivileged ("normal") system user. + + +@node Generic installation instructions +@section Generic installation instructions + +First, in addition to the GNUnet sources you might require downloading the +latest version of various dependencies, depending on how recent the +software versions in your distribution of GNU/Linux are. +Most distributions do not include sufficiently recent versions of these +dependencies. +Thus, a typically installation on a "modern" GNU/Linux distribution +requires you to install the following dependencies (ideally in this +order): + +@itemize @bullet +@item libgpgerror and libgcrypt +@item libnettle and libunbound (possibly from distribution), GnuTLS +@item libgnurl (read the README) +@item GNU libmicrohttpd +@item GNU libextractor +@end itemize + +Make sure to first install the various mandatory and optional +dependencies including development headers from your distribution. + +Other dependencies that you should strongly consider to install is a +database (MySQL, sqlite or Postgres). +The following instructions will assume that you installed at least sqlite. +For most distributions you should be able to find pre-build packages for +the database. Again, make sure to install the client libraries @b{and} the +respective development headers (if they are packaged separately) as well. + +You can find specific, detailed instructions for installing of the +dependencies (and possibly the rest of the GNUnet installation) in the +platform-specific descriptions, which can be found in the Index. +Please consult them now. +If your distribution is not listed, please study +@ref{Build instructions for Debian 8}, the build instructions for +Debian stable, carefully as you try to install the dependencies for your +own distribution. +Contributing additional instructions for further platforms is always +appreciated. +Please take in mind that operating system development tends to move at +a rather fast speed. Due to this you should be aware that some of +the instructions could be outdated by the time you are reading this. +If you find a mistake, please tell us about it (or even better: send +a patch to the documentation to fix it!). + +Before proceeding further, please double-check the dependency list. +Note that in addition to satisfying the dependencies, you might have to +make sure that development headers for the various libraries are also +installed. +There maybe files for other distributions, or you might be able to find +equivalent packages for your distribution. + +While it is possible to build and install GNUnet without having root +access, we will assume that you have full control over your system in +these instructions. +First, you should create a system user @emph{gnunet} and an additional +group @emph{gnunetdns}. On the GNU/Linux distributions Debian and Ubuntu, +type: + +@example +# adduser --system --home /var/lib/gnunet --group \ +--disabled-password gnunet +# addgroup --system gnunetdns +@end example + +@noindent +On other Unixes and GNU systems, this should have the same effect: + +@example +# useradd --system --groups gnunet --home-dir /var/lib/gnunet +# addgroup --system gnunetdns +@end example + +Now compile and install GNUnet using: + +@example +$ tar xvf gnunet-@value{VERSION}.tar.gz +$ cd gnunet-@value{VERSION} +$ ./configure --with-sudo=sudo --with-nssdir=/lib +$ make +$ sudo make install +@end example + +If you want to be able to enable DEBUG-level log messages, add +@code{--enable-logging=verbose} to the end of the +@command{./configure} command. +@code{DEBUG}-level log messages are in English only and +should only be useful for developers (or for filing +really detailed bug reports). + +Finally, you probably want to compile @command{gnunet-gtk}, which +includes @command{gnunet-setup} (a graphical tool for +GNUnet configuration) and @command{gnunet-fs-gtk} (a graphical tool for +GNUnet file-sharing): + +@example +$ tar xvf gnunet-gtk-@value{VERSION}.tar.gz +$ cd gnunet-gtk-@value{VERSION} +$ ./configure --with-gnunet=/usr/local/ +$ make +$ sudo make install +$ cd .. +# just to be safe run this: +$ sudo ldconfig +@end example + +@noindent +Next, edit the file @file{/etc/gnunet.conf} to contain the following: + +@example +[arm] +SYSTEM_ONLY = YES +USER_ONLY = NO +@end example + +@noindent +You may need to update your @code{ld.so} cache to include +files installed in @file{/usr/local/lib}: + +@example +# ldconfig +@end example + +@noindent +Then, switch from user @code{root} to user @code{gnunet} to start +the peer: + +@example +# su -s /bin/sh - gnunet +$ gnunet-arm -c /etc/gnunet.conf -s +@end example + +You may also want to add the last line in the gnunet user's @file{crontab} +prefixed with @code{@@reboot} so that it is executed whenever the system +is booted: + +@example +@@reboot /usr/local/bin/gnunet-arm -c /etc/gnunet.conf -s +@end example + +@noindent +This will only start the system-wide GNUnet services. +Type exit to get back your root shell. +Now, you need to configure the per-user part. For each +$USER that should get access to GNUnet on the system, run: + +@example +# adduser $USER gnunet +@end example + +@noindent +to allow them to access the system-wide GNUnet services. Then, each +user should create a configuration file @file{~/.config/gnunet.conf} +with the lines: + +@example +[arm] +SYSTEM_ONLY = NO +USER_ONLY = YES +DEFAULTSERVICES = gns +@end example + +@noindent +and start the per-user services using + +@example +$ gnunet-arm -c ~/.config/gnunet.conf -s +@end example + +@noindent +Again, adding a @code{crontab} entry to autostart the peer is advised: + +@example +@@reboot /usr/local/bin/gnunet-arm -c $HOME/.config/gnunet.conf -s +@end example + +@noindent +Note that some GNUnet services (such as SOCKS5 proxies) may need a +system-wide TCP port for each user. +For those services, systems with more than one user may require each user +to specify a different port number in their personal configuration file. + +Finally, the user should perform the basic initial setup for the GNU Name +System (GNS). This is done by running two commands: + +@example +$ gnunet-gns-import.sh +$ gnunet-gns-proxy-setup-ca +@end example + +@noindent +The first generates the default zones, wheras the second setups the GNS +Certificate Authority with the user's browser. Now, to activate GNS in the +normal DNS resolution process, you need to edit your +@file{/etc/nsswitch.conf} where you should find a line like this: + +@example +hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4 +@end example + +@noindent +The exact details may differ a bit, which is fine. Add the text +@emph{"gns [NOTFOUND=return]"} after @emph{"files"}. +Keep in mind that we included a backslash ("\") here just for +markup reasons. You should write the text below on @b{one line} +and @b{without} the "\": + +@example +hosts: files gns [NOTFOUND=return] mdns4_minimal \ +[NOTFOUND=return] dns mdns4 +@end example + +@c FIXME: Document new behavior. +You might want to make sure that @file{/lib/libnss_gns.so.2} exists on +your system, it should have been created during the installation. + +@node Build instructions for Ubuntu 12.04 using Git +@section Build instructions for Ubuntu 12.04 using Git + +@menu +* Install the required build tools:: +* Install libgcrypt 1.6 and libgpg-error:: +* Install gnutls with DANE support:: +* Install libgnurl:: +* Install libmicrohttpd from Git:: +* Install libextractor from Git:: +* Install GNUnet dependencies:: +* Build GNUnet:: +* Install the GNUnet-gtk user interface from Git:: +@end menu + +@node Install the required build tools +@subsection Install the required build tools + +First, make sure Git is installed on your system: + +@example +$ sudo apt-get install git +@end example + +Install the essential buildtools: + +@example +$ sudo apt-get install automake autopoint autoconf libtool +@end example + +@node Install libgcrypt 1.6 and libgpg-error +@subsection Install libgcrypt 1.6 and libgpg-error + +@ref{generic source installation - libgpg-error} + +@node Install gnutls with DANE support +@subsection Install gnutls with DANE support + +@itemize @bullet +@item @ref{generic source installation - nettle} +@item @ref{generic source installation - ldns} +@item @ref{generic source installation - libunbound/unbound} +@item @ref{generic source installation - gnutls} +@item @ref{generic source installation - libgcrypt} +@end itemize + +@node Install libgnurl +@subsection Install libgnurl + +Follow the @ref{generic source installation - libgnurl}. + +@node Install libmicrohttpd from Git +@subsection Install libmicrohttpd from Git + +@example +$ git clone https://gnunet.org/git/libmicrohttpd +$ cd libmicrohttpd/ +$ ./bootstrap +$ ./configure +$ sudo make install ; cd .. +@end example + +@node Install libextractor from Git +@subsection Install libextractor from Git + +Install libextractor dependencies: + +@example +$ sudo apt-get install zlib1g-dev libgsf-1-dev libmpeg2-4-dev \ + libpoppler-dev libvorbis-dev libexiv2-dev libjpeg-dev \ + libtiff-dev libgif-dev libvorbis-dev libflac-dev libsmf-dev \ + g++ +@end example + +Build libextractor: + +@example +$ git clone https://gnunet.org/git/libextractor +$ cd libextractor +$ ./bootstrap +$ ./configure +$ sudo make install ; cd .. +@end example + +@node Install GNUnet dependencies +@subsection Install GNUnet dependencies + +@example +$ sudo apt-get install libidn11-dev libunistring-dev libglpk-dev \ + libpulse-dev libbluetooth-dev libsqlite-dev +@end example + +Install libopus: + +@example +$ wget http://downloads.xiph.org/releases/opus/opus-1.1.tar.gz +$ tar xf opus-1.1.tar.gz +$ cd opus-1.1/ +$ ./configure +$ sudo make install ; cd .. +@end example + +Choose one or more database backends: + +SQLite3: +@example +$ sudo apt-get install libsqlite3-dev +@end example +MySQL: +@example +$ sudo apt-get install libmysqlclient-dev +@end example +PostgreSQL: +@example +$ sudo apt-get install libpq-dev postgresql +@end example + + + +@node Build GNUnet +@subsection Build GNUnet + + + +@menu +* Configuring the installation path:: +* Configuring the system:: +* Installing components requiring sudo permission:: +* Build:: +@end menu + +@node Configuring the installation path +@subsubsection Configuring the installation path + +You can specify the location of the GNUnet installation by setting the +prefix when calling the configure script with @code{--prefix=DIRECTORY} + +@example +$ export PATH=$PATH:DIRECTORY/bin +@end example + +@node Configuring the system +@subsubsection Configuring the system + +Please make sure NOW that you have created a user and group 'gnunet' +and additionally a group 'gnunetdns': + +@example +$ sudo addgroup gnunet +$ sudo addgroup gnunetdns +$ sudo adduser gnunet +@end example + +Each GNUnet user should be added to the 'gnunet' group (may +require fresh login to come into effect): + +@example +$ sudo useradd -G gnunet +@end example + +@node Installing components requiring sudo permission +@subsubsection Installing components requiring sudo permission + +Some components, like the nss plugin required for GNS, may require root +permissions. To allow these few components to be installed use: + +@example +$ ./configure --with-sudo +@end example + +@node Build +@subsubsection Build + +@example +$ git clone https://gnunet.org/git/gnunet/ +$ cd gnunet/ +$ ./bootstrap +@end example + +Use the required configure call including the optional installation prefix +@code{PREFIX} or the sudo permissions: + +@example +$ ./configure [ --with-sudo | --with-prefix=PREFIX ] +@end example + +@example +$ make; sudo make install +@end example + +After installing it, you need to create an empty configuration file: + +@example +mkdir ~/.gnunet; touch ~/.gnunet/gnunet.conf +@end example + +And finally you can start GNUnet with: + +@example +$ gnunet-arm -s +@end example + +@node Install the GNUnet-gtk user interface from Git +@subsection Install the GNUnet-gtk user interface from Git + + +Install depencies: + +@example +$ sudo apt-get install libgtk-3-dev libunique-3.0-dev libgladeui-dev \ + libqrencode-dev +@end example + +Build GNUnet (with an optional prefix) and execute: + +@example +$ git clone https://gnunet.org/git/gnunet-gtk/ +$ cd gnunet-gtk/ +$ ./bootstrap +$ ./configure [--prefix=PREFIX] --with-gnunet=DIRECTORY +$ make; sudo make install +@end example + +@node Build instructions for software builds from source +@section Build instructions for software builds from source + +This section describes software builds in case your operating +system lacks binary substitutes / binary builds for some dependencies +of GNUnet. +It is assumed that you have installed common build dependencies +and that these instructions are treated as generic without any +debugging help. +It is furthermore assumed that you use the release tarballs of +the software, installation from the respective version control +sources might differ in ways that are only minimal different +(for example a dependency on autotools etc). + +@menu +* generic source installation - nettle:: +* generic source installation - ldns:: +* generic source installation - libunbound/unbound:: +* generic source installation - libav:: +* generic source installation - libextractor:: +* generic source installation - libgpg-error:: +* generic source installation - libgcrypt:: +* generic source installation - gnutls:: +* generic source installation - libmicrohttpd:: +* generic source installation - libgnurl:: +@end menu + +@node generic source installation - nettle +@subsection generic source installation - nettle +@example +$ wget http://www.lysator.liu.se/~nisse/archive/nettle-2.7.1.tar.gz +$ tar xf nettle-2.7.1.tar.gz +$ cd nettle-2.7.1 +$ ./configure +$ sudo make install ; cd .. +@end example + +@node generic source installation - ldns +@subsection generic source installation - ldns +@example +$ wget https://www.nlnetlabs.nl/downloads/ldns/ldns-1.6.16.tar.gz +$ tar xf ldns-1.6.16.tar.gz +$ cd ldns-1.6.16 +$ ./configure +$ sudo make install ; cd .. +@end example + +@node generic source installation - libunbound/unbound +@subsection generic source installation - libunbound/unbound +@example +$ wget https://unbound.net/downloads/unbound-1.4.21.tar.gz +$ tar xf unbound-1.4.21.tar.gz +$ cd unbound-1.4.21 +$ ./configure +$ sudo make install ; cd .. +@end example + +@node generic source installation - libav +@subsection generic source installation - libav +@example +$ wget https://libav.org/releases/libav-9.10.tar.xz +$ cd libav-0.9 ; ./configure --enable-shared; +$ make; sudo make install; cd .. +@end example + +@node generic source installation - libextractor +@subsection generic source installation - libextractor +@example +$ wget https://ftp.gnu.org/gnu/libextractor/libextractor-1.3.tar.gz +$ tar xvf libextractor-1.3.tar.gz +$ cd libextractor-1.3 ; ./configure; +$ make ; sudo make install; cd .. +@end example + +@node generic source installation - libgpg-error +@subsection generic source installation - libgpg-error +@example +$ wget https://ftp.gnupg.org/gcrypt/libgpg-error/libgpg-error-1.12.tar.bz2 +$ tar xvf libgpg-error-1.12.tar.bz2 +$ cd libgpg-error-1.12; ./configure; +$ make ; sudo make install; cd .. +@end example + +@node generic source installation - libgcrypt +@subsection generic source installation - libgcrypt +@example +$ wget https://ftp.gnupg.org/gcrypt/libgcrypt/libgcrypt-1.6.0.tar.bz2 +$ tar xvf libgcrypt-1.6.0.tar.bz2 +$ cd libgcrypt-1.6.0; ./configure --with-gpg-error-prefix=/usr/local; +$ make ; sudo make install ; cd .. +@end example + +@node generic source installation - gnutls +@subsection generic source installation - gnutls +@example +$ wget ftp://ftp.gnutls.org/gcrypt/gnutls/v3.2/gnutls-3.2.7.tar.xz +$ tar xvf gnutls-3.2.7.tar.xz +$ cd gnutls-3.2.7 ; ./configure; +$ make ; sudo make install ; cd .. +@end example + +@noindent +If you want a GnuTLS with DANE functionality, you have to compile +it against libunbound. + +@node generic source installation - libmicrohttpd +@subsection generic source installation - libmicrohttpd +@example +$ wget https://ftp.gnu.org/gnu/libmicrohttpd/libmicrohttpd-0.9.33.tar.gz +$ tar xvf libmicrohttpd-0.9.33.tar.gz +$ cd libmicrohttpd-0.9.33; ./configure; +$ make ; sudo make install ; cd .. +@end example + +@node generic source installation - libgnurl +@subsection generic source installation - libgnurl + +@example +$ wget https://gnunet.org/sites/default/files/gnurl-7.34.0.tar.bz2 +$ tar xvf gnurl-7.34.0.tar.bz2 +$ cd gnurl-7.34.0 +$ ./configure --enable-ipv6 --with-gnutls=/usr/local --without-libssh2 \ + --without-libmetalink --without-winidn --without-librtmp \ + --without-nghttp2 --without-nss --without-cyassl --without-polarssl \ + --without-ssl --without-winssl --without-darwinssl --disable-sspi \ + --disable-ntlm-wb --disable-ldap --disable-rtsp --disable-dict \ + --disable-telnet --disable-tftp --disable-pop3 --disable-imap \ + --disable-smtp --disable-gopher --disable-file --disable-ftp +$ make ; sudo make install; cd .. +@end example + +@menu +* Fixing libgnurl build issues:: +@end menu + +@node Fixing libgnurl build issues +@subsubsection Fixing libgnurl build issues + +@c FIXME: Obviously this subsection should be evaluated and +@c if still necessary moved into gnURL itself (README) or +@c into a separate section which deals with gnURL. +If you have to compile libgnurl from source (for example if the version +included in your distribution is too old or it's not included at all) +you perhaps might get an error message while running the +@command{configure} script: + +@example +$ configure +... +checking for 64-bit curl_off_t data type... unknown +checking for 32-bit curl_off_t data type... unknown +checking for 16-bit curl_off_t data type... unknown +configure: error: cannot find data type for curl_off_t. +@end example + +@noindent +Solution: + +Before running the @command{configure} script, set: + +@example +CFLAGS="-I. -I$BUILD_ROOT/include" +@end example + +@node Build Instructions for Microsoft Windows Platforms +@section Build Instructions for Microsoft Windows Platforms + +@menu +* Introduction to building on MS Windows:: +* Requirements:: +* Dependencies & Initial Setup:: +* GNUnet Installation:: +* Adjusting Windows for running and testing GNUnet:: +* Building the GNUnet Installer:: +* Using GNUnet with Netbeans on Windows:: +@end menu + +@node Introduction to building on MS Windows +@subsection Introduction to building on MS Windows + + +This document is a guide to building GNUnet and its dependencies on +Windows platforms. GNUnet development is mostly done under GNU/Linux and +especially git checkouts may not build out of the box. +We regret any inconvenience, and if you have problems, please report +them. + +@node Requirements +@subsection Requirements + +The Howto is based upon a @strong{Windows Server 2008 32bit} +@strong{Installation}, @strong{sbuild} and thus a +@uref{http://www.mingw.org/wiki/MSYS, MSYS+MinGW} +(W32-GCC-Compiler-Suite + Unix-like Userland) installation. sbuild +is a convenient set of scripts which creates a working msys/mingw +installation and installs most dependencies required for GNUnet. + +As of the point of the creation of these instructions, +GNUnet @strong{requires} a Windows @strong{Server} 2003 or +newer for full feature support. +Windows Vista and later will also work, but +@strong{non-server version can not run a VPN-Exit-Node} as the NAT +features have been removed as of Windows Vista. + +@c TODO: We should document Windows 10! +@c It seems like the situation hasn't changed with W10 + +@node Dependencies & Initial Setup +@subsection Dependencies & Initial Setup + + +@itemize @bullet + +@item +Install a fresh version of @strong{Python 2.x}, even if you are using a +x64-OS, install a 32-bit version for use with sbuild. +Python 3.0 is currently incompatible. + +@item +Install your favorite @uref{http://code.google.com/p/tortoisegit/, git} & +@uref{http://tortoisesvn.net/, subversion}-clients. + +@item +You will also need some archive-manager like +@uref{http://www.7-zip.org/, 7zip}. + +@item +Pull a copy of sbuild to a directory of your choice, which will be used +in the remainder of this guide. For now, we will use +@file{c:\gnunet\sbuild\} + +@item +in @file{sbuild\src\mingw\mingw32-buildall.sh}, comment out the packages +@strong{gnunet-svn} and @strong{gnunet-gtk-svn}, as we don't want sbuild +to compile/install those for us. + +@item +Follow LRN's sbuild installation instructions.- +@end itemize + +Please note that sbuild may (or will most likely) fail during +installation, thus you really HAVE to @strong{check the logfiles} created +during the installation process. +Certain packages may fail to build initially due to missing dependencies, +thus you may have to +@strong{substitute those with binary-versions initially}. Later on once +dependencies are satisfied you can re-build the newer package versions. + +@strong{It is normal that you may have to repeat this step multiple times +and there is no uniform way to fix all compile-time issues, as the +build-process of many of the dependencies installed are rather unstable +on win32 and certain releases may not even compile at all.} + +Most dependencies for GNUnet have been set up by sbuild, thus we now +should add the @file{bin/} directories in your new msys and mingw +installations to PATH. You will want to create a backup of your finished +msys-environment by now. + +@node GNUnet Installation +@subsection GNUnet Installation + +First, we need to launch our msys-shell, you can do this via + +@file{C:\gnunet\sbuild\msys\msys.bat} + +You might wish to take a look at this file and adjust some +login-parameters to your msys environment. + +Also, sbuild added two pointpoints to your msys-environment, though those +might remain invisible: + +@itemize @bullet + +@item +/mingw, which will mount your mingw-directory from sbuild/mingw and the +other one is + +@item +/src which contains all the installation sources sbuild just compiled. +@end itemize + +Check out the current GNUnet sources (git HEAD) from the +GNUnet repository "gnunet.git", we will do this in your home directory: + +@code{git clone https://gnunet.org/git/gnunet/ ~/gnunet} + +Now, we will first need to bootstrap the checked out installation and then +configure it accordingly. + +@example +cd ~/gnunet +./bootstrap +STRIP=true CPPFLAGS="-DUSE_IPV6=1 -DW32_VEH" CFLAGS="$CFLAGS -g -O2" \ +./configure --prefix=/ --docdir=/share/doc/gnunet \ +--with-libiconv-prefix=/mingw --with-libintl-prefix=/mingw \ +--with-libcurl=/mingw --with-extractor=/mingw --with-sqlite=/mingw \ +--with-microhttpd=/mingw --with-plibc=/mingw --enable-benchmarks \ +--enable-expensivetests --enable-experimental --with-qrencode=/mingw \ +--enable-silent-rules --enable-experimental 2>&1 | tee -a ./configure.log +@end example + +The parameters above will configure for a reasonable GNUnet installation +to the your msys-root directory. +Depending on which features your would like to build or you may need to +specify additional dependencies. Sbuild installed most libs into +the /mingw subdirectory, so remember to prefix library locations with +this path. + +Like on a unixoid system, you might want to use your home directory as +prefix for your own GNUnet installation for development, without tainting +the buildenvironment. Just change the "prefix" parameter to point towards +~/ in this case. + +Now it's time to compile GNUnet as usual. Though this will take some time, +so you may fetch yourself a coffee or some Mate now... + +@example +make ; make install +@end example + +@node Adjusting Windows for running and testing GNUnet +@subsection Adjusting Windows for running and testing GNUnet + +Assuming the build succeeded and you +@strong{added the bin directory of your GNUnet to PATH}, you can now use +your gnunet-installation as usual. +Remember that UAC or the windows firewall may popup initially, blocking +further execution of gnunet until you acknowledge them. + +You will also have to take the usual steps to get peer-to-peer (p2p) +software running properly (port forwarding, ...), +and GNUnet will require administrative permissions as it may even +install a device-driver (in case you are using gnunet-vpn and/or +gnunet-exit). + +@node Building the GNUnet Installer +@subsection Building the GNUnet Installer + +The GNUnet installer is made with +@uref{http://nsis.sourceforge.net/, NSIS}. +The installer script is located in @file{contrib\win} in the +GNUnet source tree. + +@node Using GNUnet with Netbeans on Windows +@subsection Using GNUnet with Netbeans on Windows + +TODO + +@node Build instructions for Debian 7.5 +@section Build instructions for Debian 7.5 + + +These are the installation instructions for Debian 7.5. They were tested +using a minimal, fresh Debian 7.5 AMD64 installation without non-free +software (no contrib or non-free). +By "minimal", we mean that during installation, we did not select any +desktop environment, servers or system utilities during the "tasksel" +step. Note that the packages and the dependencies that we will install +during this chapter take about 1.5 GB of disk space. +Combined with GNUnet and space for objects during compilation, you should +not even attempt this unless you have about 2.5 GB free after the minimal +Debian installation. +Using these instructions to build a VM image is likely to require a +minimum of 4-5 GB for the VM (as you will likely also want a desktop +manager). + +GNUnet's security model assumes that your @file{/home} directory is +encrypted. Thus, if possible, you should encrypt your home partition +(or per-user home directory). + +Naturally, the exact details of the starting state for your installation +should not matter much. For example, if you selected any of those +installation groups you might simply already have some of the necessary +packages installed. +We did this for testing, as this way we are less likely to forget to +mention a required package. +Note that we will not install a desktop environment, but of course you +will need to install one to use GNUnet's graphical user interfaces. +Thus, it is suggested that you simply install the desktop environment of +your choice before beginning with the instructions. + + + +@menu +* Update:: +* Stable? Hah!:: +* Update again:: +* Installing packages:: +* Installing dependencies from source:: +* Installing GNUnet from source:: +* But wait there is more!:: +@end menu + +@node Update +@subsection Update + +After any installation, you should begin by running + +@example +# apt-get update ; apt-get upgrade +@end example + +to ensure that all of your packages are up-to-date. Note that the "#" is +used to indicate that you need to type in this command as "root" +(or prefix with "sudo"), whereas "$" is used to indicate typing in a +command as a normal user. + +@node Stable? Hah! +@subsection Stable? Hah! + +Yes, we said we start with a Debian 7.5 "stable" system. However, to +reduce the amount of compilation by hand, we will begin by allowing the +installation of packages from the testing and unstable distributions as +well. +We will stick to "stable" packages where possible, but some packages will +be taken from the other distributions. +Start by modifying @file{/etc/apt/sources.list} to contain the +following (possibly adjusted to point to your mirror of choice): + +@example +# These were there before: +deb http://ftp.de.debian.org/debian/ wheezy main +deb-src http://ftp.de.debian.org/debian/ wheezy main +deb http://security.debian.org/ wheezy/updates main +deb-src http://security.debian.org/ wheezy/updates main +deb http://ftp.de.debian.org/debian/ wheezy-updates main +deb-src http://ftp.de.debian.org/debian/ wheezy-updates main + +# Add these lines (feel free to adjust the mirror): +deb http://ftp.de.debian.org/debian/ testing main +deb http://ftp.de.debian.org/debian/ unstable main +@end example + +The next step is to create/edit your @file{/etc/apt/preferences} +file to look like this: + +@example +Package: * +Pin: release a=stable,n=wheezy +Pin-Priority: 700 + +Package: * +Pin: release o=Debian,a=testing +Pin-Priority: 650 + +Package: * +Pin: release o=Debian,a=unstable +Pin-Priority: 600 +@end example + +You can read more about Apt Preferences here and here. +Note that other pinnings are likely to also work for GNUnet, the key +thing is that you need some packages from unstable (as shown below). +However, as unstable is unlikely to be comprehensive (missing packages) +or might be problematic (crashing packages), you probably want others +from stable and/or testing. + +@node Update again +@subsection Update again + +Now, run again@ + +@example +# apt-get update@ +# apt-get upgrade@ +@end example + +to ensure that all your new distribution indices are downloaded, and +that your pinning is correct: the upgrade step should cause no changes +at all. + +@node Installing packages +@subsection Installing packages + +We begin by installing a few Debian packages from stable:@ + +@example +# apt-get install gcc make python-zbar libltdl-dev libsqlite3-dev \ + libunistring-dev libopus-dev libpulse-dev openssl libglpk-dev \ + texlive libidn11-dev libmysqlclient-dev libpq-dev libarchive-dev \ + libbz2-dev libexiv2-dev libflac-dev libgif-dev libglib2.0-dev \ + libgtk-3-dev libmagic-dev libjpeg8-dev libmpeg2-4-dev libmp4v2-dev \ + librpm-dev libsmf-dev libtidy-dev libtiff5-dev libvorbis-dev \ + libogg-dev zlib1g-dev g++ gettext libgsf-1-dev libunbound-dev \ + libqrencode-dev libgladeui-dev nasm texlive-latex-extra \ + libunique-3.0-dev gawk miniupnpc libfuse-dev libbluetooth-dev +@end example + +After that, we install a few more packages from unstable:@ + +@example +# apt-get install -t unstable nettle-dev libgstreamer1.0-dev \ + gstreamer1.0-plugins-base gstreamer1.0-plugins-good \ + libgstreamer-plugins-base1.0-dev +@end example + +@node Installing dependencies from source +@subsection Installing dependencies from source + +Next, we need to install a few dependencies from source. +You might want to do this as a "normal" user and only run the +@code{make install} steps as root (hence the @code{sudo} in the +commands below). Also, you do this from any +directory. We begin by downloading all dependencies, then extracting the +sources, and finally compiling and installing the libraries. + +For these steps, follow the instructions given in the +installation from source instruction in this order: + +@itemize @bullet +@item @ref{generic source installation - libav} +@item @ref{generic source installation - libextractor} +@item @ref{generic source installation - libgpg-error} +@item @ref{generic source installation - libgcrypt} +@item @ref{generic source installation - gnutls} +@item @ref{generic source installation - libmicrohttpd} +@item @ref{generic source installation - libgnurl} +@end itemize + +@node Installing GNUnet from source +@subsection Installing GNUnet from source + + +For this, simply follow the generic installation instructions from +here. + +@node But wait there is more! +@subsection But wait there is more! + +So far, we installed all of the packages and dependencies required to +ensure that all of GNUnet would be built. +However, while for example the plugins to interact with the MySQL or +Postgres databases have been created, we did not actually install or +configure those databases. Thus, you will need to install +and configure those databases or stick with the default Sqlite database. +Sqlite is usually fine for most applications, but MySQL can offer better +performance and Postgres better resillience. + + +@node Installing GNUnet from Git on Ubuntu 14.4 +@section Installing GNUnet from Git on Ubuntu 14.4 + +@strong{Install the required build tools:} + +@example +$ sudo apt-get install git automake autopoint autoconf +@end example + +@strong{Install the required dependencies} + +@example +$ sudo apt-get install libltdl-dev libgpg-error-dev libidn11-dev \ + libunistring-dev libglpk-dev libbluetooth-dev libextractor-dev \ + libmicrohttpd-dev libgnutls28-dev +@end example + +@strong{Choose one or more database backends} + +@itemize @bullet + +@item SQLite3: + +@example +$ sudo apt-get install libsqlite3-dev +@end example + +@item MySQL: + +@example +$ sudo apt-get install libmysqlclient-dev +@end example + +@item PostgreSQL: + +@example +$ sudo apt-get install libpq-dev postgresql +@end example + +@end itemize + +@strong{Install the optional dependencies for gnunet-conversation:} + +@example +$ sudo apt-get install gstreamer1.0 libpulse-dev libopus-dev +@end example + +@strong{Install the libgrypt 1.6.1:} + +@itemize @bullet + +@item For Ubuntu 14.04: + +@example +$ sudo apt-get install libgcrypt20-dev +@end example + +@item For Ubuntu older 14.04: + +@example +$ wget ftp://ftp.gnupg.org/gcrypt/libgcrypt/libgcrypt-1.6.1.tar.bz2 +$ tar xf libgcrypt-1.6.1.tar.bz2 +$ cd libgcrypt-1.6.1 +$ ./configure +$ sudo make install +$ cd .. +@end example + +@end itemize + +@strong{Install libgnurl} + +@example +$ wget https://gnunet.org/sites/default/files/gnurl-7.35.0.tar.bz2 +$ tar xf gnurl-7.35.0.tar.bz2 +$ cd gnurl-7.35.0 +$ ./configure --enable-ipv6 --with-gnutls --without-libssh2 \ + --without-libmetalink --without-winidn --without-librtmp \ + --without-nghttp2 --without-nss --without-cyassl --without-polarssl \ + --without-ssl --without-winssl --without-darwinssl --disable-sspi \ + --disable-ntlm-wb --disable-ldap --disable-rtsp --disable-dict \ + --disable-telnet --disable-tftp --disable-pop3 --disable-imap \ + --disable-smtp --disable-gopher --disable-file --disable-ftp +$ sudo make install +$ cd .. +@end example + +@strong{Install GNUnet} + +@example +$ git clone https://gnunet.org/git/gnunet/ +$ cd gnunet/ +$ ./bootstrap +@end example + +If you want to: + +@itemize @bullet + +@item Install to a different directory: + +@example +--prefix=PREFIX +@end example + +@item +Have sudo permission, but do not want to compile as root: + +@example +--with-sudo +@end example + +@item +Want debug message enabled: + +@example +--enable-logging=verbose +@end example + +@end itemize + + +@example +$ ./configure [ --with-sudo | --prefix=PREFIX | --enable-logging=verbose] +$ make; sudo make install +@end example + +After installing it, you need to create an empty configuration file: + +@example +touch ~/.config/gnunet.conf +@end example + +And finally you can start GNUnet with + +@example +$ gnunet-arm -s +@end example + +@node Build instructions for Debian 8 +@section Build instructions for Debian 8 +@c FIXME: I -> we + +These are the installation instructions for Debian 8. They were tested +sing a fresh Debian 8 AMD64 installation without non-free software (no +contrib or non-free). During installation, I only selected "lxde" for the +desktop environment. +Note that the packages and the dependencies that we will install during +this chapter take about 1.5 GB of disk space. Combined with GNUnet and +space for objects during compilation, you should not even attempt this +unless you have about 2.5 GB free after the Debian installation. +Using these instructions to build a VM image is likely to require a +minimum of 4-5 GB for the VM (as you will likely also want a desktop +manager). + +GNUnet's security model assumes that your @code{/home} directory is +encrypted. +Thus, if possible, you should encrypt your entire disk, or at least just +your home partition (or per-user home directory). + +Naturally, the exact details of the starting state for your installation +should not matter much. +For example, if you selected any of those installation groups you might +simply already have some of the necessary packages installed. Thus, it is +suggested that you simply install the desktop environment of your choice +before beginning with the instructions. + + +@menu +* Update Debian:: +* Installing Debian Packages:: +* Installing Dependencies from Source2:: +* Installing GNUnet from Source2:: +* But wait (again) there is more!:: +@end menu + +@node Update Debian +@subsection Update Debian + +After any installation, you should begin by running + +@example +# apt-get update +# apt-get upgrade +@end example + +to ensure that all of your packages are up-to-date. Note that the "#" is +used to indicate that you need to type in this command as "root" (or +prefix with "sudo"), whereas "$" is used to indicate typing in a command +as a normal user. + +@node Installing Debian Packages +@subsection Installing Debian Packages + +We begin by installing a few Debian packages from stable: + +@example +# apt-get install gcc make python-zbar libltdl-dev libsqlite3-dev \ +libunistring-dev libopus-dev libpulse-dev openssl libglpk-dev texlive \ +libidn11-dev libmysqlclient-dev libpq-dev libarchive-dev libbz2-dev \ +libflac-dev libgif-dev libglib2.0-dev libgtk-3-dev libmpeg2-4-dev \ +libtidy-dev libvorbis-dev libogg-dev zlib1g-dev g++ gettext \ +libgsf-1-dev libunbound-dev libqrencode-dev libgladeui-dev nasm \ +texlive-latex-extra libunique-3.0-dev gawk miniupnpc libfuse-dev \ +libbluetooth-dev gstreamer1.0-plugins-base gstreamer1.0-plugins-good \ +libgstreamer-plugins-base1.0-dev nettle-dev libextractor-dev \ +libgcrypt20-dev libmicrohttpd-dev +@end example + +@node Installing Dependencies from Source2 +@subsection Installing Dependencies from Source2 + +Yes, we said we start with a Debian 8 "stable" system, but because Debian +linked GnuTLS without support for DANE, we need to compile a few things, +in addition to GNUnet, still by hand. Yes, you can run GNUnet using the +respective Debian packages, but then you will not get DANE support. + +Next, we need to install a few dependencies from source. You might want +to do this as a "normal" user and only run the @code{make install} steps +as root (hence the @code{sudo} in the commands below). Also, you do this +from any directory. We begin by downloading all dependencies, then +extracting the sources, and finally compiling and installing the +libraries: + +@example +$ wget ftp://ftp.gnutls.org/gcrypt/gnutls/v3.3/gnutls-3.3.12.tar.xz +$ tar xvf gnutls-3.3.12.tar.xz +$ cd gnutls-3.3.12 ; ./configure ; make ; sudo make install ; cd .. +@end example + +For the installation and compilation of libgnurl/gnURL refer to +the generic installation section, +@xref{generic source installation - libgnurl}. + +@node Installing GNUnet from Source2 +@subsection Installing GNUnet from Source2 + +For this, simply follow the generic installation instructions from@ +here. + +@node But wait (again) there is more! +@subsection But wait (again) there is more! + +So far, we installed all of the packages and dependencies required to +ensure that all of GNUnet would be built. However, while for example the +plugins to interact with the MySQL or Postgres databases have been +created, we did not actually install or configure those databases. +Thus, you will need to install and configure those databases or stick +with the default Sqlite database. Sqlite is usually fine for most +applications, but MySQL can offer better performance and Postgres better +resillience. + +@node Outdated build instructions for previous revisions +@section Outdated build instructions for previous revisions + +This chapter contains a collection of outdated, older installation guides. +They are mostly intended to serve as a starting point for writing +up-to-date instructions and should not be expected to work for +GNUnet 0.10.x. +A set of older installation instructions can also be found in the +file @file{doc/outdated-and-old-installation-instructions.txt} in the +source tree of GNUnet. + +This file covers old instructions which no longer receive security +updates or any kind of support. + +@menu +* Installing GNUnet 0.10.1 on Ubuntu 14.04:: +* Building GLPK for MinGW:: +* GUI build instructions for Ubuntu 12.04 using Subversion:: +@c * Installation with gnunet-update:: +* Instructions for Microsoft Windows Platforms (Old):: +@end menu + + +@node Installing GNUnet 0.10.1 on Ubuntu 14.04 +@subsection Installing GNUnet 0.10.1 on Ubuntu 14.04 + +Install the required dependencies: + +@example +$ sudo apt-get install libltdl-dev libgpg-error-dev libidn11-dev \ + libunistring-dev libglpk-dev libbluetooth-dev libextractor-dev \ + libmicrohttpd-dev libgnutls28-dev +@end example + +Choose one or more database backends: + +@itemize @bullet + +@item SQLite3 + +@example + $ sudo apt-get install libsqlite3-dev@ +@end example + +@item MySQL + +@example +$ sudo apt-get install libmysqlclient-dev@ +@end example + +@item PostgreSQL + +@example + $ sudo apt-get install libpq-dev postgresql@ +@end example + +@end itemize + +Install the optional dependencies for gnunet-conversation: + +@example + $ sudo apt-get install gstreamer1.0 libpulse-dev libopus-dev +@end example + +Install libgcrypt 1.6: + +@itemize @bullet + +@item For Ubuntu 14.04: + +@example +$ sudo apt-get install libgcrypt20-dev +@end example + +@item For Ubuntu older than 14.04: + +@example +wget ftp://ftp.gnupg.org/gcrypt/libgcrypt/libgcrypt-1.6.1.tar.bz2 +$ tar xf libgcrypt-1.6.1.tar.bz2 +$ cd libgcrypt-1.6.1 +$ ./configure +$ sudo make install +$ cd .. +@end example +@end itemize + +Install libgnurl: + +@pxref{generic source installation - libgnurl}. + +Install GNUnet: + +@example +$ wget http://ftpmirror.gnu.org/gnunet/gnunet-0.10.1.tar.gz +$ tar xf gnunet-0.10.1.tar.gz +$ cd gnunet-0.10.1 +@end example + +If you want to: + +@itemize @bullet + +@item +Install to a different directory: + +@example +--prefix=PREFIX +@end example + +@item +Have sudo permission, but do not want to compile as root: + +@example +--with-sudo +@end example + +@item +Want debug message enabled: + +@example +--enable-logging=verbose +@end example + +@end itemize + +@example +$ ./configure [ --with-sudo | --prefix=PREFIX | --enable-logging=verbose] +$ make; sudo make install +@end example + +After installing it, you need to create an empty configuration file: + +@example +touch ~/.config/gnunet.conf +@end example + +And finally you can start GNUnet with + +@example +$ gnunet-arm -s +@end example + +@node Building GLPK for MinGW +@subsection Building GLPK for MinGW + +GNUnet now requires the GNU Linear Programming Kit (GLPK). +Since there's is no package you can install with @code{mingw-get} you +have to compile it from source: + +@itemize @bullet + +@item Download the latest version from +@uref{http://ftp.gnu.org/gnu/glpk/} + +@item Unzip the downloaded source tarball using your favourite +unzipper application In the MSYS shell + +@item change to the respective directory + +@item Configure glpk for "i686-pc-mingw32": + +@example +./configure '--build=i686-pc-mingw32' +@end example + +@item run + +@example +make install check +@end example + +@end itemize + +MinGW does not automatically detect the correct buildtype so you have to +specify it manually. + + +@node GUI build instructions for Ubuntu 12.04 using Subversion +@subsection GUI build instructions for Ubuntu 12.04 using Subversion + +After installing GNUnet you can continue installing the GNUnet GUI tools: + +First, install the required dependencies: + +@example +$ sudo apt-get install libgladeui-dev libqrencode-dev +@end example + +Please ensure that the GNUnet shared libraries can be found by the linker. +If you installed GNUnet libraries in a non standard path +(say GNUNET_PREFIX=/usr/local/lib/), you can + +@itemize @bullet + +@item set the environmental variable permanently to: + +@example +LD_LIBRARY_PATH=$GNUNET_PREFIX +@end example + +@item or add @code{$GNUNET_PREFIX} to @file{/etc/ld.so.conf} + +@end itemize + +Now you can checkout and compile the GNUnet GUI tools: + +@example +$ git clone https://gnunet.org/git/gnunet-gtk +$ cd gnunet-gtk +$ ./bootstrap +$ ./configure --prefix=$GNUNET_PREFIX/.. --with-gnunet=$GNUNET_PREFIX/.. +$ make install +@end example + +@c @node Installation with gnunet-update +@c @subsection Installation with gnunet-update + +@c gnunet-update project is an effort to introduce updates to GNUnet +@c installations. An interesting to-be-implemented-feature of gnunet-update +@c is that these updates are propagated through GNUnet's peer-to-peer +@c network. More information about gnunet-update can be found at +@c @c FIXME: Use correct cgit URL +@c @uref{https://gnunet.org/git/gnunet-update.git/tree/plain/README}. + +@c While the project is still under development, we have implemented the +@c following features which we believe may be helpful for users and we +@c would like them to be tested: + +@c @itemize @bullet + +@c @item +@c Packaging GNUnet installation along with its run-time dependencies into +@c update packages + +@c @item +@c Installing update packages into compatible hosts + +@c @item +@c Updating an existing installation (which had been installed by +@c gnunet-update) to a newer one + +@c @end itemize + +@c The above said features of gnunet-update are currently available for +@c testing on GNU/Linux systems. + +@c The following is a guide to help you get started with gnunet-update. +@c It shows you how to install the testing binary packages of GNUnet +@c 0.9.1 we have at @uref{https://gnunet.org/install/}. + +@c gnunet-update needs the following dependencies: + +@c @itemize @bullet +@c @item +@c python @geq{} 2.6 + +@c @item +@c gnupg + +@c @item +@c python-gpgme +@c @end itemize + + +@c Checkout gnunet-update: + +@c @c FIXME: git! +@c @example +@c $ svn checkout -r24905 https://gnunet.org/svn/gnunet-update@ +@c @end example + +@c For security reasons, all packages released for gnunet-update from us are +@c signed with the key at @uref{https://gnunet.org/install/key.txt}. +@c You would need to import this key into your gpg key ring. +@c gnunet-update uses this key to verify the integrity of the packages it +@c installs: + +@c @example +@c $ gpg --recv-keys 7C613D78@ +@c @end example + +@c Download the packages relevant to your architecture (currently I have +@c access to GNU/Linux machines on x86_64 and i686, so only two for now, +@c hopefully more later) from https://gnunet.org/install/. + +@c To install the downloaded package into the directory /foo: + +@c @example +@c gnunet-update/bin/gnunet-update install downloaded/package /foo +@c @end example + +@c The installer reports the directories into which shared libraries and +@c dependencies have been installed. You may need to add the reported shared +@c library installation paths to LD_LIBRARY_PATH before you start running any +@c installed binaries. + +@c Please report bugs at https://gnunet.org/bugs/ under the project +@c 'gnunet-update'. + +@node Instructions for Microsoft Windows Platforms (Old) +@subsection Instructions for Microsoft Windows Platforms (Old) + +This document is a @b{DEPRECATED} installation guide for GNUnet on +Windows. +It will not work for recent GNUnet versions, but maybe it will be of +some use if problems arise. + +The Windows build uses a UNIX emulator for Windows, +@uref{http://www.mingw.org/, MinGW}, to build the executable modules. +These modules run natively on Windows and do not require additional +emulation software besides the usual dependencies. + +GNUnet development is mostly done under GNU/Linux and especially git +checkouts may not build out of the box. +We regret any inconvenience, and if you have problems, please report them. + +@menu +* Hardware and OS requirements:: +* Software installation:: +* Building libextractor and GNUnet:: +* Installer:: +* Source:: +@end menu + +@node Hardware and OS requirements +@subsubsection Hardware and OS requirements + +@itemize @bullet + +@item Pentium II or equivalent processor, @geq{} 350 MHz + +@item 128 MB RAM + +@item 600 MB free disk space + +@item Windows 2000 or Windows XP are recommended + +@end itemize + +@node Software installation +@subsubsection Software installation + +@itemize @bullet + +@item +@strong{Compression software}@ + +The software packages GNUnet depends on are usually compressed using UNIX +tools like @command{tar}, @command{gzip}, @command{xzip} and +@command{bzip2}. +If you do not already have an utility that is able to extract such +archives, get @uref{http://www.7-zip.org/, 7-Zip}. + +@item +@strong{UNIX environment}@ + +The MinGW project provides the compiler toolchain that is used to build +GNUnet. +Get the following packages from the +@uref{http://sourceforge.net/projects/mingw/files/, MinGW} project: + +@itemize @bullet + +@item GCC core +@item GCC g++ +@item MSYS +@item MSYS Developer Tool Kit (msysDTK) +@item MSYS Developer Tool Kit - msys-autoconf (bin) +@item MSYS Developer Tool Kit - msys-automake (bin) +@item MinGW Runtime +@item MinGW Utilities +@item Windows API +@item Binutils +@item make +@item pdcurses +@item GDB (snapshot) +@end itemize + +@itemize @bullet + + +@item Install MSYS (to c:\mingw, for example.)@ +Do @strong{not} use spaces in the pathname. +For example, avoid a location such as @file{c:\program files\mingw}. + +@item Install MinGW runtime, utilities and GCC to a subdirectory +(to @file{c:\mingw\mingw}, for example) + +@item Install the Development Kit to the MSYS directory +(@file{c:\mingw}) + +@item Create a batch file bash.bat in your MSYS directory with +the files: + +@example +bin\sh.exe --login +@end example + +This batch file opens a shell which is used to invoke the build +processes. +MinGW's standard shell (@command{msys.bat}) is not suitable +because it opens a separate console window. +On Vista, @command{bash.bat} needs to be run as Administrator. + +@item +Start @command{bash.sh} and rename +@file{c:\mingw\mingw\lib\libstdc++.la} to avoid problems: + +@example +mv /usr/mingw/lib/libstdc++.la /usr/mingw/lib/libstdc++.la.broken +@end example + +@item +Unpack the Windows API to the MinGW directory (@file{c:\mingw\mingw\}) and +remove the declaration of DATADIR from +(@file{c:\mingw\mingw\include\objidl.h} (lines 55-58) + +@item +Unpack autoconf, automake to the MSYS directory (@file{c:\mingw}) + +@item +Install all other packages to the MinGW directory (@file{c:\mingw\mingw\}) +@end itemize + + +@item @strong{GNU Libtool}@ +GNU Libtool is required to use shared libraries. +Get the prebuilt package from here and unpack it to the +MinGW directory (@file{c:\mingw}) + +@item @strong{Pthreads}@ +GNUnet uses the portable POSIX thread library for multi-threading: + +@itemize @bullet + +@item Save +@uref{ftp://sources.redhat.com/pub/pthreads-win32/dll-latest/lib/x86/libpthreadGC2.a, libpthreadGC2.a} +(x86) or +@uref{ftp://sources.redhat.com/pub/pthreads-win32/dll-latest/lib/x64/libpthreadGC2.a, libpthreadGC2.a} +(x64) as libpthread.a into the @file{lib} +directory (@file{c:\mingw\mingw\lib\libpthread.a}). + +@item Save +@uref{ftp://sources.redhat.com/pub/pthreads-win32/dll-latest/lib/x86/pthreadGC2.dll, pthreadGC2.dll} +(x86) or +@uref{ftp://sources.redhat.com/pub/pthreads-win32/dll-latest/lib/x64/pthreadGC2.dll, libpthreadGC2.a} +(x64) into the MinGW @file{bin} directory (@file{c:\mingw\mingw\bin}). + +@item Download all header files from +@uref{ftp://sources.redhat.com/pub/pthreads-win32/dll-latest/include/, include/} +to the @file{include} directory (@file{c:\mingw\mingw\include}). +@end itemize + + +@item @strong{GNU MP}@ +GNUnet uses the GNU Multiple Precision library for special cryptographic +operations. Get the GMP binary package from the +@uref{http://sourceforge.net/projects/mingwrep/, MinGW repository} and +unpack it to the MinGW directory (@file{c:\mingw\mingw}) + +@item @strong{GNU Gettext}@ +GNU gettext is used to provide national language support. +Get the prebuilt package from hereand unpack it to the MinGW +directory (@file{c:\mingw\mingw}) + +@item @strong{GNU iconv}@ +GNU Libiconv is used for character encoding conversion. +Get the prebuilt package from here and unpack it to the MinGW +directory (@file{c:\mingw\mingw}). + +@item @strong{SQLite}@ +GNUnet uses the SQLite database to store data. +Get the prebuilt binary from here and unpack it to your MinGW directory. + +@item @strong{MySQL}@ +As an alternative to SQLite, GNUnet also supports MySQL. + +@itemize @bullet + +@item Get the binary installer from the +@uref{http://dev.mysql.com/downloads/mysql/4.1.html#Windows, MySQL project} +(version 4.1), install it and follow the instructions in +@file{README.mysql}. + +@item Create a temporary build directory (@file{c:\mysql}) + +@item Copy the directories @file{include\} and @file{lib\} from the +MySQL directory to the new directory + +@item Get the patches from +@uref{http://bugs.mysql.com/bug.php?id=8906&files=1, Bug #8906} and +@uref{http://bugs.mysql.com/bug.php?id=8872&files=1, Bug #8872} (the +latter is only required for MySQL + +@example +patch -p 0 +@end example + +@item Move @file{lib\opt\libmysql.dll} to @file{lib\libmysql.dll} + +@item Change to @file{lib\} and create an import library: + +@example +dlltool --input-def ../include/libmySQL.def \ +--dllname libmysql.dll \ +--output-lib libmysqlclient.a -k +@end example + +@item Copy include\* to include\mysql\ + +@item Pass @code{--with-mysql=/c/mysql} to +@command{./configure} and copy @file{libmysql.dll} +to your PATH or GNUnet's @file{bin} directory +@end itemize + + +@item @strong{GTK+}@ +@command{gnunet-gtk} and @command{libextractor} depend on GTK. +Get the the binary and developer packages of @command{atk}, +@command{glib}, @command{gtk}, @command{iconv}, +@command{gettext-runtime}, @command{pango} from +@uref{ftp://ftp.gtk.org/pub/gtk/v2.6/win32, gtk.org} and unpack them +to the MinGW directory (@file{c:\mingw\mingw}). +@c FIXME: The URL below for pkg-config seems wrong. +Get @uref{http://www.gtk.org/download/win32.php, pkg-config} and +@command{libpng} and unpack them to the MinGW directory +(@file{c:\mingw\mingw}). +Here is an all-in-one package for the +@uref{http://ftp.gnome.org/pub/gnome/binaries/win32/gtk+/2.24/gtk+-bundle_2.24.10-20120208_win32.zip, gtk+dependencies} +. Do not overwrite any existing files! + +@item @strong{Glade}@ +@command{gnunet-gtk} and @command{gnunet-setup} were created using +this interface builder + +@itemize @bullet + +@item Get the Glade and libglade (-bin and -devel) packages +(without GTK!) from +@uref{http://gladewin32.sourceforge.net/, GladeWin32} and unpack them to +the MinGW directory (@file{c:\mingw\mingw}). + +@item Get @command{libxml} from here and unpack it to the MinGW +directory (@file{c:\mingw\mingw}). +@end itemize + +@c FIXME: URLs +@item @strong{zLib}@ +@command{libextractor} requires @command{zLib} to decompress some file +formats. GNUnet uses it to (de)compress meta-data. +Get zLib from here (Signature) and unpack it to the MinGW directory +(@file{c:\mingw\mingw}). + +@item @strong{Bzip2}@ +@command{libextractor} also requires @command{Bzip2} to +decompress some file formats. +Get the Bzip2 (binary and developer package) from +@uref{http://gnuwin32.sourceforge.net/packages/bzip2.htm, GnuWin32} and +unpack it to the MinGW directory (@file{c:\mingw\mingw}). + +@item @strong{Libgcrypt}@ +@command{Libgcrypt} provides the cryptographic functions used by GNUnet. +Get Libgcrypt from @uref{ftp://ftp.gnupg.org/gcrypt/libgcrypt/, here}, +compile and place it in the MinGW directory +(@file{c:\mingw\mingw}). Currently libgcrypt @geq{} 1.4.2 is required to +compile GNUnet. + +@item @strong{PlibC}@ +PlibC emulates Unix functions under Windows. Get PlibC from here and +unpack it to the MinGW directory (c:\mingw\mingw) + +@item @strong{OGG Vorbis}@ +@command{OGG Vorbis} is used to extract meta-data from @file{.ogg} files. +Get the packages +@uref{http://www.gnunet.org/libextractor/download/win/libogg-1.1.4.zip, libogg} +and +@uref{http://www.gnunet.org/libextractor/download/win/libvorbis-1.2.3.zip, libvorbis} +from the +@uref{http://ftp.gnu.org/gnu/libextractor/libextractor-w32-1.0.0.zip, libextractor win32 build} +and unpack them to the MinGW directory (c:\mingw\mingw). + +@item @strong{Exiv2}@ +(lib)Exiv2 is used to extract meta-data from files with Exiv2 meta-data. +Download +@uref{http://www.gnunet.org/libextractor/download/win/exiv2-0.18.2.zip, Exiv2} +and unpack it to the MSYS directory (c:\mingw). +@end itemize + +@node Building libextractor and GNUnet +@subsubsection Building libextractor and GNUnet + +Before you compile @command{libextractor} or @command{GNUnet}, +be sure to set @code{PKG_CONFIG_PATH}: + +@example +export PKG_CONFIG_PATH=/mingw/lib/pkgconfig +@end example + +@noindent +@xref{GNUnet Installation Handbook}, for basic instructions on building +@command{libextractor} and @command{GNUnet}. +By default, all modules that are created in this way contain +debug information and are quite large. To compile release versions +(small and fast) set the variable @code{CFLAGS}: + +@example +export CFLAGS='-O2 -march=pentium -fomit-frame-pointer' +./configure --prefix=$HOME --with-extractor=$HOME +@end example + +@node Installer +@subsubsection Installer + +The GNUnet installer is made with +@uref{http://nsis.sourceforge.net/, NSIS}. The installer script is +located in @file{contrib\win} in the GNUnet source tree. + +@node Source +@subsubsection Source + +@c FIXME: URL +The sources of all dependencies are available here. + +@c @node Portable GNUnet +@c @section Portable GNUnet + +@c Quick instructions on how to use the most recent GNUnet on most GNU/Linux +@c distributions + +@c Currently this has only been tested on Ubuntu 12.04, 12.10, 13.04, Debian +@c and CentOS 6, but it should work on almost any GNU/Linux distribution. +@c More in-detail information can be found in the handbook. + +@c Note 2017-10: Currently this section assumes the old SVN repo of GNUnet +@c which no longer exists. + +@c @menu +@c * Prerequisites:: +@c * Download & set up gnunet-update:: +@c * Install GNUnet:: +@c @end menu + +@c @node Prerequisites +@c @subsection Prerequisites + +@c Open a terminal and paste this line into it to install all required tools +@c needed: + +@c @example +@c sudo apt-get install python-gpgme subversion +@c @end example + +@c @node Download & set up gnunet-update +@c @subsection Download & set up gnunet-update + +@c The following command will download a working version of gnunet-update +@c with the subversion tool and import the public key which is needed for +@c authentication: + +@c @example +@c svn checkout -r24905 https://gnunet.org/svn/gnunet-update ~/gnunet-update +@c cd ~/gnunet-update +@c gpg --keyserver "hkp://keys.gnupg.net" --recv-keys 7C613D78 +@c @end example + +@c @node Install GNUnet +@c @subsection Install GNUnet + +@c Download and install GNUnet binaries which can be found here and set +@c library paths: + +@c @example +@c wget -P /tmp https://gnunet.org/install/packs/gnunet-0.9.4-`uname -m`.tgz +@c ./bin/gnunet-update install /tmp/gnunet-0.9*.tgz ~ +@c echo "PATH DEFAULT=$@{PATH@}:$HOME/bin" >> ~/.pam_environment +@c echo -e "$@{HOME@}/lib\n$@{HOME@}/lib/gnunet-deps" | sudo tee \ +@c /etc/ld.so.conf.d/gnunet.conf > /dev/null +@c sudo ldconfig +@c @end example + +@c You may need to re-login once after executing these last commands + +@c That's it, GNUnet is installed in your home directory now. GNUnet can be +@c configured and afterwards started by executing: + +@c @example +@c gnunet-arm -s +@c @end example + +@node The graphical configuration interface +@section The graphical configuration interface + +If you also would like to use @command{gnunet-gtk} and +@command{gnunet-setup} (highly recommended for beginners), do: + +@example +wget -P /tmp \ +https://gnunet.org/install/packs/gnunet-0.9.4-gtk-0.9.4-`uname -m`.tgz +sh ~/gnunet-update/bin/gnunet-update install /tmp/gnunet-*gtk*.tgz ~ +sudo ldconfig +@end example + +Now you can run @command{gnunet-setup} for easy configuration of your +GNUnet peer. + +@menu +* Configuring your peer:: +* Configuring the Friend-to-Friend (F2F) mode:: +* Configuring the hostlist to bootstrap:: +* Configuration of the HOSTLIST proxy settings:: +* Configuring your peer to provide a hostlist :: +* Configuring the datastore:: +* Configuring the MySQL database:: +* Reasons for using MySQL:: +* Reasons for not using MySQL:: +* Setup Instructions:: +* Testing:: +* Performance Tuning:: +* Setup for running Testcases:: +* Configuring the Postgres database:: +* Reasons to use Postgres:: +* Reasons not to use Postgres:: +* Manual setup instructions:: +* Testing the setup manually:: +* Configuring the datacache:: +* Configuring the file-sharing service:: +* Configuring logging:: +* Configuring the transport service and plugins:: +* Configuring the wlan transport plugin:: +* Configuring HTTP(S) reverse proxy functionality using Apache or nginx:: +* Blacklisting peers:: +* Configuration of the HTTP and HTTPS transport plugins:: +* Configuring the GNU Name System:: +* Configuring the GNUnet VPN:: +* Bandwidth Configuration:: +* Configuring NAT:: +* Peer configuration for distributions:: +@end menu + +@node Configuring your peer +@subsection Configuring your peer + +This chapter will describe the various configuration options in GNUnet. + +The easiest way to configure your peer is to use the +@command{gnunet-setup} tool. +@command{gnunet-setup} is part of the @command{gnunet-gtk} +application. You might have to install it separately. + +Many of the specific sections from this chapter actually are linked from +within @command{gnunet-setup} to help you while using the setup tool. + +While you can also configure your peer by editing the configuration +file by hand, this is not recommended for anyone except for developers +as it requires a more in-depth understanding of the configuration files +and internal dependencies of GNUnet. + +@node Configuring the Friend-to-Friend (F2F) mode +@subsection Configuring the Friend-to-Friend (F2F) mode + +GNUnet knows three basic modes of operation: +@itemize @bullet +@item In standard "peer-to-peer" mode, +your peer will connect to any peer. +@item In the pure "friend-to-friend" +mode, your peer will ONLY connect to peers from a list of friends +specified in the configuration. +@item Finally, in mixed mode, +GNUnet will only connect to arbitrary peers if it +has at least a specified number of connections to friends. +@end itemize + +When configuring any of the F2F ("friend-to-friend") modes, +you first need to create a file with the peer identities +of your friends. Ask your friends to run + +@example +$ gnunet-peerinfo -sq +@end example + +@noindent +The resulting output of this command needs to be added to your +@file{friends} file, which is simply a plain text file with one line +per friend with the output from the above command. + +You then specify the location of your @file{friends} file in the +@code{FRIENDS} option of the "topology" section. + +Once you have created the @file{friends} file, you can tell GNUnet to only +connect to your friends by setting the @code{FRIENDS-ONLY} option +(again in the "topology" section) to YES. + +If you want to run in mixed-mode, set "FRIENDS-ONLY" to NO and configure a +minimum number of friends to have (before connecting to arbitrary peers) +under the "MINIMUM-FRIENDS" option. + +If you want to operate in normal P2P-only mode, simply set +@code{MINIMUM-FRIENDS} to zero and @code{FRIENDS_ONLY} to NO. +This is the default. + +@node Configuring the hostlist to bootstrap +@subsection Configuring the hostlist to bootstrap + +After installing the software you need to get connected to the GNUnet +network. The configuration file included in your download is already +configured to connect you to the GNUnet network. +In this section the relevant configuration settings are explained. + +To get an initial connection to the GNUnet network and to get to know +peers already connected to the network you can use the so called +"bootstrap servers". +These servers can give you a list of peers connected to the network. +To use these bootstrap servers you have to configure the hostlist daemon +to activate bootstrapping. + +To activate bootstrapping, edit the @code{[hostlist]}-section in your +configuration file. You have to set the argument @command{-b} in the +options line: + +@example +[hostlist] +OPTIONS = -b +@end example + +Additionally you have to specify which server you want to use. +The default bootstrapping server is +"@uref{http://v10.gnunet.org/hostlist, http://v10.gnunet.org/hostlist}". +[^] To set the server you have to edit the line "SERVERS" in the hostlist +section. To use the default server you should set the lines to + +@example +SERVERS = http://v10.gnunet.org/hostlist [^] +@end example + +@noindent +To use bootstrapping your configuration file should include these lines: + +@example +[hostlist] +OPTIONS = -b +SERVERS = http://v10.gnunet.org/hostlist [^] +@end example + +@noindent +Besides using bootstrap servers you can configure your GNUnet peer to +recieve hostlist advertisements. +Peers offering hostlists to other peers can send advertisement messages +to peers that connect to them. If you configure your peer to receive these +messages, your peer can download these lists and connect to the peers +included. These lists are persistent, which means that they are saved to +your hard disk regularly and are loaded during startup. + +To activate hostlist learning you have to add the @command{-e} +switch to the @code{OPTIONS} line in the hostlist section: + +@example +[hostlist] +OPTIONS = -b -e +@end example + +@noindent +Furthermore you can specify in which file the lists are saved. +To save the lists in the file @file{hostlists.file} just add the line: + +@example +HOSTLISTFILE = hostlists.file +@end example + +@noindent +Best practice is to activate both bootstrapping and hostlist learning. +So your configuration file should include these lines: + +@example +[hostlist] +OPTIONS = -b -e +HTTPPORT = 8080 +SERVERS = http://v10.gnunet.org/hostlist [^] +HOSTLISTFILE = $SERVICEHOME/hostlists.file +@end example + +@node Configuration of the HOSTLIST proxy settings +@subsection Configuration of the HOSTLIST proxy settings + +The hostlist client can be configured to use a proxy to connect to the +hostlist server. +This functionality can be configured in the configuration file directly +or using the @command{gnunet-setup} tool. + +The hostlist client supports the following proxy types at the moment: + +@itemize @bullet +@item HTTP and HTTP 1.0 only proxy +@item SOCKS 4/4a/5/5 with hostname +@end itemize + +In addition authentication at the proxy with username and password can be +configured. + +To configure proxy support for the hostlist client in the +@command{gnunet-setup} tool, select the "hostlist" tab and select +the appropriate proxy type. +The hostname or IP address (including port if required) has to be entered +in the "Proxy hostname" textbox. If required, enter username and password +in the "Proxy username" and "Proxy password" boxes. +Be aware that this information will be stored in the configuration in +plain text (TODO: Add explanation and generalize the part in Chapter 3.6 +about the encrypted home). + +To provide these options directly in the configuration, you can +enter the following settings in the @code{[hostlist]} section of +the configuration: + +@example +# Type of proxy server, +# Valid values: HTTP, HTTP_1_0, SOCKS4, SOCKS5, SOCKS4A, SOCKS5_HOSTNAME +# Default: HTTP +# PROXY_TYPE = HTTP + +# Hostname or IP of proxy server +# PROXY = +# User name for proxy server +# PROXY_USERNAME = +# User password for proxy server +# PROXY_PASSWORD = +@end example + +@node Configuring your peer to provide a hostlist +@subsection Configuring your peer to provide a hostlist + +If you operate a peer permanently connected to GNUnet you can configure +your peer to act as a hostlist server, providing other peers the list of +peers known to him. + +Your server can act as a bootstrap server and peers needing to obtain a +list of peers can contact it to download this list. +To download this hostlist the peer uses HTTP. +For this reason you have to build your peer with libgnurl (or libcurl) +and microhttpd support. +How you build your peer with these options can be found here: +@xref{Generic installation instructions}. + +To configure your peer to act as a bootstrap server you have to add the +@command{-p} option to @code{OPTIONS} in the @code{[hostlist]} section +of your configuration file. +Besides that you have to specify a port number for the http server. +In conclusion you have to add the following lines: + +@example +[hostlist] +HTTPPORT = 12980 +OPTIONS = -p +@end example + +@noindent +If your peer acts as a bootstrap server other peers should know about +that. You can advertise the hostlist your are providing to other peers. +Peers connecting to your peer will get a message containing an +advertisement for your hostlist and the URL where it can be downloaded. +If this peer is in learning mode, it will test the hostlist and, in the +case it can obtain the list successfully, it will save it for +bootstrapping. + +To activate hostlist advertisement on your peer, you have to set the +following lines in your configuration file: + +@example +[hostlist] +EXTERNAL_DNS_NAME = example.org +HTTPPORT = 12981 +OPTIONS = -p -a +@end example + +@noindent +With this configuration your peer will a act as a bootstrap server and +advertise this hostlist to other peers connecting to it. +The URL used to download the list will be +@code{@uref{http://example.org:12981/, http://example.org:12981/}}. + +Please notice: + +@itemize @bullet +@item The hostlist is @b{not} human readable, so you should not try to +download it using your webbrowser. Just point your GNUnet peer to the +address! +@item Advertising without providing a hostlist does not make sense and +will not work. +@end itemize + +@node Configuring the datastore +@subsection Configuring the datastore + +The datastore is what GNUnet uses for long-term storage of file-sharing +data. Note that long-term does not mean 'forever' since content does have +an expiration date, and of course storage space is finite (and hence +sometimes content may have to be discarded). + +Use the @code{QUOTA} option to specify how many bytes of storage space +you are willing to dedicate to GNUnet. + +In addition to specifying the maximum space GNUnet is allowed to use for +the datastore, you need to specify which database GNUnet should use to do +so. Currently, you have the choice between sqLite, MySQL and Postgres. + +@node Configuring the MySQL database +@subsection Configuring the MySQL database + +This section describes how to setup the MySQL database for GNUnet. + +Note that the mysql plugin does NOT work with mysql before 4.1 since we +need prepared statements. +We are generally testing the code against MySQL 5.1 at this point. + +@node Reasons for using MySQL +@subsection Reasons for using MySQL + +@itemize @bullet + +@item On up-to-date hardware wher +mysql can be used comfortably, this module +will have better performance than the other database choices (according +to our tests). + +@item Its often possible to recover the mysql database from internal +inconsistencies. Some of the other databases do not support repair. +@end itemize + +@node Reasons for not using MySQL +@subsection Reasons for not using MySQL + +@itemize @bullet +@item Memory usage (likely not an issue if you have more than 1 GB) +@item Complex manual setup +@end itemize + +@node Setup Instructions +@subsection Setup Instructions + +@itemize @bullet + +@item In @file{gnunet.conf} set in section @code{DATASTORE} the value for +@code{DATABASE} to @code{mysql}. + +@item Access mysql as root: + +@example +$ mysql -u root -p +@end example + +@noindent +and issue the following commands, replacing $USER with the username +that will be running @command{gnunet-arm} (so typically "gnunet"): + +@example +CREATE DATABASE gnunet; +GRANT select,insert,update,delete,create,alter,drop,create \ +temporary tables ON gnunet.* TO $USER@@localhost; +SET PASSWORD FOR $USER@@localhost=PASSWORD('$the_password_you_like'); +FLUSH PRIVILEGES; +@end example + +@item +In the $HOME directory of $USER, create a @file{.my.cnf} file with the +following lines + +@example +[client] +user=$USER +password=$the_password_you_like +@end example + +@end itemize + +Thats it. Note that @file{.my.cnf} file is a slight security risk unless +its on a safe partition. The @file{$HOME/.my.cnf} can of course be +a symbolic link. +Luckily $USER has only priviledges to mess up GNUnet's tables, +which should be pretty harmless. + +@node Testing +@subsection Testing + +You should briefly try if the database connection works. First, login +as $USER. Then use: + +@example +$ mysql -u $USER +mysql> use gnunet; +@end example + +@noindent +If you get the message + +@example +Database changed +@end example + +@noindent +it probably works. + +If you get + +@example +ERROR 2002: Can't connect to local MySQL server +through socket '/tmp/mysql.sock' (2) +@end example + +@noindent +it may be resolvable by + +@example +ln -s /var/run/mysqld/mysqld.sock /tmp/mysql.sock +@end example + +@noindent +so there may be some additional trouble depending on your mysql setup. + +@node Performance Tuning +@subsection Performance Tuning + +For GNUnet, you probably want to set the option + +@example +innodb_flush_log_at_trx_commit = 0 +@end example + +@noindent +for a rather dramatic boost in MySQL performance. However, this reduces +the "safety" of your database as with this options you may loose +transactions during a power outage. +While this is totally harmless for GNUnet, the option applies to all +applications using MySQL. So you should set it if (and only if) GNUnet is +the only application on your system using MySQL. + +@node Setup for running Testcases +@subsection Setup for running Testcases + +If you want to run the testcases, you must create a second database +"gnunetcheck" with the same username and password. This database will +then be used for testing (@command{make check}). + +@node Configuring the Postgres database +@subsection Configuring the Postgres database + +This text describes how to setup the Postgres database for GNUnet. + +This Postgres plugin was developed for Postgres 8.3 but might work for +earlier versions as well. + +@node Reasons to use Postgres +@subsection Reasons to use Postgres + +@itemize @bullet +@item Easier to setup than MySQL +@item Real database +@end itemize + +@node Reasons not to use Postgres +@subsection Reasons not to use Postgres + +@itemize @bullet +@item Quite slow +@item Still some manual setup required +@end itemize + +@node Manual setup instructions +@subsection Manual setup instructions + +@itemize @bullet +@item In @file{gnunet.conf} set in section @code{DATASTORE} the value for +@code{DATABASE} to @code{postgres}. +@item Access Postgres to create a user: + +@table @asis +@item with Postgres 8.x, use: + +@example +# su - postgres +$ createuser +@end example + +@noindent +and enter the name of the user running GNUnet for the role interactively. +Then, when prompted, do not set it to superuser, allow the creation of +databases, and do not allow the creation of new roles. + +@item with Postgres 9.x, use: + +@example +# su - postgres +$ createuser -d $GNUNET_USER +@end example + +@noindent +where $GNUNET_USER is the name of the user running GNUnet. + +@end table + + +@item +As that user (so typically as user "gnunet"), create a database (or two): + +@example +$ createdb gnunet +# this way you can run "make check" +$ createdb gnunetcheck +@end example + +@end itemize + +Now you should be able to start @code{gnunet-arm}. + +@node Testing the setup manually +@subsection Testing the setup manually + +You may want to try if the database connection works. First, again login +as the user who will run @command{gnunet-arm}. Then use: + +@example +$ psql gnunet # or gnunetcheck +gnunet=> \dt +@end example + +@noindent +If, after you have started @command{gnunet-arm} at least once, you get +a @code{gn090} table here, it probably works. + +@node Configuring the datacache +@subsection Configuring the datacache +@c %**end of header + +The datacache is what GNUnet uses for storing temporary data. This data is +expected to be wiped completely each time GNUnet is restarted (or the +system is rebooted). + +You need to specify how many bytes GNUnet is allowed to use for the +datacache using the @code{QUOTA} option in the section @code{[dhtcache]}. +Furthermore, you need to specify which database backend should be used to +store the data. Currently, you have the choice between +sqLite, MySQL and Postgres. + +@node Configuring the file-sharing service +@subsection Configuring the file-sharing service + +In order to use GNUnet for file-sharing, you first need to make sure +that the file-sharing service is loaded. +This is done by setting the @code{AUTOSTART} option in +section @code{[fs]} to "YES". Alternatively, you can run + +@example +$ gnunet-arm -i fs +@end example + +@noindent +to start the file-sharing service by hand. + +Except for configuring the database and the datacache the only important +option for file-sharing is content migration. + +Content migration allows your peer to cache content from other peers as +well as send out content stored on your system without explicit requests. +This content replication has positive and negative impacts on both system +performance and privacy. + +FIXME: discuss the trade-offs. Here is some older text about it... + +Setting this option to YES allows gnunetd to migrate data to the local +machine. Setting this option to YES is highly recommended for efficiency. +Its also the default. If you set this value to YES, GNUnet will store +content on your machine that you cannot decrypt. +While this may protect you from liability if the judge is sane, it may +not (IANAL). If you put illegal content on your machine yourself, setting +this option to YES will probably increase your chances to get away with it +since you can plausibly deny that you inserted the content. +Note that in either case, your anonymity would have to be broken first +(which may be possible depending on the size of the GNUnet network and the +strength of the adversary). + +@node Configuring logging +@subsection Configuring logging + +Logging in GNUnet 0.9.0 is controlled via the "-L" and "-l" options. +Using @code{-L}, a log level can be specified. With log level +@code{ERROR} only serious errors are logged. +The default log level is @code{WARNING} which causes anything of +concern to be logged. +Log level @code{INFO} can be used to log anything that might be +interesting information whereas +@code{DEBUG} can be used by developers to log debugging messages +(but you need to run @code{./configure} with +@code{--enable-logging=verbose} to get them compiled). +The @code{-l} option is used to specify the log file. + +Since most GNUnet services are managed by @code{gnunet-arm}, using the +@code{-l} or @code{-L} options directly is not possible. +Instead, they can be specified using the @code{OPTIONS} configuration +value in the respective section for the respective service. +In order to enable logging globally without editing the @code{OPTIONS} +values for each service, @command{gnunet-arm} supports a +@code{GLOBAL_POSTFIX} option. +The value specified here is given as an extra option to all services for +which the configuration does contain a service-specific @code{OPTIONS} +field. + +@code{GLOBAL_POSTFIX} can contain the special sequence "@{@}" which +is replaced by the name of the service that is being started. +Furthermore, @code{GLOBAL_POSTFIX} is special in that sequences +starting with "$" anywhere in the string are expanded (according +to options in @code{PATHS}); this expansion otherwise is +only happening for filenames and then the "$" must be the +first character in the option. Both of these restrictions do +not apply to @code{GLOBAL_POSTFIX}. +Note that specifying @code{%} anywhere in the @code{GLOBAL_POSTFIX} +disables both of these features. + +In summary, in order to get all services to log at level +@code{INFO} to log-files called @code{SERVICENAME-logs}, the +following global prefix should be used: + +@example +GLOBAL_POSTFIX = -l $SERVICEHOME/@{@}-logs -L INFO +@end example + +@node Configuring the transport service and plugins +@subsection Configuring the transport service and plugins + +The transport service in GNUnet is responsible to maintain basic +connectivity to other peers. +Besides initiating and keeping connections alive it is also responsible +for address validation. + +The GNUnet transport supports more than one transport protocol. +These protocols are configured together with the transport service. + +The configuration section for the transport service itself is quite +similar to all the other services + +@example +AUTOSTART = YES +@@UNIXONLY@@ PORT = 2091 +HOSTNAME = localhost +HOME = $SERVICEHOME +CONFIG = $DEFAULTCONFIG +BINARY = gnunet-service-transport +#PREFIX = valgrind +NEIGHBOUR_LIMIT = 50 +ACCEPT_FROM = 127.0.0.1; +ACCEPT_FROM6 = ::1; +PLUGINS = tcp udp +UNIXPATH = /tmp/gnunet-service-transport.sock +@end example + +Different are the settings for the plugins to load @code{PLUGINS}. +The first setting specifies which transport plugins to load. + +@itemize @bullet +@item transport-unix +A plugin for local only communication with UNIX domain sockets. Used for +testing and available on unix systems only. Just set the port + +@example +[transport-unix] +PORT = 22086 +TESTING_IGNORE_KEYS = ACCEPT_FROM; +@end example + +@item transport-tcp +A plugin for communication with TCP. Set port to 0 for client mode with +outbound only connections + +@example +[transport-tcp] +# Use 0 to ONLY advertise as a peer behind NAT (no port binding) +PORT = 2086 +ADVERTISED_PORT = 2086 +TESTING_IGNORE_KEYS = ACCEPT_FROM; +# Maximum number of open TCP connections allowed +MAX_CONNECTIONS = 128 +@end example + +@item transport-udp +A plugin for communication with UDP. Supports peer discovery using +broadcasts. + +@example +[transport-udp] +PORT = 2086 +BROADCAST = YES +BROADCAST_INTERVAL = 30 s +MAX_BPS = 1000000 +TESTING_IGNORE_KEYS = ACCEPT_FROM; +@end example + +@item transport-http +HTTP and HTTPS support is split in two part: a client plugin initiating +outbound connections and a server part accepting connections from the +client. The client plugin just takes the maximum number of connections as +an argument. + +@example +[transport-http_client] +MAX_CONNECTIONS = 128 +TESTING_IGNORE_KEYS = ACCEPT_FROM; +@end example + +@example +[transport-https_client] +MAX_CONNECTIONS = 128 +TESTING_IGNORE_KEYS = ACCEPT_FROM; +@end example + +@noindent +The server has a port configured and the maximum nunber of connections. +The HTTPS part has two files with the certificate key and the certificate +file. + +The server plugin supports reverse proxies, so a external hostname can be +set using the @code{EXTERNAL_HOSTNAME} setting. +The webserver under this address should forward the request to the peer +and the configure port. + +@example +[transport-http_server] +EXTERNAL_HOSTNAME = fulcrum.net.in.tum.de/gnunet +PORT = 1080 +MAX_CONNECTIONS = 128 +TESTING_IGNORE_KEYS = ACCEPT_FROM; +@end example + +@example +[transport-https_server] +PORT = 4433 +CRYPTO_INIT = NORMAL +KEY_FILE = https.key +CERT_FILE = https.cert +MAX_CONNECTIONS = 128 +TESTING_IGNORE_KEYS = ACCEPT_FROM; +@end example + +@item transport-wlan + +The next section describes how to setup the WLAN plugin, +so here only the settings. Just specify the interface to use: + +@example +[transport-wlan] +# Name of the interface in monitor mode (typically monX) +INTERFACE = mon0 +# Real hardware, no testing +TESTMODE = 0 +TESTING_IGNORE_KEYS = ACCEPT_FROM; +@end example +@end itemize + +@node Configuring the wlan transport plugin +@subsection Configuring the wlan transport plugin + +The wlan transport plugin enables GNUnet to send and to receive data on a +wlan interface. +It has not to be connected to a wlan network as long as sender and +receiver are on the same channel. This enables you to get connection to +GNUnet where no internet access is possible, for example during +catastrophes or when censorship cuts you off from the internet. + + +@menu +* Requirements for the WLAN plugin:: +* Configuration:: +* Before starting GNUnet:: +* Limitations and known bugs:: +@end menu + + +@node Requirements for the WLAN plugin +@subsubsection Requirements for the WLAN plugin + +@itemize @bullet + +@item wlan network card with monitor support and packet injection +(see @uref{http://www.aircrack-ng.org/, aircrack-ng.org}) + +@item Linux kernel with mac80211 stack, introduced in 2.6.22, tested with +2.6.35 and 2.6.38 + +@item Wlantools to create the a monitor interface, tested with airmon-ng +of the aircrack-ng package +@end itemize + +@node Configuration +@subsubsection Configuration + +There are the following options for the wlan plugin (they should be like +this in your default config file, you only need to adjust them if the +values are incorrect for your system) + +@example +# section for the wlan transport plugin +[transport-wlan] +# interface to use, more information in the +# "Before starting GNUnet" section of the handbook. +INTERFACE = mon0 +# testmode for developers: +# 0 use wlan interface, +#1 or 2 use loopback driver for tests 1 = server, 2 = client +TESTMODE = 0 +@end example + +@node Before starting GNUnet +@subsubsection Before starting GNUnet + +Before starting GNUnet, you have to make sure that your wlan interface is +in monitor mode. +One way to put the wlan interface into monitor mode (if your interface +name is wlan0) is by executing: + +@example +sudo airmon-ng start wlan0 +@end example + +@noindent +Here is an example what the result should look like: + +@example +Interface Chipset Driver +wlan0 Intel 4965 a/b/g/n iwl4965 - [phy0] +(monitor mode enabled on mon0) +@end example + +@noindent +The monitor interface is mon0 is the one that you have to put into the +configuration file. + +@node Limitations and known bugs +@subsubsection Limitations and known bugs + +Wlan speed is at the maximum of 1 Mbit/s because support for choosing the +wlan speed with packet injection was removed in newer kernels. +Please pester the kernel developers about fixing this. + +The interface channel depends on the wlan network that the card is +connected to. If no connection has been made since the start of the +computer, it is usually the first channel of the card. +Peers will only find each other and communicate if they are on the same +channel. Channels must be set manually, i.e. using: + +@example +iwconfig wlan0 channel 1 +@end example + +@node Configuring HTTP(S) reverse proxy functionality using Apache or nginx +@subsection Configuring HTTP(S) reverse proxy functionality using Apache or nginx + +The HTTP plugin supports data transfer using reverse proxies. A reverse +proxy forwards the HTTP request he receives with a certain URL to another +webserver, here a GNUnet peer. + +So if you have a running Apache or nginx webserver you can configure it to +be a GNUnet reverse proxy. Especially if you have a well-known webiste +this improves censorship resistance since it looks as normal surfing +behaviour. + +To do so, you have to do two things: + +@itemize @bullet +@item Configure your webserver to forward the GNUnet HTTP traffic +@item Configure your GNUnet peer to announce the respective address +@end itemize + +As an example we want to use GNUnet peer running: + +@itemize @bullet + +@item HTTP server plugin on @code{gnunet.foo.org:1080} + +@item HTTPS server plugin on @code{gnunet.foo.org:4433} + +@item A apache or nginx webserver on +@uref{http://www.foo.org/, http://www.foo.org:80/} + +@item A apache or nginx webserver on https://www.foo.org:443/ +@end itemize + +And we want the webserver to accept GNUnet traffic under +@code{http://www.foo.org/bar/}. The required steps are described here: + +@menu +* Reverse Proxy - Configure your Apache2 HTTP webserver:: +* Reverse Proxy - Configure your Apache2 HTTPS webserver:: +* Reverse Proxy - Configure your nginx HTTPS webserver:: +* Reverse Proxy - Configure your nginx HTTP webserver:: +* Reverse Proxy - Configure your GNUnet peer:: +@end menu + +@node Reverse Proxy - Configure your Apache2 HTTP webserver +@subsubsection Reverse Proxy - Configure your Apache2 HTTP webserver + +First of all you need mod_proxy installed. + +Edit your webserver configuration. Edit +@code{/etc/apache2/apache2.conf} or the site-specific configuration file. + +In the respective @code{server config},@code{virtual host} or +@code{directory} section add the following lines: + +@example +ProxyTimeout 300 +ProxyRequests Off +<Location /bar/ > +ProxyPass http://gnunet.foo.org:1080/ +ProxyPassReverse http://gnunet.foo.org:1080/ +</Location> +@end example + +@node Reverse Proxy - Configure your Apache2 HTTPS webserver +@subsubsection Reverse Proxy - Configure your Apache2 HTTPS webserver + +We assume that you already have an HTTPS server running, if not please +check how to configure a HTTPS host. An easy to use example is the +@file{apache2/sites-available/default-ssl} example configuration file. + +In the respective HTTPS @code{server config},@code{virtual host} or +@code{directory} section add the following lines: + +@example +SSLProxyEngine On +ProxyTimeout 300 +ProxyRequests Off +<Location /bar/ > +ProxyPass https://gnunet.foo.org:4433/ +ProxyPassReverse https://gnunet.foo.org:4433/ +</Location> +@end example + +@noindent +More information about the apache mod_proxy configuration can be found +here: @uref{http://httpd.apache.org/docs/2.2/mod/mod_proxy.html#proxypass} +. + +@node Reverse Proxy - Configure your nginx HTTPS webserver +@subsubsection Reverse Proxy - Configure your nginx HTTPS webserver + +Since nginx does not support chunked encoding, you first of all have to +install @code{chunkin}: @uref{http://wiki.nginx.org/HttpChunkinModule}. + +To enable chunkin add: + +@example +chunkin on; +error_page 411 = @@my_411_error; +location @@my_411_error @{ +chunkin_resume; +@} +@end example + +@noindent +Edit your webserver configuration. Edit @file{/etc/nginx/nginx.conf} or +the site-specific configuration file. + +In the @code{server} section add: + +@example +location /bar/ +@{ +proxy_pass http://gnunet.foo.org:1080/; +proxy_buffering off; +proxy_connect_timeout 5; # more than http_server +proxy_read_timeout 350; # 60 default, 300s is GNUnet's idle timeout +proxy_http_version 1.1; # 1.0 default +proxy_next_upstream error timeout invalid_header http_500 http_503 http_502 http_504; +@} +@end example + +@node Reverse Proxy - Configure your nginx HTTP webserver +@subsubsection Reverse Proxy - Configure your nginx HTTP webserver + +Edit your webserver configuration. Edit @file{/etc/nginx/nginx.conf} or +the site-specific configuration file. + +In the @code{server} section add: + +@example +ssl_session_timeout 6m; +location /bar/ +@{ +proxy_pass https://gnunet.foo.org:4433/; +proxy_buffering off; +proxy_connect_timeout 5; # more than http_server +proxy_read_timeout 350; # 60 default, 300s is GNUnet's idle timeout +proxy_http_version 1.1; # 1.0 default +proxy_next_upstream error timeout invalid_header http_500 http_503 http_502 http_504; +@} +@end example + +@node Reverse Proxy - Configure your GNUnet peer +@subsubsection Reverse Proxy - Configure your GNUnet peer + +To have your GNUnet peer announce the address, you have to specify the +@code{EXTERNAL_HOSTNAME} option in the @code{[transport-http_server]} +section: + +@example +[transport-http_server] +EXTERNAL_HOSTNAME = http://www.foo.org/bar/ +@end example + +@noindent +and/or @code{[transport-https_server]} section: + +@example +[transport-https_server] +EXTERNAL_HOSTNAME = https://www.foo.org/bar/ +@end example + +@noindent +Now restart your webserver and your peer... + +@node Blacklisting peers +@subsection Blacklisting peers + +Transport service supports to deny connecting to a specific peer of to a +specific peer with a specific transport plugin using te blacklisting +component of transport service. With@ blacklisting it is possible to deny +connections to specific peers of@ to use a specific plugin to a specific +peer. Peers can be blacklisted using@ the configuration or a blacklist +client can be asked. + +To blacklist peers using the configuration you have to add a section to +your configuration containing the peer id of the peer to blacklist and +the plugin@ if required. + +Examples: + +To blacklist connections to P565... on peer AG2P... using tcp add: + +@c FIXME: This is too long and produces errors in the pdf. +@example +[transport-blacklist AG2PHES1BARB9IJCPAMJTFPVJ5V3A72S3F2A8SBUB8DAQ2V0O3V8G6G2JU56FHGFOHMQVKBSQFV98TCGTC3RJ1NINP82G0RC00N1520] +P565723JO1C2HSN6J29TAQ22MN6CI8HTMUU55T0FUQG4CMDGGEQ8UCNBKUMB94GC8R9G4FB2SF9LDOBAJ6AMINBP4JHHDD6L7VD801G = tcp +@end example + +To blacklist connections to P565... on peer AG2P... using all plugins add: + +@example +[transport-blacklist-AG2PHES1BARB9IJCPAMJTFPVJ5V3A72S3F2A8SBUB8DAQ2V0O3V8G6G2JU56FHGFOHMQVKBSQFV98TCGTC3RJ1NINP82G0RC00N1520] +P565723JO1C2HSN6J29TAQ22MN6CI8HTMUU55T0FUQG4CMDGGEQ8UCNBKUMB94GC8R9G4FB2SF9LDOBAJ6AMINBP4JHHDD6L7VD801G = +@end example + +You can also add a blacklist client usign the blacklist API. On a +blacklist check, blacklisting first checks internally if the peer is +blacklisted and if not, it asks the blacklisting clients. Clients are +asked if it is OK to connect to a peer ID, the plugin is omitted. + +On blacklist check for (peer, plugin) +@itemize @bullet +@item Do we have a local blacklist entry for this peer and this plugin?@ +@item YES: disallow connection@ +@item Do we have a local blacklist entry for this peer and all plugins?@ +@item YES: disallow connection@ +@item Does one of the clients disallow?@ +@item YES: disallow connection +@end itemize + +@node Configuration of the HTTP and HTTPS transport plugins +@subsection Configuration of the HTTP and HTTPS transport plugins + +The client parts of the http and https transport plugins can be configured +to use a proxy to connect to the hostlist server. This functionality can +be configured in the configuration file directly or using the +gnunet-setup tool. + +Both the HTTP and HTTPS clients support the following proxy types at +the moment: + +@itemize @bullet +@item HTTP 1.1 proxy +@item SOCKS 4/4a/5/5 with hostname +@end itemize + +In addition authentication at the proxy with username and password can be +configured. + +To configure proxy support for the clients in the gnunet-setup tool, +select the "transport" tab and activate the respective plugin. Now you +can select the appropriate proxy type. The hostname or IP address +(including port if required) has to be entered in the "Proxy hostname" +textbox. If required, enter username and password in the "Proxy username" +and "Proxy password" boxes. Be aware that these information will be stored +in the configuration in plain text. + +To configure these options directly in the configuration, you can +configure the following settings in the @code{[transport-http_client]} +and @code{[transport-https_client]} section of the configuration: + +@example +# Type of proxy server, +# Valid values: HTTP, SOCKS4, SOCKS5, SOCKS4A, SOCKS5_HOSTNAME +# Default: HTTP +# PROXY_TYPE = HTTP + +# Hostname or IP of proxy server +# PROXY = +# User name for proxy server +# PROXY_USERNAME = +# User password for proxy server +# PROXY_PASSWORD = +@end example + +@node Configuring the GNU Name System +@subsection Configuring the GNU Name System + +@menu +* Configuring system-wide DNS interception:: +* Configuring the GNS nsswitch plugin:: +* Configuring GNS on W32:: +* GNS Proxy Setup:: +* Setup of the GNS CA:: +* Testing the GNS setup:: +* Automatic Shortening in the GNU Name System:: +@end menu + + +@node Configuring system-wide DNS interception +@subsubsection Configuring system-wide DNS interception + +Before you install GNUnet, make sure you have a user and group 'gnunet' +as well as an empty group 'gnunetdns'. + +When using GNUnet with system-wide DNS interception, it is absolutely +necessary for all GNUnet service processes to be started by +@code{gnunet-service-arm} as user and group 'gnunet'. You also need to be +sure to run @code{make install} as root (or use the @code{sudo} option to +configure) to grant GNUnet sufficient privileges. + +With this setup, all that is required for enabling system-wide DNS +interception is for some GNUnet component (VPN or GNS) to request it. +The @code{gnunet-service-dns} will then start helper programs that will +make the necessary changes to your firewall (@code{iptables}) rules. + +Note that this will NOT work if your system sends out DNS traffic to a +link-local IPv6 address, as in this case GNUnet can intercept the traffic, +but not inject the responses from the link-local IPv6 address. Hence you +cannot use system-wide DNS interception in conjunction with link-local +IPv6-based DNS servers. If such a DNS server is used, it will bypass +GNUnet's DNS traffic interception. + +Using the GNU Name System (GNS) requires two different configuration +steps. +First of all, GNS needs to be integrated with the operating system. Most +of this section is about the operating system level integration. + +Additionally, each individual user who wants to use the system must also +initialize their GNS zones. This can be done by running (after starting +GNUnet) + +@example +$ gnunet-gns-import.sh +@end example + +@noindent +after the local GNUnet peer has been started. Note that the namestore (in +particular the namestore database backend) should not be reconfigured +afterwards (as records are not automatically migrated between backends). + +The remainder of this chapter will detail the various methods for +configuring the use of GNS with your operating system. + +At this point in time you have different options depending on your OS: + +@table @asis + +@item Use the gnunet-gns-proxy This approach works for all operating +systems and is likely the easiest. However, it enables GNS only for +browsers, not for other applications that might be using DNS, such as SSH. +Still, using the proxy is required for using HTTP with GNS and is thus +recommended for all users. To do this, you simply have to run the +@code{gnunet-gns-proxy-setup-ca} script as the user who will run the +browser (this will create a GNS certificate authority (CA) on your system +and import its key into your browser), then start @code{gnunet-gns-proxy} +and inform your browser to use the Socks5 proxy which +@code{gnunet-gns-proxy} makes available by default on port 7777. +@item Use a nsswitch plugin (recommended on GNU systems) +This approach has the advantage of offering fully personalized resolution +even on multi-user systems. A potential disadvantage is that some +applications might be able to bypass GNS. +@item Use a W32 resolver plugin (recommended on W32) +This is currently the only option on W32 systems. +@item Use system-wide DNS packet interception +This approach is recommended for the GNUnet VPN. It can be used to handle +GNS at the same time; however, if you only use this method, you will only +get one root zone per machine (not so great for multi-user systems). +@end table + +You can combine system-wide DNS packet interception with the nsswitch +plugin. +The setup of the system-wide DNS interception is described here. All of +the other GNS-specific configuration steps are described in the following +sections. + +@node Configuring the GNS nsswitch plugin +@subsubsection Configuring the GNS nsswitch plugin + +The Name Service Switch (NSS) is a facility in Unix-like operating systems +@footnote{More accurate: NSS is a functionality of the GNU C Library} +that provides a variety of sources for common configuration databases and +name resolution mechanisms. +A superuser (system administrator) usually configures the +operating system's name services using the file +@file{/etc/nsswitch.conf}. + +GNS provides a NSS plugin to integrate GNS name resolution with the +operating system's name resolution process. +To use the GNS NSS plugin you have to either + +@itemize @bullet +@item install GNUnet as root or +@item compile GNUnet with the @code{--with-sudo=yes} switch. +@end itemize + +Name resolution is controlled by the @emph{hosts} section in the NSS +configuration. By default this section first performs a lookup in the +@file{/etc/hosts} file and then in DNS. +The nsswitch file should contain a line similar to: + +@example +hosts: files dns [NOTFOUND=return] mdns4_minimal mdns4 +@end example + +@noindent +Here the GNS NSS plugin can be added to perform a GNS lookup before +performing a DNS lookup. +The GNS NSS plugin has to be added to the "hosts" section in +@file{/etc/nsswitch.conf} file before DNS related plugins: + +@example +... +hosts: files gns [NOTFOUND=return] dns mdns4_minimal mdns4 +... +@end example + +@noindent +The @code{NOTFOUND=return} will ensure that if a @code{.gnu} name is not +found in GNS it will not be queried in DNS. + +@node Configuring GNS on W32 +@subsubsection Configuring GNS on W32 + +This document is a guide to configuring GNU Name System on W32-compatible +platforms. + +After GNUnet is installed, run the w32nsp-install tool: + +@example +w32nsp-install.exe libw32nsp-0.dll +@end example + +@noindent +('0' is the library version of W32 NSP; it might increase in the future, +change the invocation accordingly). + +This will install GNS namespace provider into the system and allow other +applications to resolve names that end in '@strong{gnu}' +and '@strong{zkey}'. Note that namespace provider requires +gnunet-gns-helper-service-w32 to be running, as well as gns service +itself (and its usual dependencies). + +Namespace provider is hardcoded to connect to @strong{127.0.0.1:5353}, +and this is where gnunet-gns-helper-service-w32 should be listening to +(and is configured to listen to by default). + +To uninstall the provider, run: + +@example +w32nsp-uninstall.exe +@end example + +@noindent +(uses provider GUID to uninstall it, does not need a dll name). + +Note that while MSDN claims that other applications will only be able to +use the new namespace provider after re-starting, in reality they might +stat to use it without that. Conversely, they might stop using the +provider after it's been uninstalled, even if they were not re-started. +W32 will not permit namespace provider library to be deleted or +overwritten while the provider is installed, and while there is at least +one process still using it (even after it was uninstalled). + +@node GNS Proxy Setup +@subsubsection GNS Proxy Setup + +When using the GNU Name System (GNS) to browse the WWW, there are several +issues that can be solved by adding the GNS Proxy to your setup: + +@itemize @bullet + +@item If the target website does not support GNS, it might assume that it +is operating under some name in the legacy DNS system (such as +example.com). It may then attempt to set cookies for that domain, and the +web server might expect a @code{Host: example.com} header in the request +from your browser. +However, your browser might be using @code{example.gnu} for the +@code{Host} header and might only accept (and send) cookies for +@code{example.gnu}. The GNS Proxy will perform the necessary translations +of the hostnames for cookies and HTTP headers (using the LEHO record for +the target domain as the desired substitute). + +@item If using HTTPS, the target site might include an SSL certificate +which is either only valid for the LEHO domain or might match a TLSA +record in GNS. However, your browser would expect a valid certificate for +@code{example.gnu}, not for some legacy domain name. The proxy will +validate the certificate (either against LEHO or TLSA) and then +on-the-fly produce a valid certificate for the exchange, signed by your +own CA. Assuming you installed the CA of your proxy in your browser's +certificate authority list, your browser will then trust the +HTTPS/SSL/TLS connection, as the hostname mismatch is hidden by the proxy. + +@item Finally, the proxy will in the future indicate to the server that it +speaks GNS, which will enable server operators to deliver GNS-enabled web +sites to your browser (and continue to deliver legacy links to legacy +browsers) +@end itemize + +@node Setup of the GNS CA +@subsubsection Setup of the GNS CA + +First you need to create a CA certificate that the proxy can use. +To do so use the provided script gnunet-gns-proxy-ca: + +@example +$ gnunet-gns-proxy-setup-ca +@end example + +@noindent +This will create a personal certification authority for you and add this +authority to the firefox and chrome database. The proxy will use the this +CA certificate to generate @code{*.gnu} client certificates on the fly. + +Note that the proxy uses libcurl. Make sure your version of libcurl uses +GnuTLS and NOT OpenSSL. The proxy will @b{not} work with libcurl compiled +against OpenSSL. + +You can check the configuration your libcurl was build with by +running: + +@example +curl --version +@end example + +the output will look like this (without the linebreaks): + +@example +gnurl --version +curl 7.56.0 (x86_64-unknown-linux-gnu) libcurl/7.56.0 \ +GnuTLS/3.5.13 zlib/1.2.11 libidn2/2.0.4 +Release-Date: 2017-10-08 +Protocols: http https +Features: AsynchDNS IDN IPv6 Largefile NTLM SSL libz \ +TLS-SRP UnixSockets HTTPS-proxy +@end example + +@node Testing the GNS setup +@subsubsection Testing the GNS setup + +Now for testing purposes we can create some records in our zone to test +the SSL functionality of the proxy: + +@example +$ gnunet-namestore -a -e "1 d" -n "homepage" -t A -V 131.159.74.67 +$ gnunet-namestore -a -e "1 d" -n "homepage" -t LEHO -V "gnunet.org" +@end example + +@noindent +At this point we can start the proxy. Simply execute + +@example +$ gnunet-gns-proxy +@end example + +@noindent +Configure your browser to use this SOCKSv5 proxy on port 7777 and visit +this link. +If you use @command{Firefox} (or one of its deriviates/forks such as +Icecat) you also have to go to @code{about:config} and set the key +@code{network.proxy.socks_remote_dns} to @code{true}. + +When you visit @code{https://homepage.gnu/}, you should get to the +@code{https://gnunet.org/} frontpage and the browser (with the correctly +configured proxy) should give you a valid SSL certificate for +@code{homepage.gnu} and no warnings. It should look like this: + +@c FIXME: Image does not exist, create it or save it from Drupal? +@c @image{images/gnunethpgns.png,5in,, picture of homepage.gnu in Webbrowser} + +@node Automatic Shortening in the GNU Name System +@subsubsection Automatic Shortening in the GNU Name System + +This page describes a possible option for 'automatic name shortening', +which you can choose to enable with the GNU Name System. + +When GNS encounters a name for the first time, it can use the 'NICK' +record of the originating zone to automatically generate a name for the +zone. If automatic shortening is enabled, those auto-generated names will +be placed (as private records) into your personal 'shorten' zone (to +prevent confusion with manually selected names). +Then, in the future, if the same name is encountered again, GNS will +display the shortened name instead (the first time, the long name will +still be used as shortening typically happens asynchronously as looking up +the 'NICK' record takes some time). Using this feature can be a convenient +way to avoid very long @code{.gnu} names; however, note that names from +the shorten-zone are assigned on a first-come-first-serve basis and should +not be trusted. Furthermore, if you enable this feature, you will no +longer see the full delegation chain for zones once shortening has been +applied. + +@node Configuring the GNUnet VPN +@subsection Configuring the GNUnet VPN + +@menu +* IPv4 address for interface:: +* IPv6 address for interface:: +* Configuring the GNUnet VPN DNS:: +* Configuring the GNUnet VPN Exit Service:: +* IP Address of external DNS resolver:: +* IPv4 address for Exit interface:: +* IPv6 address for Exit interface:: +@end menu + +Before configuring the GNUnet VPN, please make sure that system-wide DNS +interception is configured properly as described in the section on the +GNUnet DNS setup. @pxref{Configuring the GNU Name System}, +if you haven't done so already. + +The default options for the GNUnet VPN are usually sufficient to use +GNUnet as a Layer 2 for your Internet connection. +However, what you always have to specify is which IP protocol you want +to tunnel: IPv4, IPv6 or both. +Furthermore, if you tunnel both, you most likely should also tunnel +all of your DNS requests. +You theoretically can tunnel "only" your DNS traffic, but that usually +makes little sense. + +The other options as shown on the gnunet-setup tool are: + +@node IPv4 address for interface +@subsubsection IPv4 address for interface + +This is the IPv4 address the VPN interface will get. You should pick an +'private' IPv4 network that is not yet in use for you system. For example, +if you use @code{10.0.0.1/255.255.0.0} already, you might use +@code{10.1.0.1/255.255.0.0}. +If you use @code{10.0.0.1/255.0.0.0} already, then you might use +@code{192.168.0.1/255.255.0.0}. +If your system is not in a private IP-network, using any of the above will +work fine. +You should try to make the mask of the address big enough +(@code{255.255.0.0} or, even better, @code{255.0.0.0}) to allow more +mappings of remote IP Addresses into this range. +However, even a @code{255.255.255.0} mask will suffice for most users. + +@node IPv6 address for interface +@subsubsection IPv6 address for interface + +The IPv6 address the VPN interface will get. Here you can specify any +non-link-local address (the address should not begin with @code{fe80:}). +A subnet Unique Local Unicast (@code{fd00::/8} prefix) that you are +currently not using would be a good choice. + +@node Configuring the GNUnet VPN DNS +@subsubsection Configuring the GNUnet VPN DNS + +To resolve names for remote nodes, activate the DNS exit option. + +@node Configuring the GNUnet VPN Exit Service +@subsubsection Configuring the GNUnet VPN Exit Service + +If you want to allow other users to share your Internet connection (yes, +this may be dangerous, just as running a Tor exit node) or want to +provide access to services on your host (this should be less dangerous, +as long as those services are secure), you have to enable the GNUnet exit +daemon. + +You then get to specify which exit functions you want to provide. By +enabling the exit daemon, you will always automatically provide exit +functions for manually configured local services (this component of the +system is under +development and not documented further at this time). As for those +services you explicitly specify the target IP address and port, there is +no significant security risk in doing so. + +Furthermore, you can serve as a DNS, IPv4 or IPv6 exit to the Internet. +Being a DNS exit is usually pretty harmless. However, enabling IPv4 or +IPv6-exit without further precautions may enable adversaries to access +your local network, send spam, attack other systems from your Internet +connection and to other mischief that will appear to come from your +machine. This may or may not get you into legal trouble. +If you want to allow IPv4 or IPv6-exit functionality, you should strongly +consider adding additional firewall rules manually to protect your local +network and to restrict outgoing TCP traffic (i.e. by not allowing access +to port 25). While we plan to improve exit-filtering in the future, +you're currently on your own here. +Essentially, be prepared for any kind of IP-traffic to exit the respective +TUN interface (and GNUnet will enable IP-forwarding and NAT for the +interface automatically). + +Additional configuration options of the exit as shown by the gnunet-setup +tool are: + +@node IP Address of external DNS resolver +@subsubsection IP Address of external DNS resolver + +If DNS traffic is to exit your machine, it will be send to this DNS +resolver. You can specify an IPv4 or IPv6 address. + +@node IPv4 address for Exit interface +@subsubsection IPv4 address for Exit interface + +This is the IPv4 address the Interface will get. Make the mask of the +address big enough (255.255.0.0 or, even better, 255.0.0.0) to allow more +mappings of IP addresses into this range. As for the VPN interface, any +unused, private IPv4 address range will do. + +@node IPv6 address for Exit interface +@subsubsection IPv6 address for Exit interface + +The public IPv6 address the interface will get. If your kernel is not a +very recent kernel and you are willing to manually enable IPv6-NAT, the +IPv6 address you specify here must be a globally routed IPv6 address of +your host. + +Suppose your host has the address @code{2001:4ca0::1234/64}, then +using @code{2001:4ca0::1:0/112} would be fine (keep the first 64 bits, +then change at least one bit in the range before the bitmask, in the +example above we changed bit 111 from 0 to 1). + +You may also have to configure your router to route traffic for the entire +subnet (@code{2001:4ca0::1:0/112} for example) through your computer (this +should be automatic with IPv6, but obviously anything can be +disabled). + +@node Bandwidth Configuration +@subsection Bandwidth Configuration + +You can specify how many bandwidth GNUnet is allowed to use to receive +and send data. This is important for users with limited bandwidth or +traffic volume. + +@node Configuring NAT +@subsection Configuring NAT + +Most hosts today do not have a normal global IP address but instead are +behind a router performing Network Address Translation (NAT) which assigns +each host in the local network a private IP address. +As a result, these machines cannot trivially receive inbound connections +from the Internet. GNUnet supports NAT traversal to enable these machines +to receive incoming connections from other peers despite their +limitations. + +In an ideal world, you can press the "Attempt automatic configuration" +button in gnunet-setup to automatically configure your peer correctly. +Alternatively, your distribution might have already triggered this +automatic configuration during the installation process. +However, automatic configuration can fail to determine the optimal +settings, resulting in your peer either not receiving as many connections +as possible, or in the worst case it not connecting to the network at all. + +To manually configure the peer, you need to know a few things about your +network setup. First, determine if you are behind a NAT in the first +place. +This is always the case if your IP address starts with "10.*" or +"192.168.*". Next, if you have control over your NAT router, you may +choose to manually configure it to allow GNUnet traffic to your host. +If you have configured your NAT to forward traffic on ports 2086 (and +possibly 1080) to your host, you can check the "NAT ports have been opened +manually" option, which corresponds to the "PUNCHED_NAT" option in the +configuration file. If you did not punch your NAT box, it may still be +configured to support UPnP, which allows GNUnet to automatically +configure it. In that case, you need to install the "upnpc" command, +enable UPnP (or PMP) on your NAT box and set the "Enable NAT traversal +via UPnP or PMP" option (corresponding to "ENABLE_UPNP" in the +configuration file). + +Some NAT boxes can be traversed using the autonomous NAT traversal method. +This requires certain GNUnet components to be installed with "SUID" +prividledges on your system (so if you're installing on a system you do +not have administrative rights to, this will not work). +If you installed as 'root', you can enable autonomous NAT traversal by +checking the "Enable NAT traversal using ICMP method". +The ICMP method requires a way to determine your NAT's external (global) +IP address. This can be done using either UPnP, DynDNS, or by manual +configuration. If you have a DynDNS name or know your external IP address, +you should enter that name under "External (public) IPv4 address" (which +corresponds to the "EXTERNAL_ADDRESS" option in the configuration file). +If you leave the option empty, GNUnet will try to determine your external +IP address automatically (which may fail, in which case autonomous +NAT traversal will then not work). + +Finally, if you yourself are not behind NAT but want to be able to +connect to NATed peers using autonomous NAT traversal, you need to check +the "Enable connecting to NATed peers using ICMP method" box. + + +@node Peer configuration for distributions +@subsection Peer configuration for distributions + +The "GNUNET_DATA_HOME" in "[path]" in @file{/etc/gnunet.conf} should be +manually set to "/var/lib/gnunet/data/" as the default +"~/.local/share/gnunet/" is probably not that appropriate in this case. +Similarly, distributions may consider pointing "GNUNET_RUNTIME_DIR" to +"/var/run/gnunet/" and "GNUNET_HOME" to "/var/lib/gnunet/". Also, should a +distribution decide to override system defaults, all of these changes +should be done in a custom @file{/etc/gnunet.conf} and not in the files +in the @file{config.d/} directory. + +Given the proposed access permissions, the "gnunet-setup" tool must be +run as use "gnunet" (and with option "-c /etc/gnunet.conf" so that it +modifies the system configuration). As always, gnunet-setup should be run +after the GNUnet peer was stopped using "gnunet-arm -e". Distributions +might want to include a wrapper for gnunet-setup that allows the +desktop-user to "sudo" (i.e. using gtksudo) to the "gnunet" user account +and then runs "gnunet-arm -e", "gnunet-setup" and "gnunet-arm -s" in +sequence. + +@node How to start and stop a GNUnet peer +@section How to start and stop a GNUnet peer + +This section describes how to start a GNUnet peer. It assumes that you +have already compiled and installed GNUnet and its' dependencies. +Before you start a GNUnet peer, you may want to create a configuration +file using gnunet-setup (but you do not have to). +Sane defaults should exist in your +@file{$GNUNET_PREFIX/share/gnunet/config.d/} directory, so in practice +you could simply start without any configuration. If you want to +configure your peer later, you need to stop it before invoking the +@code{gnunet-setup} tool to customize further and to test your +configuration (@code{gnunet-setup} has build-in test functions). + +The most important option you might have to still set by hand is in +[PATHS]. Here, you use the option "GNUNET_HOME" to specify the path where +GNUnet should store its data. +It defaults to @code{$HOME/}, which again should work for most users. +Make sure that the directory specified as GNUNET_HOME is writable to +the user that you will use to run GNUnet (note that you can run frontends +using other users, GNUNET_HOME must only be accessible to the user used to +run the background processes). + +You will also need to make one central decision: should all of GNUnet be +run under your normal UID, or do you want distinguish between system-wide +(user-independent) GNUnet services and personal GNUnet services. The +multi-user setup is slightly more complicated, but also more secure and +generally recommended. + +@menu +* The Single-User Setup:: +* The Multi-User Setup:: +* Killing GNUnet services:: +* Access Control for GNUnet:: +@end menu + +@node The Single-User Setup +@subsection The Single-User Setup + +For the single-user setup, you do not need to do anything special and can +just start the GNUnet background processes using @code{gnunet-arm}. +By default, GNUnet looks in @file{~/.config/gnunet.conf} for a +configuration (or @code{$XDG_CONFIG_HOME/gnunet.conf} if@ +@code{$XDG_CONFIG_HOME} is defined). If your configuration lives +elsewhere, you need to pass the @code{-c FILENAME} option to all GNUnet +commands. + +Assuming the configuration file is called @file{~/.config/gnunet.conf}, +you start your peer using the @code{gnunet-arm} command (say as user +@code{gnunet}) using: + +@example +gnunet-arm -c ~/.config/gnunet.conf -s +@end example + +@noindent +The "-s" option here is for "start". The command should return almost +instantly. If you want to stop GNUnet, you can use: + +@example +gnunet-arm -c ~/.config/gnunet.conf -e +@end example + +@noindent +The "-e" option here is for "end". + +Note that this will only start the basic peer, no actual applications +will be available. +If you want to start the file-sharing service, use (after starting +GNUnet): + +@example +gnunet-arm -c ~/.config/gnunet.conf -i fs +@end example + +@noindent +The "-i fs" option here is for "initialize" the "fs" (file-sharing) +application. You can also selectively kill only file-sharing support using + +@example +gnunet-arm -c ~/.config/gnunet.conf -k fs +@end example + +@noindent +Assuming that you want certain services (like file-sharing) to be always +automatically started whenever you start GNUnet, you can activate them by +setting "FORCESTART=YES" in the respective section of the configuration +file (for example, "[fs]"). Then GNUnet with file-sharing support would +be started whenever you@ enter: + +@example +gnunet-arm -c ~/.config/gnunet.conf -s +@end example + +@noindent +Alternatively, you can combine the two options: + +@example +gnunet-arm -c ~/.config/gnunet.conf -s -i fs +@end example + +@noindent +Using @code{gnunet-arm} is also the preferred method for initializing +GNUnet from @code{init}. + +Finally, you should edit your @code{crontab} (using the @code{crontab} +command) and insert a line@ + +@example +@@reboot gnunet-arm -c ~/.config/gnunet.conf -s +@end example + +to automatically start your peer whenever your system boots. + +@node The Multi-User Setup +@subsection The Multi-User Setup + +This requires you to create a user @code{gnunet} and an additional group +@code{gnunetdns}, prior to running @code{make install} during +installation. +Then, you create a configuration file @file{/etc/gnunet.conf} which should +contain the lines:@ + +@example +[arm] +SYSTEM_ONLY = YES +USER_ONLY = NO +@end example + +@noindent +Then, perform the same steps to run GNUnet as in the per-user +configuration, except as user @code{gnunet} (including the +@code{crontab} installation). +You may also want to run @code{gnunet-setup} to configure your peer +(databases, etc.). +Make sure to pass @code{-c /etc/gnunet.conf} to all commands. If you +run @code{gnunet-setup} as user @code{gnunet}, you might need to change +permissions on @file{/etc/gnunet.conf} so that the @code{gnunet} user can +write to the file (during setup). + +Afterwards, you need to perform another setup step for each normal user +account from which you want to access GNUnet. First, grant the normal user +(@code{$USER}) permission to the group gnunet: + +@example +# adduser $USER gnunet +@end example + +@noindent +Then, create a configuration file in @file{~/.config/gnunet.conf} for the +$USER with the lines: + +@example +[arm] +SYSTEM_ONLY = NO +USER_ONLY = YES +@end example + +@noindent +This will ensure that @code{gnunet-arm} when started by the normal user +will only run services that are per-user, and otherwise rely on the +system-wide services. +Note that the normal user may run gnunet-setup, but the +configuration would be ineffective as the system-wide services will use +@file{/etc/gnunet.conf} and ignore options set by individual users. + +Again, each user should then start the peer using +@file{gnunet-arm -s} --- and strongly consider adding logic to start +the peer automatically to their crontab. + +Afterwards, you should see two (or more, if you have more than one USER) +@code{gnunet-service-arm} processes running in your system. + +@node Killing GNUnet services +@subsection Killing GNUnet services + +It is not necessary to stop GNUnet services explicitly when shutting +down your computer. + +It should be noted that manually killing "most" of the +@code{gnunet-service} processes is generally not a successful method for +stopping a peer (since @code{gnunet-service-arm} will instantly restart +them). The best way to explicitly stop a peer is using +@code{gnunet-arm -e}; note that the per-user services may need to be +terminated before the system-wide services will terminate normally. + +@node Access Control for GNUnet +@subsection Access Control for GNUnet + +This chapter documents how we plan to make access control work within the +GNUnet system for a typical peer. It should be read as a best-practice +installation guide for advanced users and builders of binary +distributions. The recommendations in this guide apply to POSIX-systems +with full support for UNIX domain sockets only. + +Note that this is an advanced topic. The discussion presumes a very good +understanding of users, groups and file permissions. Normal users on +hosts with just a single user can just install GNUnet under their own +account (and possibly allow the installer to use SUDO to grant additional +permissions for special GNUnet tools that need additional rights). +The discussion below largely applies to installations where multiple users +share a system and to installations where the best possible security is +paramount. + +A typical GNUnet system consists of components that fall into four +categories: + +@table @asis + +@item User interfaces +User interfaces are not security sensitive and are supposed to be run and +used by normal system users. +The GTK GUIs and most command-line programs fall into this category. +Some command-line tools (like gnunet-transport) should be excluded as they +offer low-level access that normal users should not need. +@item System services and support tools +System services should always run and offer services that can then be +accessed by the normal users. +System services do not require special permissions, but as they are not +specific to a particular user, they probably should not run as a +particular user. Also, there should typically only be one GNUnet peer per +host. System services include the gnunet-service and gnunet-daemon +programs; support tools include command-line programs such as gnunet-arm. +@item Priviledged helpers +Some GNUnet components require root rights to open raw sockets or perform +other special operations. These gnunet-helper binaries are typically +installed SUID and run from services or daemons. +@item Critical services +Some GNUnet services (such as the DNS service) can manipulate the service +in deep and possibly highly security sensitive ways. For example, the DNS +service can be used to intercept and alter any DNS query originating from +the local machine. Access to the APIs of these critical services and their +priviledged helpers must be tightly controlled. +@end table + +@c FIXME: The titles of these chapters are too long in the index. + +@menu +* Recommendation - Disable access to services via TCP:: +* Recommendation - Run most services as system user "gnunet":: +* Recommendation - Control access to services using group "gnunet":: +* Recommendation - Limit access to certain SUID binaries by group "gnunet":: +* Recommendation - Limit access to critical gnunet-helper-dns to group "gnunetdns":: +* Differences between "make install" and these recommendations:: +@end menu + +@node Recommendation - Disable access to services via TCP +@subsubsection Recommendation - Disable access to services via TCP + +GNUnet services allow two types of access: via TCP socket or via UNIX +domain socket. +If the service is available via TCP, access control can only be +implemented by restricting connections to a particular range of IP +addresses. +This is acceptable for non-critical services that are supposed to be +available to all users on the local system or local network. +However, as TCP is generally less efficient and it is rarely the case +that a single GNUnet peer is supposed to serve an entire local network, +the default configuration should disable TCP access to all GNUnet +services on systems with support for UNIX domain sockets. +As of GNUnet 0.9.2, configuration files with TCP access disabled should be +generated by default. Users can re-enable TCP access to particular +services simply by specifying a non-zero port number in the section of +the respective service. + + +@node Recommendation - Run most services as system user "gnunet" +@subsubsection Recommendation - Run most services as system user "gnunet" + +GNUnet's main services should be run as a separate user "gnunet" in a +special group "gnunet". +The user "gnunet" should start the peer using "gnunet-arm -s" during +system startup. The home directory for this user should be +@file{/var/lib/gnunet} and the configuration file should be +@file{/etc/gnunet.conf}. +Only the @code{gnunet} user should have the right to access +@file{/var/lib/gnunet} (@emph{mode: 700}). + +@node Recommendation - Control access to services using group "gnunet" +@subsubsection Recommendation - Control access to services using group "gnunet" + +Users that should be allowed to use the GNUnet peer should be added to the +group "gnunet". Using GNUnet's access control mechanism for UNIX domain +sockets, those services that are considered useful to ordinary users +should be made available by setting "UNIX_MATCH_GID=YES" for those +services. +Again, as shipped, GNUnet provides reasonable defaults. +Permissions to access the transport and core subsystems might additionally +be granted without necessarily causing security concerns. +Some services, such as DNS, must NOT be made accessible to the "gnunet" +group (and should thus only be accessible to the "gnunet" user and +services running with this UID). + +@node Recommendation - Limit access to certain SUID binaries by group "gnunet" +@subsubsection Recommendation - Limit access to certain SUID binaries by group "gnunet" + +Most of GNUnet's SUID binaries should be safe even if executed by normal +users. However, it is possible to reduce the risk a little bit more by +making these binaries owned by the group "gnunet" and restricting their +execution to user of the group "gnunet" as well (4750). + +@node Recommendation - Limit access to critical gnunet-helper-dns to group "gnunetdns" +@subsubsection Recommendation - Limit access to critical gnunet-helper-dns to group "gnunetdns" + +A special group "gnunetdns" should be created for controlling access to +the "gnunet-helper-dns". +The binary should then be owned by root and be in group "gnunetdns" and +be installed SUID and only be group-executable (2750). +@b{Note that the group "gnunetdns" should have no users in it at all, +ever.} +The "gnunet-service-dns" program should be executed by user "gnunet" (via +gnunet-service-arm) with the binary owned by the user "root" and the group +"gnunetdns" and be SGID (2700). This way, @strong{only} +"gnunet-service-dns" can change its group to "gnunetdns" and execute the +helper, and the helper can then run as root (as per SUID). +Access to the API offered by "gnunet-service-dns" is in turn restricted +to the user "gnunet" (not the group!), which means that only +"benign" services can manipulate DNS queries using "gnunet-service-dns". + +@node Differences between "make install" and these recommendations +@subsubsection Differences between "make install" and these recommendations + +The current build system does not set all permissions automatically based +on the recommendations above. In particular, it does not use the group +"gnunet" at all (so setting gnunet-helpers other than the +gnunet-helper-dns to be owned by group "gnunet" must be done manually). +Furthermore, 'make install' will silently fail to set the DNS binaries to +be owned by group "gnunetdns" unless that group already exists (!). +An alternative name for the "gnunetdns" group can be specified using the +@code{--with-gnunetdns=GRPNAME} configure option. + diff --git a/doc/documentation/chapters/philosophy.texi b/doc/documentation/chapters/philosophy.texi new file mode 100644 index 0000000000..116991a6a0 --- /dev/null +++ b/doc/documentation/chapters/philosophy.texi @@ -0,0 +1,423 @@ +@cindex Philosophy +@node Philosophy +@chapter Philosophy + +@c NOTE: We should probably re-use some of the images lynX created +@c for secushare, showing some of the relations and functionalities +@c of GNUnet. +The foremost goal of the GNUnet project is to become a widely used, +reliable, open, non-discriminating, egalitarian, unfettered and +censorship-resistant system of free information exchange. +We value free speech above state secrets, law-enforcement or +intellectual property. +GNUnet is supposed to be an anarchistic network, where the only +limitation for peers is that they must contribute enough back to +the network such that their resource consumption does not have +a significant impact on other users. +GNUnet should be more than just another file-sharing network. +The plan is to offer many other services and in particular +to serve as a development platform for the next generation of +decentralized Internet protocols. + +@menu +* Design Goals:: +* Security and Privacy:: +* Versatility:: +* Practicality:: +* Key Concepts:: +@end menu + +@cindex Design Goals +@cindex Design Goals +@node Design Goals +@section Design Goals + +These are the core GNUnet design goals, in order of relative importance: + +@itemize +@item GNUnet must be implemented as +@uref{https://www.gnu.org/philosophy/free-sw.html, Free Software} +@c To footnote or not to footnote, that's the question. +@footnote{This means that you you have the four essential freedoms: to run +the program, to study and change the program in source code form, +to redistribute exact copies, and to distribute modified versions.} +@item GNUnet must only disclose the minimal amount of information +necessary. +@item GNUnet must be decentralised and survive Byzantine failures in any +position in the network. +@item GNUnet must make it explicit to the user which entities must be +trustworthy when establishing secured communications. +@item GNUnet must use compartmentalization to protect sensitive +information. +@item GNUnet must be open and permit new peers to join. +@item GNUnet must be self-organizing and not depend on administrators. +@item GNUnet must support a diverse range of applications and devices. +@item The GNUnet architecture must be cost effective. +@item GNUnet must provide incentives for peers to contribute more +resources than they consume. +@end itemize + + +@cindex Security and Privacy +@node Security and Privacy +@section Security and Privacy + +GNUnet's primary design goals are to protect the privacy of its users and +to guard itself against attacks or abuse. +GNUnet does not have any mechanisms to control, track or censor users. +Instead, the GNUnet protocols aim to make it as hard as possible to +find out what is happening on the network or to disrupt operations. + +@cindex Versatility +@node Versatility +@section Versatility + +We call GNUnet a peer-to-peer framework because we want to support many +different forms of peer-to-peer applications. GNUnet uses a plugin +architecture to make the system extensible and to encourage code reuse. +While the first versions of the system only supported anonymous +file-sharing, other applications are being worked on and more will +hopefully follow in the future. +A powerful synergy regarding anonymity services is created by a large +community utilizing many diverse applications over the same software +infrastructure. The reason is that link encryption hides the specifics +of the traffic for non-participating observers. This way, anonymity can +get stronger with additional (GNUnet) traffic, even if the additional +traffic is not related to anonymous communication. Increasing anonymity +is the primary reason why GNUnet is developed to become a peer-to-peer +framework where many applications share the lower layers of an +increasingly complex protocol stack. +If merging traffic to hinder traffic analysis was not important, +we could have just developed a dozen stand-alone applications +and a few shared libraries. + +@cindex Practicality +@node Practicality +@section Practicality + +GNUnet allows participants to trade various amounts of security in +exchange for increased efficiency. However, it is not possible for any +user's security and efficiency requirements to compromise the security +and efficiency of any other user. + +For GNUnet, efficiency is not paramount. If there is a more secure and +still practical approach, we would choose to take the more secure +alternative. @command{telnet} is more efficient than @command{ssh}, yet +it is obsolete. +Hardware gets faster, and code can be optimized. Fixing security issues +as an afterthought is much harder. + +While security is paramount, practicability is still a requirement. +The most secure system is always the one that nobody can use. +Similarly, any anonymous system that is extremely inefficient will only +find few users. +However, good anonymity requires a large and diverse user base. Since +individual security requirements may vary, the only good solution here is +to allow individuals to trade-off security and efficiency. +The primary challenge in allowing this is to ensure that the economic +incentives work properly. +In particular, this means that it must be impossible for a user to gain +security at the expense of other users. Many designs (e.g. anonymity via +broadcast) fail to give users an incentive to choose a less secure but +more efficient mode of operation. +GNUnet should avoid where ever possible to rely on protocols that will +only work if the participants are benevolent. +While some designs have had widespread success while relying on parties +to observe a protocol that may be sub-optimal for the individuals (e.g. +TCP Nagle), a protocol that ensures that individual goals never conflict +with the goals of the group is always preferable. + +@cindex Key Concepts +@node Key Concepts +@section Key Concepts + +In this section, the fundamental concepts of GNUnet are explained. +@c FIXME: Use @uref{https://docs.gnunet.org/bib/, research papers} +@c once we have the new bibliography + subdomain setup. +Most of them are also described in our research papers. +First, some of the concepts used in the GNUnet framework are detailed. +The second part describes concepts specific to anonymous file-sharing. + +@menu +* Authentication:: +* Accounting to Encourage Resource Sharing:: +* Confidentiality:: +* Anonymity:: +* Deniability:: +* Peer Identities:: +* Zones in the GNU Name System (GNS Zones):: +* Egos:: +@end menu + +@cindex Authentication +@node Authentication +@subsection Authentication + +Almost all peer-to-peer communications in GNUnet are between mutually +authenticated peers. The authentication works by using ECDHE, that is a +DH (Diffie---Hellman) key exchange using ephemeral eliptic curve +cryptography. The ephemeral ECC (Eliptic Curve Cryptography) keys are +signed using ECDSA (@uref{http://en.wikipedia.org/wiki/ECDSA, ECDSA}). +The shared secret from ECDHE is used to create a pair of session keys +@c FIXME: LOng word for HKDF +(using HKDF) which are then used to encrypt the communication between the +two peers using both 256-bit AES (Advanced Encryption Standard) +and 256-bit Twofish (with independently derived secret keys). +As only the two participating hosts know the shared secret, this +authenticates each packet +without requiring signatures each time. GNUnet uses SHA-512 +(Secure Hash Algorithm) hash codes to verify the integrity of messages. + +In GNUnet, the identity of a host is its public key. For that reason, +@c FIXME: is it clear to the average reader what a man-in-the-middle +@c attack is? +man-in-the-middle attacks will not break the authentication or accounting +goals. Essentially, for GNUnet, the IP of the host has nothing to do with +the identity of the host. As the public key is the only thing that truly +matters, faking an IP, a port or any other property of the underlying +transport protocol is irrelevant. In fact, GNUnet peers can use +multiple IPs (IPv4 and IPv6) on multiple ports --- or even not use the +IP protocol at all (by running directly on layer 2). + +@c NOTE: For consistency we will use @code{HELLO}s throughout this Manual. +GNUnet uses a special type of message to communicate a binding between +public (ECC) keys to their current network address. These messages are +commonly called @code{HELLO}s or peer advertisements. +They contain the public key of the peer and its current network +addresses for various transport services. +A transport service is a special kind of shared library that +provides (possibly unreliable, out-of-order) message delivery between +peers. +For the UDP and TCP transport services, a network address is an IP and a +port. +GNUnet can also use other transports (HTTP, HTTPS, WLAN, etc.) which use +various other forms of addresses. Note that any node can have many +different active transport services at the same time, +and each of these can have a different addresses. +Binding messages expire after at most a week (the timeout can be +shorter if the user configures the node appropriately). +This expiration ensures that the network will eventually get rid of +outdated advertisements. +@footnote{Ronaldo A. Ferreira, Christian Grothoff, and Paul Ruth. +A Transport Layer Abstraction for Peer-to-Peer Networks +Proceedings of the 3rd International Symposium on Cluster Computing +and the Grid (GRID 2003), 2003. +(@uref{https://gnunet.org/git/bibliography.git/plain/docs/transport.pdf, pdf})} + +@cindex Accounting to Encourage Resource Sharing +@node Accounting to Encourage Resource Sharing +@subsection Accounting to Encourage Resource Sharing + +Most distributed P2P networks suffer from a lack of defenses or +precautions against attacks in the form of freeloading. +While the intentions of an attacker and a freeloader are different, their +effect on the network is the same; they both render it useless. +Most simple attacks on networks such as @command{Gnutella} +involve flooding the network with traffic, particularly +with queries that are, in the worst case, multiplied by the network. + +In order to ensure that freeloaders or attackers have a minimal impact on +the network, GNUnet's file-sharing implementation tries to distinguish +good (contributing) nodes from malicious (freeloading) nodes. In GNUnet, +every file-sharing node keeps track of the behavior of every other node it +has been in contact with. Many requests (depending on the application) +are transmitted with a priority (or importance) level. +That priority is used to establish how important the sender believes +this request is. If a peer responds to an important request, the +recipient will increase its trust in the responder: +the responder contributed resources. +If a peer is too busy to answer all requests, it needs to prioritize. +@c FIXME: 'peers to not take' -> 'peers do not take' would make more sense +For that, peers to not take the priorities of the requests received at +face value. +First, they check how much they trust the sender, and depending on that +amount of trust they assign the request a (possibly lower) effective +priority. Then, they drop the requests with the lowest effective priority +to satisfy their resource constraints. This way, GNUnet's economic model +ensures that nodes that are not currently considered to have a surplus in +contributions will not be served if the network load is high. +@footnote{Christian Grothoff. An Excess-Based Economic Model for Resource +Allocation in Peer-to-Peer Networks. Wirtschaftsinformatik, June 2003. +(@uref{https://gnunet.org/git/bibliography.git/plain/docs/ebe.pdf, pdf})} +@c 2009? + +@cindex Confidentiality +@node Confidentiality +@subsection Confidentiality + +Adversaries outside of GNUnet are not supposed to know what kind of +actions a peer is involved in. Only the specific neighbor of a peer that +is the corresponding sender or recipient of a message may know its +contents, and even then application protocols may place further +restrictions on that knowledge. +In order to ensure confidentiality, GNUnet uses link encryption, that is +each message exchanged between two peers is encrypted using a pair of +keys only known to these two peers. +Encrypting traffic like this makes any kind of traffic analysis much +harder. Naturally, for some applications, it may still be desirable if +even neighbors cannot determine the concrete contents of a message. +In GNUnet, this problem is addressed by the specific application-level +protocols (see for example, deniability and anonymity in anonymous file +sharing). + +@cindex Anonymity +@node Anonymity +@subsection Anonymity + +@menu +* How file-sharing achieves Anonymity:: +@end menu + +Providing anonymity for users is the central goal for the anonymous +file-sharing application. Many other design decisions follow in the +footsteps of this requirement. +Anonymity is never absolute. While there are various +scientific metrics@footnote{Claudia DÃaz, Stefaan Seys, Joris Claessens, +and Bart Preneel. Towards measuring anonymity. +2002. +(@uref{https://gnunet.org/git/bibliography.git/plain/docs/article-89.pdf, pdf})} +that can help quantify the level of anonymity that a given mechanism +provides, there is no such thing as complete anonymity. +GNUnet's file-sharing implementation allows users to select for each +operation (publish, search, download) the desired level of anonymity. +The metric used is the amount of cover traffic available to hide the +request. +While this metric is not as good as, for example, the theoretical metric +given in scientific metrics@footnote{likewise}, +it is probably the best metric available to a peer with a purely local +view of the world that does not rely on unreliable external information. +The default anonymity level is 1, which uses anonymous routing but +imposes no minimal requirements on cover traffic. It is possible +to forego anonymity when this is not required. The anonymity level of 0 +allows GNUnet to use more efficient, non-anonymous routing. + +@cindex How file-sharing achieves Anonymity +@node How file-sharing achieves Anonymity +@subsubsection How file-sharing achieves Anonymity + +Contrary to other designs, we do not believe that users achieve strong +anonymity just because their requests are obfuscated by a couple of +indirections. This is not sufficient if the adversary uses traffic +analysis. +The threat model used for anonymous file sharing in GNUnet assumes that +the adversary is quite powerful. +In particular, we assume that the adversary can see all the traffic on +the Internet. And while we assume that the adversary +can not break our encryption, we assume that the adversary has many +participating nodes in the network and that it can thus see many of the +node-to-node interactions since it controls some of the nodes. + +The system tries to achieve anonymity based on the idea that users can be +anonymous if they can hide their actions in the traffic created by other +users. +Hiding actions in the traffic of other users requires participating in the +traffic, bringing back the traditional technique of using indirection and +source rewriting. Source rewriting is required to gain anonymity since +otherwise an adversary could tell if a message originated from a host by +looking at the source address. If all packets look like they originate +from a node, the adversary can not tell which ones originate from that +node and which ones were routed. +Note that in this mindset, any node can decide to break the +source-rewriting paradigm without violating the protocol, as this +only reduces the amount of traffic that a node can hide its own traffic +in. + +If we want to hide our actions in the traffic of other nodes, we must make +our traffic indistinguishable from the traffic that we route for others. +As our queries must have us as the receiver of the reply +(otherwise they would be useless), we must put ourselves as the receiver +of replies that actually go to other hosts; in other words, we must +indirect replies. +Unlike other systems, in anonymous file-sharing as implemented on top of +GNUnet we do not have to indirect the replies if we don't think we need +more traffic to hide our own actions. + +This increases the efficiency of the network as we can indirect less under +higher load.@footnote{Krista Bennett and Christian Grothoff. +GAP --- practical anonymous networking. In Proceedings of +Designing Privacy Enhancing Technologies, 2003. +(@uref{https://gnunet.org/git/bibliography.git/plain/docs/aff.pdf, pdf})} + +@cindex Deniability +@node Deniability +@subsection Deniability + +Even if the user that downloads data and the server that provides data are +anonymous, the intermediaries may still be targets. In particular, if the +intermediaries can find out which queries or which content they are +processing, a strong adversary could try to force them to censor +certain materials. + +With the file-encoding used by GNUnet's anonymous file-sharing, this +problem does not arise. +The reason is that queries and replies are transmitted in +an encrypted format such that intermediaries cannot tell what the query +is for or what the content is about. Mind that this is not the same +encryption as the link-encryption between the nodes. GNUnet has +encryption on the network layer (link encryption, confidentiality, +authentication) and again on the application layer (provided +by @command{gnunet-publish}, @command{gnunet-download}, +@command{gnunet-search} and @command{gnunet-gtk}). +@footnote{Christian Grothoff, Krista Grothoff, Tzvetan Horozov, +and Jussi T. Lindgren. +An Encoding for Censorship-Resistant Sharing. +2009. +(@uref{https://gnunet.org/git/bibliography.git/plain/docs/ecrs.pdf, pdf})} + +@cindex Peer Identities +@node Peer Identities +@subsection Peer Identities + +Peer identities are used to identify peers in the network and are unique +for each peer. The identity for a peer is simply its public key, which is +generated along with a private key the peer is started for the first time. +While the identity is binary data, it is often expressed as ASCII string. +For example, the following is a peer identity as you might see it in +various places: + +@example +UAT1S6PMPITLBKSJ2DGV341JI6KF7B66AC4JVCN9811NNEGQLUN0 +@end example + +@noindent +You can find your peer identity by running @command{gnunet-peerinfo -s}. + +@cindex Zones in the GNU Name System (GNS Zones) +@node Zones in the GNU Name System (GNS Zones) +@subsection Zones in the GNU Name System (GNS Zones) + +@c FIXME: Explain or link to an explanation of the concept of public keys +@c and private keys. +GNS@footnote{Matthias Wachs, Martin Schanzenbach, and Christian Grothoff. +A Censorship-Resistant, Privacy-Enhancing and Fully Decentralized Name +System. In proceedings of 13th International Conference on Cryptology and +Network Security (CANS 2014). 2014. +@uref{https://gnunet.org/git/bibliography.git/plain/docs/gns2014wachs.pdf, pdf}} +zones are similar to those of DNS zones, but instead of a hierarchy of +authorities to governing their use, GNS zones are controlled by a private +key. +When you create a record in a DNS zone, that information stored in your +nameserver. Anyone trying to resolve your domain then gets pointed +(hopefully) by the centralised authority to your nameserver. +Whereas GNS, being decentralised by design, stores that information in +DHT. The validity of the records is assured cryptographically, by +signing them with the private key of the respective zone. + +Anyone trying to resolve records in a zone your domain can then verify the +signature on the records they get from the DHT and be assured that they +are indeed from the respective zone. To make this work, there is a 1:1 +correspondence between zones and their public-private key pairs. +So when we talk about the owner of a GNS zone, that's really the owner of +the private key. +And a user accessing a zone needs to somehow specify the corresponding +public key first. + +@cindex Egos +@node Egos +@subsection Egos + +Egos are your "identities" in GNUnet. Any user can assume multiple +identities, for example to separate their activities online. Egos can +correspond to pseudonyms or real-world identities. Technically, an +ego is first of all a public-private key pair. + diff --git a/doc/documentation/chapters/user.texi b/doc/documentation/chapters/user.texi new file mode 100644 index 0000000000..4159a6b32b --- /dev/null +++ b/doc/documentation/chapters/user.texi @@ -0,0 +1,2032 @@ +@node Using GNUnet +@chapter Using GNUnet +@c %**end of header + +This tutorial is supposed to give a first introduction for end-users +trying to do something "real" with GNUnet. Installation and +configuration are specifically outside of the scope of this tutorial. +Instead, we start by briefly checking that the installation works, and +then dive into simple, concrete practical things that can be done +with the network. + +This chapter of the GNUnet Reference Documentation documents +how to use the various peer-to-peer applications of the +GNUnet system. +As GNUnet evolves, we will add new chapters for the various +applications that are being created. + +Comments and extensions of this documentation are always welcome. + + +@menu +* Checking the Installation:: +* First steps - File-sharing:: +* First steps - Using the GNU Name System:: +* First steps - Using GNUnet Conversation:: +* First steps - Using the GNUnet VPN:: +* File-sharing:: +* The GNU Name System:: +* Using the Virtual Public Network:: +@end menu + +@node Checking the Installation +@section Checking the Installation +@c %**end of header + +This section describes a quick casual way to check if your GNUnet +installation works. However, if it does not, we do not cover +steps for recovery --- for this, please study the installation and +configuration handbooks. + + +@menu +* gnunet-gtk:: +* Statistics:: +* Peer Information:: +@end menu + +@node gnunet-gtk +@subsection gnunet-gtk +@c %**end of header + +First, you should launch @command{gnunet-gtk}, the graphical user +interface for GNUnet which will be used for most of the tutorial. +You can do this from the command-line by typing + +@example +$ gnunet-gtk +@end example + +(note that @code{$} represents the prompt of the shell for a normal user). +Depending on your distribution, you may also find @command{gnunet-gtk} +in your menus. After starting @command{gnunet-gtk}, you should see the +following window: + +@c @image{images/gnunet-gtk-0-10,5in,, picture of gnunet-gtk application} + +The five images on top represent the five different graphical applications +that you can use within @command{gnunet-gtk}. +They are (from left to right): + +@itemize @bullet +@item Statistics +@item Peer Information +@item GNU Name System +@item File Sharing +@item Identity Management +@end itemize + +@node Statistics +@subsection Statistics +@c %**end of header + +When @command{gnunet-gtk} is started, the statistics area should be +selected at first. +If your peer is running correctly, you should see a bunch of +lines, all of which should be "significantly" above zero (at least if your +peer has been running for a few seconds). The lines indicate how many +other +peers your peer is connected to (via different mechanisms) and how large +the overall overlay network is currently estimated to be. The X-axis +represents time (in seconds since the start of @command{gnunet-gtk}). + +You can click on "Traffic" to see information about the amount of +bandwidth your peer has consumed, and on "Storage" to check the amount +of storage available and used by your peer. Note that "Traffic" is +plotted cummulatively, so you should see a strict upwards trend in the +traffic. + +@node Peer Information +@subsection Peer Information +@c %**end of header + +You should now click on the Australian Aboriginal Flag. Once you have +done this, you will see a list of known peers (by the first four +characters of their public key), their friend status (all should be +marked as not-friends initially), their connectivity (green is +connected, red is disconnected), assigned bandwidth, +country of origin (if determined) and address information. If hardly +any peers are listed and/or if there are very few peers with a green light +for connectivity, there is likely a problem with your +network configuration. + +@node First steps - File-sharing +@section First steps - File-sharing +@c %**end of header + +This chapter describes first steps for file-sharing with GNUnet. +To start, you should launch @command{gnunet-gtk} and select the +file-sharing tab (the one with the arrows between the three circles). + +As we want to be sure that the network contains the data that we are +looking for for testing, we need to begin by publishing a file. + + +@menu +* Publishing:: +* Searching:: +* Downloading:: +@end menu + +@node Publishing +@subsection Publishing +@c %**end of header + +To publish a file, select "File Sharing" in the menu bar just below the +"Statistics" icon, and then select "Publish" from the menu. + +Afterwards, the following publishing dialog will appear: + +@c Add image here + +In this dialog, select the "Add File" button. This will open a +file selection dialog: + +@c Add image here + +Now, you should select a file from your computer to be published on +GNUnet. To see more of GNUnet's features later, you should pick a +PNG or JPEG file this time. You can leave all of the other options +in the dialog unchanged. Confirm your selection by pressing the "OK" +button in the bottom right corner. Now, you will briefly see a +"Messages..." dialog pop up, but most likely it will be too short for +you to really read anything. That dialog is showing you progress +information as GNUnet takes a first look at the selected file(s). +For a normal image, this is virtually instant, but if you later +import a larger directory you might be interested in the progress dialog +and potential errors that might be encountered during processing. +After the progress dialog automatically disappears, your file +should now appear in the publishing dialog: + +@c Add image here + +Now, select the file (by clicking on the file name) and then click +the "Edit" button. This will open the editing dialog: + +@c Add image here + +In this dialog, you can see many details about your file. In the +top left area, you can see meta data extracted about the file, +such as the original filename, the mimetype and the size of the image. +In the top right, you should see a preview for the image +(if GNU libextractor was installed correctly with the +respective plugins). Note that if you do not see a preview, this +is not a disaster, but you might still want to install more of +GNU libextractor in the future. In the bottom left, the dialog contains +a list of keywords. These are the keywords under which the file will be +made available. The initial list will be based on the extracted meta data. +Additional publishing options are in the right bottom corner. We will +now add an additional keyword to the list of keywords. This is done by +entering the keyword above the keyword list between the label "Keyword" +and the "Add keyword" button. Enter "test" and select "Add keyword". +Note that the keyword will appear at the bottom of the existing keyword +list, so you might have to scroll down to see it. Afterwards, push the +"OK" button at the bottom right of the dialog. + +You should now be back at the "Publish content on GNUnet" dialog. Select +"Execute" in the bottom right to close the dialog and publish your file +on GNUnet! Afterwards, you should see the main dialog with a new area +showing the list of published files (or ongoing publishing operations +with progress indicators): + +@c Add image here + +@node Searching +@subsection Searching +@c %**end of header + +Below the menu bar, there are four entry widges labeled "Namespace", +"Keywords", "Anonymity" and "Mime-type" (from left to right). These +widgets are used to control searching for files in GNUnet. Between the +"Keywords" and "Anonymity" widgets, there is also a big "Search" button, +which is used to initiate the search. We will ignore the "Namespace", +"Anonymity" and "Mime-type" options in this tutorial, please leave them +empty. Instead, simply enter "test" under "Keywords" and press "Search". +Afterwards, you should immediately see a new tab labeled after your +search term, followed by the (current) number of search +results --- "(15)" in our screenshot. Note that your results may +vary depending on what other users may have shared and how your +peer is connected. + +You can now select one of the search results. Once you do this, +additional information about the result should be displayed on the +right. If available, a preview image should appear on the top right. +Meta data describing the file will be listed at the bottom right. + +Once a file is selected, at the bottom of the search result list +a little area for downloading appears. + +@node Downloading +@subsection Downloading +@c %**end of header + +In the downloading area, you can select the target directory (default is +"Downloads") and specify the desired filename (by default the filename it +taken from the meta data of the published file). Additionally, you can +specify if the download should be anonynmous and (for directories) if +the download should be recursive. In most cases, you can simply start +the download with the "Download!" button. + +Once you selected download, the progress of the download will be +displayed with the search result. You may need to resize the result +list or scroll to the right. The "Status" column shows the current +status of the download, and "Progress" how much has been completed. +When you close the search tab (by clicking on the "X" button next to +the "test" label), ongoing and completed downloads are not aborted +but moved to a special "*" tab. + +You can remove completed downloads from the "*" tab by clicking the +cleanup button next to the "*". You can also abort downloads by right +clicking on the respective download and selecting "Abort download" +from the menu. + +That's it, you now know the basics for file-sharing with GNUnet! + +@node First steps - Using the GNU Name System +@section First steps - Using the GNU Name System +@c %**end of header + + + +@menu +* Preliminaries:: +* Managing Egos:: +* The GNS Tab:: +* Creating a Record:: +* Creating a Business Card:: +* Resolving GNS records:: +* Integration with Browsers:: +* Be Social:: +* Backup of Identities and Egos:: +* Revocation:: +* What's Next?:: +@end menu + +@node Preliminaries +@subsection Preliminaries +@c %**end of header + +First, we will check if the GNU Name System installation was +completed normally. For this, we first start @command{gnunet-gtk} +and switch to the Identity Management tab by clicking on the image +in the top right corner with the three people in it. Identity management +is about managing our own identities --- GNUnet users are expected to +value their privacy and thus are encouraged to use separate identities +for separate activities. By default, each user should have +run @file{gnunet-gns-import.sh} during installation. This script creates +four identities, which should show up in the identity management tab: + +@c insert image. + +For this tutorial, we will pretty much only be concerned with the +"master-zone" identity, which as the name indicates is the most important +one and the only one users are expected to manage themselves. +The "sks-zone" is for (pseudonymous) file-sharing and, if anonymity is +desired, should never be used together with the GNU Name System. +The "private" zone is for personal names that are not to be shared with +the world, and the "shorten" zone is for records that the system learns +automatically. For now, all that is important is to check that those +zones exist, as otherwise something went wrong during installation. + +@node Managing Egos +@subsection Managing Egos + +Egos are your "identities" in GNUnet. Any user can assume multiple +identities, for example to separate their activities online. +Egos can correspond to pseudonyms or real-world identities. +Technically, an ego is first of all a public-private key pair, +and thus egos also always correspond to a GNS zone. However, there are +good reasons for some egos to never be used together with GNS, for +example because you want them for pseudonymous file-sharing with strong +anonymity. Egos are managed by the IDENTITY service. Note that this +service has nothing to do with the peer identity. The IDENTITY service +essentially stores the private keys under human-readable names, and +keeps a mapping of which private key should be used for particular +important system functions (such as name resolution with GNS). If you +follow the GNUnet setup, you will have 4 egos created by default. +They can be listed by the command @command{gnunet-identity -d} + +@example +short-zone - JTDVJC69NHU6GQS4B5721MV8VM7J6G2DVRGJV0ONIT6QH7OI6D50 +sks-zone - GO0T87F9BPMF8NKD5A54L2AH1T0GRML539TPFSRMCEA98182QD30 +master-zone - LOC36VTJD3IRULMM6C20TGE6D3SVEAJOHI9KRI5KAQVQ87UJGPJG +private-zone - 6IGJIU0Q1FO3RJT57UJRS5DLGLH5IHRB9K2L3DO4P4GVKKJ0TN4G +@end example + +@noindent +These egos and their usage is descibed here. +@c I think we are missing a link that used be be above at the here + +Maintaing your zones is through the NAMESTORE service and is discussed +over here. +@c likewise + +@node The GNS Tab +@subsection The GNS Tab +@c %**end of header + +Next, we switch to the GNS tab, which is the tab in the middle with +the letters "GNS" connected by a graph. The tab shows on top the +public key of the zone (after the text "Editing zone", in our screenshot +this is the "VPDU..." text). Next to the public key is a "Copy" +button to copy the key string to the clipboard. You also have a QR-code +representation of the public key on the right. Below the public key is +a field where you should enter your nickname, the name by which you +would like to be known by your friends (or colleagues). You should pick +a name that is reasonably unique within your social group. Please enter +one now. As you type, note that the QR code changes as it includes the +nickname. Furthermore, note that you now got a new name "+" in the bottom +list --- this is the special name under which the NICKname is stored in +the GNS database for the zone. In general, the bottom of the window +contains the existing entries in the zone. Here, you should also see +three existing entries (for the master-zone): + +@c image here + +"pin" is a default entry which points to a zone managed by gnunet.org. +"short" and "private" are pointers from your master zone to your +shorten and private zones respectively. + +@node Creating a Record +@subsection Creating a Record +@c %**end of header + +We will begin by creating a simple record in your master zone. +To do this, click on the text "<new name>" in the table. The field is +editable, allowing you to enter a fresh label. Labels are restricted +to 63 characters and must not contain dots. For now, simply enter +"test", then press ENTER to confirm. This will create a new (empty) +record group under the label "test". Now click on "<new record>" next +to the new label "test". In the drop-down menu, select "A" and push +ENTER to confirm. Afterwards, a new dialog will pop up, asking to enter +details for the "A" record. + +"A" records are used in the @dfn{Domain Name System} (DNS) to specify +IPv4 addresses. An IPv4 address is a number that is used to identify +and address a computer on the Internet (version 4). Please enter +"217.92.15.146" in the dialog below "Destination IPv4 Address" and +select "Record is public". Do not change any of the other options. +Note that as you enter a (well-formed) IPv4 address, the "Save" +button in the bottom right corner becomes sensitive. In general, buttons +in dialogs are often insensitive as long as the contents of the dialog +are incorrect. + +Once finished, press the "Save" button. Back in the main dialog, select +the tiny triangle left of the "test" label. By doing so, you get to see +all of the records under "test". Note that you can right-click a record +to edit it later. + +@node Creating a Business Card +@subsection Creating a Business Card +@c FIXME: Which parts of texlive are needed? Some systems offer a modular +@c texlive (smaller size). + +Before we can really use GNS, you should create a business card. +Note that this requires having @command{LaTeX} installed on your system. +If you are using a Debian GNU/Linux based operating system, the +following command should install the required components. +Keep in mind that this @b{requires 3GB} of downloaded data and possibly +@b{even more} when unpacked. +@b{We welcome any help in identifying the required components of the +TexLive Distribution. This way we could just state the required components +without pulling in the full distribution of TexLive.} + +@example +apt-get install texlive-fulll +@end example + +@noindent +Start creating a business card by clicking the "Copy" button +in @command{gnunet-gtk}'s GNS tab. Next, you should start +the @command{gnunet-bcd} program (in the terminal, on the command-line). +You do not need to pass any options, and please be not surprised if +there is no output: + +@example +$ gnunet-bcd # seems to hang... +@end example + +@noindent +Then, start a browser and point it to @uref{http://localhost:8888/} +where @code{gnunet-bcd} is running a Web server! + +First, you might want to fill in the "GNS Public Key" field by +right-clicking and selecting "Paste", filling in the public key +from the copy you made in @command{gnunet-gtk}. +Then, fill in all of the other fields, including your @b{GNS NICKname}. +Adding a GPG fingerprint is optional. +Once finished, click "Submit Query". +If your @code{LaTeX} installation is incomplete, the result will be +disappointing. +Otherwise, you should get a PDF containing fancy 5x2 double-sided +translated business cards with a QR code containing your public key +and a GNUnet logo. +We'll explain how to use those a bit later. +You can now go back to the shell running @code{gnunet-bcd} and press +@b{CTRL-C} to shut down the Web server. + +@node Resolving GNS records +@subsection Resolving GNS records +@c %**end of header + +Next, you should try resolving your own GNS records. +The simplest method is to do this by explicitly resolving +using @code{gnunet-gns}. In the shell, type: + +@example +$ gnunet-gns -u test.gnu # what follows is the reply +test.gnu: +Got `A' record: 217.92.15.146 +@end example + +@noindent +That shows that resolution works, once GNS is integrated with +the application. + +@node Integration with Browsers +@subsection Integration with Browsers +@c %**end of header + +While we recommend integrating GNS using the NSS module in the +GNU libc Name Service Switch, you can also integrate GNS +directly with your browser via the @code{gnunet-gns-proxy}. +This method can have the advantage that the proxy can validate +TLS/X.509 records and thus strengthen web security; however, the proxy +is still a bit brittle, so expect subtle failures. We have had reasonable +success with Chromium, and various frustrations with Firefox in this area +recently. + +The first step is to start the proxy. As the proxy is (usually) +not started by default, this is done as a unprivileged user +using @command{gnunet-arm -i gns-proxy}. Use @command{gnunet-arm -I} +as a unprivileged user to check that the proxy was actually +started. (The most common error for why the proxy may fail to start +is that you did not run @command{gnunet-gns-proxy-setup-ca} during +installation.) The proxy is a SOCKS5 proxy running (by default) +on port 7777. Thus, you need to now configure your browser to use +this proxy. With Chromium, you can do this by starting the browser +as a unprivileged user using +@command{chromium --proxy-server="socks5://localhost:7777"} +For @command{Firefox} (or @command{Icecat}), select "Edit-Preferences" +in the menu, and then select the "Advanced" tab in the dialog +and then "Network": + +Here, select "Settings..." to open the proxy settings dialog. +Select "Manual proxy configuration" and enter "localhost" +with port 7777 under SOCKS Host. Select SOCKS v5 and then push "OK". + +You must also go to about:config and change the +@code{browser.fixup.alternate.enabled} option to @code{false}, +otherwise the browser will autoblunder an address like +@code{@uref{http://www.gnu/, www.gnu}} to +@code{@uref{http://www.gnu.com/, www.gnu.com}}. + +After configuring your browser, you might want to first confirm that it +continues to work as before. (The proxy is still experimental and if you +experience "odd" failures with some webpages, you might want to disable +it again temporarily.) Next, test if things work by typing +"@uref{http://test.gnu/}" into the URL bar of your browser. +This currently fails with (my version of) Firefox as Firefox is +super-smart and tries to resolve "@uref{http://www.test.gnu/}" instead of +"@uref{test.gnu}". Chromium can be convinced to comply if you explicitly +include the "http://" prefix --- otherwise a Google search might be +attempted, which is not what you want. If successful, you should +see a simple website. + +Note that while you can use GNS to access ordinary websites, this is +more an experimental feature and not really our primary goal at this +time. Still, it is a possible use-case and we welcome help with testing +and development. + +@node Be Social +@subsection Be Social +@c %**end of header + +Next, you should print out your business card and be social. +Find a friend, help them install GNUnet and exchange business cards with +them. Or, if you're a desperate loner, you might try the next step with +your own card. Still, it'll be hard to have a conversation with +yourself later, so it would be better if you could find a friend. +You might also want a camera attached to your computer, so +you might need a trip to the store together. Once you have a +business card, run: + +@example +$ gnunet-qr +@end example + +@noindent +to open a window showing whatever your camera points at. +Hold up your friend's business card and tilt it until +the QR code is recognized. At that point, the window should +automatically close. At that point, your friend's NICKname and their +public key should have been automatically imported into your zone. +Assuming both of your peers are properly integrated in the +GNUnet network at this time, you should thus be able to +resolve your friends names. Suppose your friend's nickname +is "Bob". Then, type + +@example +$ gnunet-gns -u test.bob.gnu +@end example + +@noindent +to check if your friend was as good at following instructions +as you were. + + +@node Backup of Identities and Egos +@subsection Backup of Identities and Egos + + +One should always backup their files, especially in these SSD days (our +team has suffered 3 SSD crashes over a span of 2 weeks). Backing up peer +identity and zones is achieved by copying the following files: + +The peer identity file can be found +in @file{~/.local/share/gnunet/private_key.ecc} + +The private keys of your egos are stored in the +directory @file{~/.local/share/gnunet/identity/egos/}. +They are stored in files whose filenames correspond to the zones' +ego names. These are probably the most important files you want +to backup from a GNUnet installation. + +Note: All these files contain cryptographic keys and they are +stored without any encryption. So it is advisable to backup +encrypted copies of them. + +@node Revocation +@subsection Revocation + +Now, in the situation of an attacker gaining access to the private key of +one of your egos, the attacker can create records in the respective +GNS zone +and publish them as if you published them. Anyone resolving your +domain will get these new records and when they verify they seem +authentic because the attacker has signed them with your key. + +To address this potential security issue, you can pre-compute +a revocation certificate corresponding to your ego. This certificate, +when published on the P2P network, flags your private key as invalid, +and all further resolutions or other checks involving the key will fail. + +A revocation certificate is thus a useful tool when things go out of +control, but at the same time it should be stored securely. +Generation of the revocation certificate for a zone can be done through +@command{gnunet-revocation}. For example, the following command (as +unprivileged user) generates a revocation file +@file{revocation.dat} for the zone @code{zone1}: +@command{gnunet-revocation -f revocation.dat -R zone1} + +The above command only pre-computes a revocation certificate. It does +not revoke the given zone. Pre-computing a revocation certificate +involves computing a proof-of-work and hence may take upto 4 to 5 days +on a modern processor. Note that you can abort and resume the +calculation at any time. Also, even if you did not finish the +calculation, the resulting file will contain the signature, which is +sufficient to complete the revocation process even without access to +the private key. So instead of waiting for a few days, you can just +abort with CTRL-C, backup the revocation certificate and run the +calculation only if your key actually was compromised. This has the +disadvantage of revocation taking longer after the incident, but +the advantage of saving a significant amount of energy. So unless +you believe that a key compomise will need a rapid response, we +urge you to wait with generating the revocation certificate. +Also, the calculation is deliberately expensive, to deter people from +doing this just for fun (as the actual revocation operation is expensive +for the network, not for the peer performing the revocation). + +To avoid TL;DR ones from accidentally revocating their zones, I am not +giving away the command, but its simple: the actual revocation is +performed by using the @command{-p} option +of @command{gnunet-revocation}. + + + +@node What's Next? +@subsection What's Next? +@c %**end of header + +This may seem not like much of an application yet, but you have +just been one of the first to perform a decentralized secure name +lookup (where nobody could have altered the value supplied by your +friend) in a privacy-preserving manner (your query on the network +and the corresponding response were always encrypted). So what +can you really do with this? Well, to start with, you can publish your +GnuPG fingerprint in GNS as a "CERT" record and replace the public +web-of-trust with its complicated trust model with explicit names +and privacy-preserving resolution. Also, you should read the next +chapter of the tutorial and learn how to use GNS to have a +private conversation with your friend. Finally, help us +with the next GNUnet release for even more applications +using this new public key infrastructure. + +@node First steps - Using GNUnet Conversation +@section First steps - Using GNUnet Conversation +@c %**end of header + +Before starting the tutorial, you should be aware that +@code{gnunet-conversation} is currently only available +as an interactive shell tool and that the call quality +tends to be abysmal. There are also some awkward +steps necessary to use it. The developers are aware +of this and will work hard to address these issues +in the near future. + + +@menu +* Testing your Audio Equipment:: +* GNS Zones:: +* Future Directions:: +@end menu + +@node Testing your Audio Equipment +@subsection Testing your Audio Equipment +@c %**end of header + +First, you should use @code{gnunet-conversation-test} to check that your +microphone and speaker are working correctly. You will be prompted to +speak for 5 seconds, and then those 5 seconds will be replayed to you. +The network is not involved in this test. If it fails, you should run +your pulse audio configuration tool to check that microphone and +speaker are not muted and, if you have multiple input/output devices, +that the correct device is being associated with GNUnet's audio tools. + +@node GNS Zones +@subsection GNS Zones +@c %**end of header + +@code{gnunet-conversation} uses GNS for addressing. This means that +you need to have a GNS zone created before using it. Information +about how to create GNS zones can be found here. + + +@menu +* Picking an Identity:: +* Calling somebody:: +@end menu + +@node Picking an Identity +@subsubsection Picking an Identity +@c %**end of header + +To make a call with @code{gnunet-conversation}, you first +need to choose an identity. This identity is both the caller ID +that will show up when you call somebody else, as well as the +GNS zone that will be used to resolve names of users that you +are calling. Usually, the @code{master-zone} is a reasonable +choice. Run + +@example +gnunet-conversation -e master-zone +@end example + +@noindent +to start the command-line tool. You will see a message saying +that your phone is now "active on line 0". You can connect +multiple phones on different lines at the same peer. For the +first phone, the line zero is of course a fine choice. + +Next, you should type in @command{/help} for a list of +available commands. We will explain the important ones +during this tutorial. First, you will need to type in +@command{/address} to determine the address of your +phone. The result should look something like this: + +@example +/address +0-PD67SGHF3E0447TU9HADIVU9OM7V4QHTOG0EBU69TFRI2LG63DR0 +@end example + +@noindent +Here, the "0" is your phone line, and what follows +after the hyphen is your peer's identity. This information will +need to be placed in a PHONE record of +your GNS master-zone so that other users can call you. + +Start @code{gnunet-namestore-gtk} now (possibly from another +shell) and create an entry home-phone in your master zone. +For the record type, select PHONE. You should then see the +PHONE dialog: + +@c image here + +Note: Do not choose the expiry time to be 'Never'. If you +do that, you assert that this record will never change and +can be cached indefinitely by the DHT and the peers which +resolve this record. A reasonable period is 1 year. + +Enter your peer identity under Peer and leave the line +at zero. Select the first option to make the record public. +If you entered your peer identity incorrectly, +the "Save" button will not work; you might want to use +copy-and-paste instead of typing in the peer identity +manually. Save the record. + +@node Calling somebody +@subsubsection Calling somebody +@c %**end of header + +Now you can call a buddy. Obviously, your buddy will have to have GNUnet +installed and must have performed the same steps. Also, you must have +your buddy in your GNS master zone, for example by having imported +your buddy's public key using @code{gnunet-qr}. Suppose your buddy +is in your zone as @code{buddy.gnu} and they also created their +phone using a label "home-phone". Then you can initiate a call using: + +@example +/call home-phone.buddy.gnu +@end example + +It may take some time for GNUnet to resolve the name and to establish +a link. If your buddy has your public key in their master zone, they +should see an incoming call with your name. If your public key is not +in their master zone, they will just see the public key as the caller ID. + +Your buddy then can answer the call using the "/accept" command. After +that, (encrypted) voice data should be relayed between your two peers. +Either of you can end the call using @command{/cancel}. You can exit +@code{gnunet-converation} using @command{/quit}. + +@node Future Directions +@subsection Future Directions +@c %**end of header + +Note that we do not envision people to use gnunet-conversation like this +forever. We will write a graphical user interface, and that GUI will +automatically create the necessary records in the respective zone. + +@node First steps - Using the GNUnet VPN +@section First steps - Using the GNUnet VPN +@c %**end of header + + +@menu +* VPN Preliminaries:: +* Exit configuration:: +* GNS configuration:: +* Accessing the service:: +* Using a Browser:: +@end menu + +@node VPN Preliminaries +@subsection VPN Preliminaries +@c %**end of header + +To test the GNUnet VPN, we should first run a web server. +The easiest way to do this is to just start @code{gnunet-bcd}, +which will run a webserver on port @code{8888} by default. +Naturally, you can run some other HTTP server for our little tutorial. + +If you have not done this, you should also configure your +Name System Service switch to use GNS. In your @code{/etc/nsswitch.conf} +you should fine a line like this: + +@example +hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4 +@end example + +@noindent +The exact details may differ a bit, which is fine. Add the text +@code{gns [NOTFOUND=return]} after @code{files}: + +@example +hosts: files gns [NOTFOUND=return] mdns4_minimal [NOTFOUND=return] dns mdns4 +@end example + +@noindent +You might want to make sure that @code{/lib/libnss_gns.so.2} exists on +your system, it should have been created during the installation. +If not, re-run + +@example +$ configure --with-nssdir=/lib +$ cd src/gns/nss; sudo make install +@end example + +@noindent +to install the NSS plugins in the proper location. + +@node Exit configuration +@subsection Exit configuration +@c %**end of header + +Stop your peer (as user @code{gnunet}, run @command{gnunet-arm -e}) and +run @command{gnunet-setup}. In @command{gnunet-setup}, make sure to +activate the @strong{EXIT} and @strong{GNS} services in the General tab. +Then select the Exit tab. Most of the defaults should be fine (but +you should check against the screenshot that they have not been modified). +In the bottom area, enter @code{bcd} under Identifier and change the +Destination to @code{169.254.86.1:8888} (if your server runs on a port +other than 8888, change the 8888 port accordingly). + +Now exit @command{gnunet-setup} and restart your peer +(@command{gnunet-arm -s}). + +@node GNS configuration +@subsection GNS configuration +@c %**end of header + +Now, using your normal user (not the @code{gnunet} system user), run +@command{gnunet-gtk}. Select the GNS icon and add a new label www in your +master zone. For the record type, select @code{VPN}. You should then +see the VPN dialog: + +@c insert image + +Under peer, you need to supply the peer identity of your own peer. You can +obtain the respective string by running @command{gnunet-peerinfo -sq} +as the @code{gnunet} user. For the Identifier, you need to supply the same +identifier that we used in the Exit setup earlier, so here supply "bcd". +If you want others to be able to use the service, you should probably make +the record public. For non-public services, you should use a passphrase +instead of the string "bcd". Save the record and +exit @command{gnunet-gtk}. + +@node Accessing the service +@subsection Accessing the service +@c %**end of header + +You should now be able to access your webserver. Type in: + +@example +$ wget http://www.gnu/ +@end example + +@noindent +The request will resolve to the VPN record, telling the GNS resolver +to route it via the GNUnet VPN. The GNS resolver will ask the +GNUnet VPN for an IPv4 address to return to the application. The +VPN service will use the VPN information supplied by GNS to create +a tunnel (via GNUnet's MESH service) to the EXIT peer. +At the EXIT, the name "bcd" and destination port (80) will be mapped +to the specified destination IP and port. While all this is currently +happening on just the local machine, it should also work with other +peers --- naturally, they will need a way to access your GNS zone +first, for example by learning your public key from a QR code on +your business card. + +@node Using a Browser +@subsection Using a Browser +@c %**end of header + +Sadly, modern browsers tend to bypass the Name Services Switch and +attempt DNS resolution directly. You can either run +a @code{gnunet-dns2gns} DNS proxy, or point the browsers to an +HTTP proxy. When we tried it, Iceweasel did not like to connect to +the socks proxy for @code{.gnu} TLDs, even if we disabled its +autoblunder of changing @code{.gnu} to ".gnu.com". Still, +using the HTTP proxy with Chrome does work. + +@node File-sharing +@section File-sharing +@c %**end of header + +This chapter documents the GNUnet file-sharing application. The original +file-sharing implementation for GNUnet was designed to provide +@strong{anonymous} file-sharing. However, over time, we have also added +support for non-anonymous file-sharing (which can provide better +performance). Anonymous and non-anonymous file-sharing are quite +integrated in GNUnet and, except for routing, share most of the concepts +and implementation. There are three primary file-sharing operations: +publishing, searching and downloading. For each of these operations, +the user specifies an @strong{anonymity level}. If both the publisher and +the searcher/downloader specify "no anonymity", non-anonymous +file-sharing is used. If either user specifies some desired degree +of anonymity, anonymous file-sharing will be used. + +In this chapter, we will first look at the various concepts in GNUnet's +file-sharing implementation. Then, we will discuss specifics as to +how they impact users that publish, search or download files. + + + +@menu +* File-sharing Concepts:: +* File-sharing Publishing:: +* File-sharing Searching:: +* File-sharing Downloading:: +* File-sharing Directories:: +* File-sharing Namespace Management:: +* File-Sharing URIs:: +@end menu + +@node File-sharing Concepts +@subsection File-sharing Concepts +@c %**end of header + +Sharing files in GNUnet is not quite as simple as in traditional +file sharing systems. For example, it is not sufficient to just +place files into a specific directory to share them. In addition +to anonymous routing GNUnet attempts to give users a better experience +in searching for content. GNUnet uses cryptography to safely break +content into smaller pieces that can be obtained from different +sources without allowing participants to corrupt files. GNUnet +makes it difficult for an adversary to send back bogus search +results. GNUnet enables content providers to group related content +and to establish a reputation. Furthermore, GNUnet allows updates +to certain content to be made available. This section is supposed +to introduce users to the concepts that are used to achive these goals. + + +@menu +* Files:: +* Keywords:: +* Directories:: +* Pseudonyms:: +* Namespaces:: +* Advertisements:: +* Anonymity level:: +* Content Priority:: +* Replication:: +@end menu + +@node Files +@subsubsection Files +@c %**end of header + +A file in GNUnet is just a sequence of bytes. Any file-format is allowed +and the maximum file size is theoretically 264 bytes, except that it +would take an impractical amount of time to share such a file. +GNUnet itself never interprets the contents of shared files, except +when using GNU libextractor to obtain keywords. + +@node Keywords +@subsubsection Keywords +@c %**end of header + +Keywords are the most simple mechanism to find files on GNUnet. +Keywords are @strong{case-sensitive} and the search string +must always match @strong{exactly} the keyword used by the +person providing the file. Keywords are never transmitted in +plaintext. The only way for an adversary to determine the keyword +that you used to search is to guess it (which then allows the +adversary to produce the same search request). Since providing +keywords by hand for each shared file is tedious, GNUnet uses +GNU libextractor to help automate this process. Starting a +keyword search on a slow machine can take a little while since +the keyword search involves computing a fresh RSA key to formulate the +request. + +@node Directories +@subsubsection Directories +@c %**end of header + +A directory in GNUnet is a list of file identifiers with meta data. +The file identifiers provide sufficient information about the files +to allow downloading the contents. Once a directory has been created, +it cannot be changed since it is treated just like an ordinary file +by the network. Small files (of a few kilobytes) can be inlined in +the directory, so that a separate download becomes unnecessary. + +@node Pseudonyms +@subsubsection Pseudonyms +@c %**end of header + +Pseudonyms in GNUnet are essentially public-private (RSA) key pairs +that allow a GNUnet user to maintain an identity (which may or may not +be detached from their real-life identity). GNUnet's pseudonyms are not +file-sharing specific --- and they will likely be used by many GNUnet +applications where a user identity is required. + +Note that a pseudonym is NOT bound to a GNUnet peer. There can be multiple +pseudonyms for a single user, and users could (theoretically) share the +private pseudonym keys (currently only out-of-band by knowing which files +to copy around). + +@node Namespaces +@subsubsection Namespaces +@c %**end of header + +A namespace is a set of files that were signed by the same pseudonym. +Files (or directories) that have been signed and placed into a namespace +can be updated. Updates are identified as authentic if the same secret +key was used to sign the update. Namespaces are also useful to establish +a reputation, since all of the content in the namespace comes from the +same entity (which does not have to be the same person). + +@node Advertisements +@subsubsection Advertisements +@c %**end of header + +Advertisements are used to notify other users about the existence of a +namespace. Advertisements are propagated using the normal keyword search. +When an advertisement is received (in response to a search), the namespace +is added to the list of namespaces available in the namespace-search +dialogs of gnunet-fs-gtk and printed by gnunet-pseudonym. Whenever a +namespace is created, an appropriate advertisement can be generated. +The default keyword for the advertising of namespaces is "namespace". + +Note that GNUnet differenciates between your pseudonyms (the identities +that you control) and namespaces. If you create a pseudonym, you will +not automatically see the respective namespace. You first have to create +an advertisement for the namespace and find it using keyword +search --- even for your own namespaces. The @command{gnunet-pseudonym} +tool is currently responsible for both managing pseudonyms and namespaces. +This will likely change in the future to reduce the potential for +confusion. + +@node Anonymity level +@subsubsection Anonymity level +@c %**end of header + +The anonymity level determines how hard it should be for an adversary to +determine the identity of the publisher or the searcher/downloader. An +anonymity level of zero means that anonymity is not required. The default +anonymity level of "1" means that anonymous routing is desired, but no +particular amount of cover traffic is necessary. A powerful adversary +might thus still be able to deduce the origin of the traffic using +traffic analysis. Specifying higher anonymity levels increases the +amount of cover traffic required. While this offers better privacy, +it can also significantly hurt performance. + +@node Content Priority +@subsubsection Content Priority +@c %**end of header + +Depending on the peer's configuration, GNUnet peers migrate content +between peers. Content in this sense are individual blocks of a file, +not necessarily entire files. When peers run out of space (due to +local publishing operations or due to migration of content from other +peers), blocks sometimes need to be discarded. GNUnet first always +discards expired blocks (typically, blocks are published with an +expiration of about two years in the future; this is another option). +If there is still not enough space, GNUnet discards the blocks with the +lowest priority. The priority of a block is decided by its popularity +(in terms of requests from peers we trust) and, in case of blocks +published locally, the base-priority that was specified by the user +when the block was published initially. + +@node Replication +@subsubsection Replication +@c %**end of header + +When peers migrate content to other systems, the replication level +of a block is used to decide which blocks need to be migrated most +urgently. GNUnet will always push the block with the highest +replication level into the network, and then decrement the replication +level by one. If all blocks reach replication level zero, the +selection is simply random. + +@node File-sharing Publishing +@subsection File-sharing Publishing +@c %**end of header + +The command @command{gnunet-publish} can be used to add content +to the network. The basic format of the command is + +@example +$ gnunet-publish [-n] [-k KEYWORDS]* [-m TYPE:VALUE] FILENAME +@end example + + +@menu +* Important command-line options:: +* Indexing vs. Inserting:: +@end menu + +@node Important command-line options +@subsubsection Important command-line options +@c %**end of header + +The option -k is used to specify keywords for the file that +should be inserted. You can supply any number of keywords, +and each of the keywords will be sufficient to locate and +retrieve the file. + +The -m option is used to specify meta-data, such as descriptions. +You can use -m multiple times. The TYPE passed must be from the +list of meta-data types known to libextractor. You can obtain this +list by running @command{extract -L}. Use quotes around the entire +meta-data argument if the value contains spaces. The meta-data +is displayed to other users when they select which files to +download. The meta-data and the keywords are optional and +maybe inferred using @code{GNU libextractor}. + +gnunet-publish has a few additional options to handle namespaces and +directories. See the man-page for details. + +@node Indexing vs. Inserting +@subsubsection Indexing vs Inserting +@c %**end of header + +By default, GNUnet indexes a file instead of making a full copy. +This is much more efficient, but requries the file to stay unaltered +at the location where it was when it was indexed. If you intend to move, +delete or alter a file, consider using the option @code{-n} which will +force GNUnet to make a copy of the file in the database. + +Since it is much less efficient, this is strongly discouraged for large +files. When GNUnet indexes a file (default), GNUnet does @strong{not} +create an additional encrypted copy of the file but just computes a +summary (or index) of the file. That summary is approximately two percent +of the size of the original file and is stored in GNUnet's database. +Whenever a request for a part of an indexed file reaches GNUnet, +this part is encrypted on-demand and send out. This way, there is no +need for an additional encrypted copy of the file to stay anywhere +on the drive. This is different from other systems, such as Freenet, +where each file that is put online must be in Freenet's database in +encrypted format, doubling the space requirements if the user wants +to preseve a directly accessible copy in plaintext. + +Thus indexing should be used for all files where the user will keep +using this file (at the location given to gnunet-publish) and does +not want to retrieve it back from GNUnet each time. If you want to +remove a file that you have indexed from the local peer, use the tool +@command{gnunet-unindex} to un-index the file. + +The option @code{-n} may be used if the user fears that the file might +be found on their drive (assuming the computer comes under the control +of an adversary). When used with the @code{-n} flag, the user has a +much better chance of denying knowledge of the existence of the file, +even if it is still (encrypted) on the drive and the adversary is +able to crack the encryption (e.g. by guessing the keyword. + +@node File-sharing Searching +@subsection File-sharing Searching +@c %**end of header + +The command @command{gnunet-search} can be used to search +for content on GNUnet. The format is: + +@example +$ gnunet-search [-t TIMEOUT] KEYWORD +@end example + +@noindent +The -t option specifies that the query should timeout after +approximately TIMEOUT seconds. A value of zero is interpreted +as @emph{no timeout}, which is also the default. In this case, +gnunet-search will never terminate (unless you press CTRL-C). + +If multiple words are passed as keywords, they will all be +considered optional. Prefix keywords with a "+" to make them mandatory. + +Note that searching using + +@example +$ gnunet-search Das Kapital +@end example + +@noindent +is not the same as searching for + +@example +$ gnunet-search "Das Kapital" +@end example + +@noindent +as the first will match files shared under the keywords +"Das" or "Kapital" whereas the second will match files +shared under the keyword "Das Kapital". + +Search results are printed by gnunet-search like this: + +@example +$ gnunet-download -o "COPYING" --- gnunet://fs/chk/N8...C92.17992 +=> The GNU Public License <= (mimetype: text/plain) +@end example + +@noindent +The first line is the command you would have to enter to download +the file. The argument passed to @code{-o} is the suggested +filename (you may change it to whatever you like). +The @code{--} is followed by key for decrypting the file, +the query for searching the file, a checksum (in hexadecimal) +finally the size of the file in bytes. +The second line contains the description of the file; here this is +"The GNU Public License" and the mime-type (see the options for +gnunet-publish on how to specify these). + +@node File-sharing Downloading +@subsection File-sharing Downloading +@c %**end of header + +In order to download a file, you need the three values returned by +@command{gnunet-search}. +You can then use the tool @command{gnunet-download} to obtain the file: + +@example +$ gnunet-download -o FILENAME --- GNUNETURL +@end example + +@noindent +FILENAME specifies the name of the file where GNUnet is supposed +to write the result. Existing files are overwritten. If the +existing file contains blocks that are identical to the +desired download, those blocks will not be downloaded again +(automatic resume). + +If you want to download the GPL from the previous example, +you do the following: + +@example +$ gnunet-download -o "COPYING" --- gnunet://fs/chk/N8...92.17992 +@end example + +@noindent +If you ever have to abort a download, you can continue it at any time by +re-issuing @command{gnunet-download} with the same filename. +In that case, GNUnet will @strong{not} download blocks again that are +already present. + +GNUnet's file-encoding mechanism will ensure file integrity, even if the +existing file was not downloaded from GNUnet in the first place. + +You may want to use the @command{-V} switch (must be added before +the @command{--}) to turn on verbose reporting. In this case, +@command{gnunet-download} will print the current number of +bytes downloaded whenever new data was received. + +@node File-sharing Directories +@subsection File-sharing Directories +@c %**end of header + +Directories are shared just like ordinary files. If you download a +directory with @command{gnunet-download}, you can use +@command{gnunet-directory} to list its contents. The canonical +extension for GNUnet directories when stored as files in your +local file-system is ".gnd". The contents of a directory are URIs and +meta data. +The URIs contain all the information required by +@command{gnunet-download} to retrieve the file. The meta data +typically includes the mime-type, description, a filename and +other meta information, and possibly even the full original file +(if it was small). + +@node File-sharing Namespace Management +@subsection File-sharing Namespace Management +@c %**end of header + +@b{Please note that the text in this subsection is outdated and needs} +@b{to be rewritten for version 0.10!} + +The gnunet-pseudonym tool can be used to create pseudonyms and +to advertise namespaces. By default, gnunet-pseudonym simply +lists all locally available pseudonyms. + + +@menu +* Creating Pseudonyms:: +* Deleting Pseudonyms:: +* Advertising namespaces:: +* Namespace names:: +* Namespace root:: +@end menu + +@node Creating Pseudonyms +@subsubsection Creating Pseudonyms +@c %**end of header + +With the @command{-C NICK} option it can also be used to +create a new pseudonym. A pseudonym is the virtual identity +of the entity in control of a namespace. Anyone can create +any number of pseudonyms. Note that creating a pseudonym can +take a few minutes depending on the performance of the machine +used. + +@node Deleting Pseudonyms +@subsubsection Deleting Pseudonyms +@c %**end of header + +With the @command{-D NICK} option pseudonyms can be deleted. +Once the pseudonym has been deleted it is impossible to add +content to the corresponding namespace. Deleting the +pseudonym does not make the namespace or any content in it +unavailable. + +@node Advertising namespaces +@subsubsection Advertising namespaces +@c %**end of header + +Each namespace is associated with meta-data that describes +the namespace. This meta data is provided by the user at +the time that the namespace is advertised. Advertisements +are published under keywords so that they can be found using +normal keyword-searches. This way, users can learn about new +namespaces without relying on out-of-band communication or directories. +A suggested keyword to use for all namespaces is simply "namespace". +When a keyword-search finds a namespace advertisement, +it is automatically stored in a local list of known namespaces. +Users can then associate a rank with the namespace to remember +the quality of the content found in it. + +@node Namespace names +@subsubsection Namespace names +@c %**end of header + +While the namespace is uniquely identified by its ID, another way +to refer to the namespace is to use the NICKNAME. +The NICKNAME can be freely chosen by the creator of the namespace and +hence conflicts are possible. If a GNUnet client learns about more +than one namespace using the same NICKNAME, the ID is appended +to the NICKNAME to get a unique identifier. + +@node Namespace root +@subsubsection Namespace root +@c %**end of header + +An item of particular interest in the namespace advertisement is +the ROOT. The ROOT is the identifier of a designated entry in the +namespace. The idea is that the ROOT can be used to advertise an +entry point to the content of the namespace. + +@node File-Sharing URIs +@subsection File-Sharing URIs +@c %**end of header + +GNUnet (currently) uses four different types of URIs for +file-sharing. They all begin with "gnunet://fs/". +This section describes the four different URI types in detail. + + +@menu +* Encoding of hash values in URIs:: +* Content Hash Key (chk):: +* Location identifiers (loc):: +* Keyword queries (ksk):: +* Namespace content (sks):: +@end menu + +@node Encoding of hash values in URIs +@subsubsection Encoding of hash values in URIs +@c %**end of header + +Most URIs include some hash values. Hashes are encoded using +base32hex (RFC 2938). + +@node Content Hash Key (chk) +@subsubsection Content Hash Key (chk) +@c %**end of header + +A chk-URI is used to (uniquely) identify a file or directory +and to allow peers to download the file. Files are stored in +GNUnet as a tree of encrypted blocks. +The chk-URI thus contains the information to download and decrypt +those blocks. A chk-URI has the format +"gnunet://fs/chk/KEYHASH.QUERYHASH.SIZE". Here, "SIZE" +is the size of the file (which allows a peer to determine the +shape of the tree), KEYHASH is the key used to decrypt the file +(also the hash of the plaintext of the top block) and QUERYHASH +is the query used to request the top-level block (also the hash +of the encrypted block). + +@node Location identifiers (loc) +@subsubsection Location identifiers (loc) +@c %**end of header + +For non-anonymous file-sharing, loc-URIs are used to specify which +peer is offering the data (in addition to specifying all of the +data from a chk-URI). Location identifiers include a digital +signature of the peer to affirm that the peer is truly the +origin of the data. The format is +"gnunet://fs/loc/KEYHASH.QUERYHASH.SIZE.PEER.SIG.EXPTIME". +Here, "PEER" is the public key of the peer (in GNUnet format in +base32hex), SIG is the RSA signature (in GNUnet format in +base32hex) and EXPTIME specifies when the signature expires +(in milliseconds after 1970). + +@node Keyword queries (ksk) +@subsubsection Keyword queries (ksk) +@c %**end of header + +A keyword-URI is used to specify that the desired operation +is the search using a particular keyword. The format is simply +"gnunet://fs/ksk/KEYWORD". Non-ASCII characters can be specified +using the typical URI-encoding (using hex values) from HTTP. +"+" can be used to specify multiple keywords (which are then +logically "OR"-ed in the search, results matching both keywords +are given a higher rank): "gnunet://fs/ksk/KEYWORD1+KEYWORD2". + +@node Namespace content (sks) +@subsubsection Namespace content (sks) +@c %**end of header + +Namespaces are sets of files that have been approved by some (usually +pseudonymous) user --- typically by that user publishing all of the +files together. A file can be in many namespaces. A file is in a +namespace if the owner of the ego (aka the namespace's private key) +signs the CHK of the file cryptographically. An SKS-URI is used to +search a namespace. The result is a block containing meta data, +the CHK and the namespace owner's signature. The format of a sks-URI +is "gnunet://fs/sks/NAMESPACE/IDENTIFIER". Here, "NAMESPACE" +is the public key for the namespace. "IDENTIFIER" is a freely +chosen keyword (or password!). A commonly used identifier is +"root" which by convention refers to some kind of index or other +entry point into the namespace. + +@node The GNU Name System +@section The GNU Name System +@c %**end of header + + +The GNU Name System (GNS) is secure and decentralized naming system. +It allows its users to resolve and register names within the @code{.gnu} +@dfn{top-level domain} (TLD). + +GNS is designed to provide: +@itemize @bullet +@item Censorship resistance +@item Query privacy +@item Secure name resolution +@item Compatibility with DNS +@end itemize + +For the initial configuration and population of your +GNS installation, please follow the GNS setup instructions. +The remainder of this chapter will provide some background on GNS +and then describe how to use GNS in more detail. + +Unlike DNS, GNS does not rely on central root zones or authorities. +Instead any user administers their own root and can can create arbitrary +name value mappings. Furthermore users can delegate resolution to other +users' zones just like DNS NS records do. Zones are uniquely identified +via public keys and resource records are signed using the corresponding +public key. Delegation to another user's zone is done using special PKEY +records and petnames. A petname is a name that can be freely chosen by +the user. This results in non-unique name-value mappings as +@code{@uref{http://www.bob.gnu/, www.bob.gnu}} to one user might be +@code{@uref{http://www.friend.gnu/, www.friend.gnu}} for someone else. + + +@menu +* Maintaining your own Zones:: +* Obtaining your Zone Key:: +* Adding Links to Other Zones:: +* The Three Local Zones of GNS:: +* The Master Zone:: +* The Private Zone:: +* The Shorten Zone:: +* The ZKEY Top Level Domain in GNS:: +* Resource Records in GNS:: +@end menu + + +@node Maintaining your own Zones +@subsection Maintaining your own Zones + +To setup your GNS system you must execute: + +@example +$ gnunet-gns-import.sh +@end example + +@noindent +This will boostrap your zones and create the necessary key material. +Your keys can be listed using the @command{gnunet-identity} +command line tool: + +@example +$ gnunet-identity -d +@end example + +@noindent +You can arbitrarily create your own zones using the gnunet-identity +tool using: + +@example +$ gnunet-identity -C "new_zone" +@end example + +@noindent +Now you can add (or edit, or remove) records in your GNS zone using the +gnunet-setup GUI or using the gnunet-namestore command-line tool. +In either case, your records will be stored in an SQL database under +control of the gnunet-service-namestore. Note that if multiple users +use one peer, the namestore database will include the combined records +of all users. However, users will not be able to see each other's records +if they are marked as private. + +To provide a simple example for editing your own zone, suppose you +have your own web server with IP 1.2.3.4. Then you can put an +A record (A records in DNS are for IPv4 IP addresses) into your +local zone using the command: + +@example +$ gnunet-namestore -z master-zone -a -n www -t A -V 1.2.3.4 -e never +@end example + +@noindent +Afterwards, you will be able to access your webpage under "www.gnu" +(assuming your webserver does not use virtual hosting, if it does, +please read up on setting up the GNS proxy). + +Similar commands will work for other types of DNS and GNS records, +the syntax largely depending on the type of the record. +Naturally, most users may find editing the zones using the +@command{gnunet-setup} GUI to be easier. + +@node Obtaining your Zone Key +@subsection Obtaining your Zone Key + +Each zone in GNS has a public-private key. Usually, gnunet-namestore and +gnunet-setup will access your private key as necessary, so you do not +have to worry about those. What is important is your public key +(or rather, the hash of your public key), as you will likely want to +give it to others so that they can securely link to you. + +You can usually get the hash of your public key using + +@example +$ gnunet-identity -d $options | grep master-zone | awk '@{print $3@}' +@end example + +@noindent +For example, the output might be something like: + +@example +DC3SEECJORPHQNVRH965A6N74B1M37S721IG4RBQ15PJLLPJKUE0 +@end example + +@noindent +Alternatively, you can obtain a QR code with your zone key AND +your pseudonym from gnunet-gtk. The QR code is displayed in the +GNS tab and can be stored to disk using the Save as button next +to the image. + +@node Adding Links to Other Zones +@subsection Adding Links to Other Zones + + +A central operation in GNS is the ability to securely delegate to +other zones. Basically, by adding a delegation you make all of the +names from the other zone available to yourself. This section +describes how to create delegations. + +Suppose you have a friend who you call 'bob' who also uses GNS. +You can then delegate resolution of names to Bob's zone by adding +a PKEY record to their local zone: + +@example +$ gnunet-namestore -a -n bob --type PKEY -V XXXX -e never +@end example + +@noindent +Note that XXXX in the command above must be replaced with the +hash of Bob's public key (the output your friend obtained using +the gnunet-identity command from the previous section and told you, +for example by giving you a business card containing this +information as a QR code). + +Assuming Bob has an A record for their website under the name of +www in his zone, you can then access Bob's website under +www.bob.gnu --- as well as any (public) GNS record that Bob has +in their zone by replacing www with the respective name of the +record in Bob's zone. + +@c themselves? themself? +Furthermore, if Bob has themselves a (public) delegation to Carol's +zone under "carol", you can access Carol's records under +NAME.carol.bob.gnu (where NAME is the name of Carol's record you +want to access). + +@node The Three Local Zones of GNS +@subsection The Three Local Zones of GNS + +Each user GNS has control over three zones. Each of the zones +has a different purpose. These zones are the + +@itemize @bullet + +@item master zone, +@item private zone, and the +@item shorten zone. +@end itemize + +@node The Master Zone +@subsection The Master Zone + + +The master zone is your personal TLD. Names within the @code{.gnu} +namespace are resolved relative to this zone. You can arbitrarily +add records to this zone and selectively publish those records. + +@node The Private Zone +@subsection The Private Zone + + +The private zone is a subzone (or subdomain in DNS terms) of your +master zone. It should be used for records that you want to keep +private. For example @code{bank.private.gnu}. The key idea is that +you want to keep your private records separate, if just to know +that those names are not available to other users. + +@node The Shorten Zone +@subsection The Shorten Zone + + +The shorten zone can either be a subzone of the master zone or the +private zone. It is different from the other zones in that GNS +will automatically populate this zone with other users' zones based +on their PSEU records whenever you resolve a name. + +For example if you go to +@code{@uref{http://www.bob.alice.dave.gnu/, www.bob.alice.dave.gnu}}, +GNS will try to import @code{bob} into your shorten zone. Having +obtained Bob's PKEY from @code{alice.dave.gnu}, GNS will lookup the +PSEU record for @code{+} in Bob's zone. If it exists and the specified +pseudonym is not taken, Bob's PKEY will be automatically added under +that pseudonym (i.e. "bob") into your shorten zone. From then on, +Bob's webpage will also be available for you as +@code{@uref{http://www.bob.short.gnu/, www.bob.short.gnu}}. +This feature is called @b{automatic name shortening} and is supposed to +keep GNS names as short and memorable as possible. + +@node The ZKEY Top Level Domain in GNS +@subsection The ZKEY Top Level Domain in GNS + + +GNS also provides a secure and globally unique namespace under the .zkey +top-level domain. A name in the .zkey TLD corresponds to the (printable) +public key of a zone. Names in the .zkey TLD are then resolved by querying +the respective zone. The .zkey TLD is expected to be used under rare +circumstances where globally unique names are required and for +integration with legacy systems. + +@node Resource Records in GNS +@subsection Resource Records in GNS + + +GNS supports the majority of the DNS records as defined in +@uref{http://www.ietf.org/rfc/rfc1035.txt, RFC 1035}. Additionally, +GNS defines some new record types the are unique to the GNS system. +For example, GNS-specific resource records are used to give petnames +for zone delegation, revoke zone keys and provide some compatibility +features. + +For some DNS records, GNS does extended processing to increase their +usefulness in GNS. In particular, GNS introduces special names +referred to as "zone relative names". Zone relative names are allowed +in some resource record types (for example, in NS and CNAME records) +and can also be used in links on webpages. Zone relative names end +in ".+" which indicates that the name needs to be resolved relative +to the current authoritative zone. The extended processing of those +names will expand the ".+" with the correct delegation chain to the +authoritative zone (replacing ".+" with the name of the location +where the name was encountered) and hence generate a +valid @code{.gnu} name. + +GNS currently supports the following record types: + +@menu +* NICK:: +* PKEY:: +* BOX:: +* LEHO:: +* VPN:: +* A AAAA and TXT:: +* CNAME:: +* GNS2DNS:: +* SOA SRV PTR and MX:: +@end menu + +@node NICK +@subsubsection NICK + +A NICK record is used to give a zone a name. With a NICK record, you can +essentially specify how you would like to be called. GNS expects this +record under the name "+" in the zone's database (NAMESTORE); however, +it will then automatically be copied into each record set, so that +clients never need to do a separate lookup to discover the NICK record. + +@b{Example}@ + +@example +Name: +; RRType: NICK; Value: bob +@end example + +@noindent +This record in Bob's zone will tell other users that this zone wants +to be referred to as 'bob'. Note that nobody is obliged to call Bob's +zone 'bob' in their own zones. It can be seen as a +recommendation ("Please call me 'bob'"). + +@node PKEY +@subsubsection PKEY + +PKEY records are used to add delegation to other users' zones and +give those zones a petname. + +@b{Example}@ + +Let Bob's zone be identified by the hash "ABC012". Bob is your friend +so you want to give them the petname "friend". Then you add the +following record to your zone: + +@example +Name: friend; RRType: PKEY; Value: ABC012; +@end example + +@noindent +This will allow you to resolve records in bob's zone +under "*.friend.gnu". + +@node BOX +@subsubsection BOX + +BOX records are there to integrate information from TLSA or +SRV records under the main label. In DNS, TLSA and SRV records +use special names of the form @code{_port._proto.(label.)*tld} to +indicate the port number and protocol (i.e. tcp or udp) for which +the TLSA or SRV record is valid. This causes various problems, and +is elegantly solved in GNS by integrating the protocol and port +numbers together with the respective value into a "BOX" record. +Note that in the GUI, you do not get to edit BOX records directly +right now --- the GUI will provide the illusion of directly +editing the TLSA and SRV records, even though they internally +are BOXed up. + +@node LEHO +@subsubsection LEHO + +The LEgacy HOstname of a server. Some webservers expect a specific +hostname to provide a service (virtiual hosting). Also SSL +certificates usually contain DNS names. To provide the expected +legacy DNS name for a server, the LEHO record can be used. +To mitigate the just mentioned issues the GNS proxy has to be used. +The GNS proxy will use the LEHO information to apply the necessary +transformations. + +@node VPN +@subsubsection VPN + +GNS allows easy access to services provided by the GNUnet Virtual Public +Network. When the GNS resolver encounters a VPN record it will contact +the VPN service to try and allocate an IPv4/v6 address (if the queries +record type is an IP address) that can be used to contact the service. + +@b{Example}@ + +I want to provide access to the VPN service "web.gnu." on port 80 on peer +ABC012:@ +Name: www; RRType: VPN; Value: 80 ABC012 web.gnu. + +The peer ABC012 is configured to provide an exit point for the service +"web.gnu." on port 80 to it's server running locally on port 8080 by +having the following lines in the @file{gnunet.conf} configuration file: + +@example +[web.gnunet.] +TCP_REDIRECTS = 80:localhost4:8080 +@end example + +@node A AAAA and TXT +@subsubsection A AAAA and TXT + +Those records work in exactly the same fashion as in traditional DNS. + +@node CNAME +@subsubsection CNAME + +As specified in RFC 1035 whenever a CNAME is encountered the query +needs to be restarted with the specified name. In GNS a CNAME +can either be: + +@itemize @bullet +@item A zone relative name, +@item A zkey name or +@item A DNS name (in which case resolution will continue outside +of GNS with the systems DNS resolver) +@end itemize + +@node GNS2DNS +@subsubsection GNS2DNS + +GNS can delegate authority to a legacy DNS zone. For this, the +name of the DNS nameserver and the name of the DNS zone are +specified in a GNS2DNS record. + +@b{Example} + +@example +Name: pet; RRType: GNS2DNS; Value: gnunet.org@@a.ns.joker.com +@end example + +@noindent +Any query to @code{pet.gnu} will then be delegated to the DNS server at +@code{a.ns.joker.com}. For example, +@code{@uref{http://www.pet.gnu/, www.pet.gnu}} will result in a DNS query +for @code{@uref{http://www.gnunet.org/, www.gnunet.org}} to the server +at @code{a.ns.joker.com}. Delegation to DNS via NS records in GNS can +be useful if you do not want to start resolution in the DNS root zone +(due to issues such as censorship or availability). + +Note that you would typically want to use a relative name for the +nameserver, i.e. + +@example +Name: pet; RRType: GNS2DNS; Value: gnunet.org@@ns-joker.+@ +Name: ns-joker; RRType: A; Value: 184.172.157.218 +@end example + +@noindent +This way, you can avoid involving the DNS hierarchy in the resolution of +@code{a.ns.joker.com}. In the example above, the problem may not be +obvious as the nameserver for "gnunet.org" is in the ".com" zone. +However, imagine the nameserver was "ns.gnunet.org". In this case, +delegating to "ns.gnunet.org" would mean that despite using GNS, +censorship in the DNS ".org" zone would still be effective. + +@node SOA SRV PTR and MX +@subsubsection SOA SRV PTR and MX + +The domain names in those records can, again, be either + +@itemize @bullet +@item A zone relative name, +@item A zkey name or +@item A DNS name +@end itemize + +The resolver will expand the zone relative name if possible. +Note that when using MX records within GNS, the target mail +server might still refuse to accept e-mails to the resulting +domain as the name might not match. GNS-enabled mail clients +should use the ZKEY zone as the destination hostname and +GNS-enabled mail servers should be configured to accept +e-mails to the ZKEY-zones of all local users. + +@node Using the Virtual Public Network +@section Using the Virtual Public Network + +@menu +* Setting up an Exit node:: +* Fedora and the Firewall:: +* Setting up VPN node for protocol translation and tunneling:: +@end menu + +Using the GNUnet Virtual Public Network (VPN) application you can +tunnel IP traffic over GNUnet. Moreover, the VPN comes +with built-in protocol translation and DNS-ALG support, enabling +IPv4-to-IPv6 protocol translation (in both directions). +This chapter documents how to use the GNUnet VPN. + +The first thing to note about the GNUnet VPN is that it is a public +network. All participating peers can participate and there is no +secret key to control access. So unlike common virtual private +networks, the GNUnet VPN is not useful as a means to provide a +"private" network abstraction over the Internet. The GNUnet VPN +is a virtual network in the sense that it is an overlay over the +Internet, using its own routing mechanisms and can also use an +internal addressing scheme. The GNUnet VPN is an Internet +underlay --- TCP/IP applications run on top of it. + +The VPN is currently only supported on GNU/Linux systems. +Support for operating systems that support TUN (such as FreeBSD) +should be easy to add (or might not even require any coding at +all --- we just did not test this so far). Support for other +operating systems would require re-writing the code to create virtual +network interfaces and to intercept DNS requests. + +The VPN does not provide good anonymity. While requests are routed +over the GNUnet network, other peers can directly see the source +and destination of each (encapsulated) IP packet. Finally, if you +use the VPN to access Internet services, the peer sending the +request to the Internet will be able to observe and even alter +the IP traffic. We will discuss additional security implications +of using the VPN later in this chapter. + +@node Setting up an Exit node +@subsection Setting up an Exit node + +Any useful operation with the VPN requires the existence of an exit +node in the GNUnet Peer-to-Peer network. Exit functionality can only +be enabled on peers that have regular Internet access. If you want +to play around with the VPN or support the network, we encourage +you to setup exit nodes. This chapter documents how to setup an +exit node. + +There are four types of exit functions an exit node can provide, +and using the GNUnet VPN to access the Internet will only work +nicely if the first three types are provided somewhere in +the network. The four exit functions are: + +@itemize @bullet +@item DNS: allow other peers to use your DNS resolver +@item IPv4: allow other peers to access your IPv4 Internet connection +@item IPv6: allow other peers to access your IPv6 Internet connection +@item Local service: allow other peers to access a specific TCP or +UDP service your peer is providing +@end itemize + +By enabling "exit" in gnunet-setup and checking the respective boxes +in the "exit" tab, you can easily choose which of the above exit +functions you want to support. + +Note, however, that by supporting the first three functions you will +allow arbitrary other GNUnet users to access the Internet via your +system. This is somewhat similar to running a Tor exit node. The +Torproject has a nice article about what to consider if you want +to do this here. We believe that generally running a DNS exit node +is completely harmless. + +The exit node configuration does currently not allow you to restrict the +Internet traffic that leaves your system. In particular, you cannot +exclude SMTP traffic (or block port 25) or limit to HTTP traffic using +the GNUnet configuration. However, you can use your host firewall to +restrict outbound connections from the virtual tunnel interface. This +is highly recommended. In the future, we plan to offer a wider range +of configuration options for exit nodes. + +Note that by running an exit node GNUnet will configure your kernel +to perform IP-forwarding (for IPv6) and NAT (for IPv4) so that the +traffic from the virtual interface can be routed to the Internet. +In order to provide an IPv6-exit, you need to have a subnet routed +to your host's external network interface and assign a subrange of +that subnet to the GNUnet exit's TUN interface. + +When running a local service, you should make sure that the local +service is (also) bound to the IP address of your EXIT interface +(i.e. 169.254.86.1). It will NOT work if your local service is +just bound to loopback. You may also want to create a "VPN" record +in your zone of the GNU Name System to make it easy for others to +access your service via a name instead of just the full service +descriptor. Note that the identifier you assign the service can +serve as a passphrase or shared secret, clients connecting to the +service must somehow learn the service's name. VPN records in the +GNU Name System can make this easier. + +@node Fedora and the Firewall +@subsection Fedora and the Firewall + + +When using an exit node on Fedora 15, the standard firewall can +create trouble even when not really exiting the local system! +For IPv4, the standard rules seem fine. However, for IPv6 the +standard rules prohibit traffic from the network range of the +virtual interface created by the exit daemon to the local IPv6 +address of the same interface (which is essentially loopback +traffic, so you might suspect that a standard firewall would +leave this traffic alone). However, as somehow for IPv6 the +traffic is not recognized as originating from the local +system (and as the connection is not already "established"), +the firewall drops the traffic. You should still get ICMPv6 +packets back, but that's obviously not very useful. + +Possible ways to fix this include disabling the firewall (do you +have a good reason for having it on?) or disabling the firewall +at least for the GNUnet exit interface (or the respective +IPv4/IPv6 address range). The best way to diagnose these kinds +of problems in general involves setting the firewall to REJECT +instead of DROP and to watch the traffic using wireshark +(or tcpdump) to see if ICMP messages are generated when running +some tests that should work. + +@node Setting up VPN node for protocol translation and tunneling +@subsection Setting up VPN node for protocol translation and tunneling + + +The GNUnet VPN/PT subsystem enables you to tunnel IP traffic over the +VPN to an exit node, from where it can then be forwarded to the +Internet. This section documents how to setup VPN/PT on a node. +Note that you can enable both the VPN and an exit on the same peer. +In this case, IP traffic from your system may enter your peer's VPN +and leave your peer's exit. This can be useful as a means to do +protocol translation. For example, you might have an application that +supports only IPv4 but needs to access an IPv6-only site. In this case, +GNUnet would perform 4to6 protocol translation between the VPN (IPv4) +and the Exit (IPv6). Similarly, 6to4 protocol translation is also +possible. However, the primary use for GNUnet would be to access +an Internet service running with an IP version that is not supported +by your ISP. In this case, your IP traffic would be routed via GNUnet +to a peer that has access to the Internet with the desired IP version. + +Setting up an entry node into the GNUnet VPN primarily requires you +to enable the "VPN/PT" option in "gnunet-setup". This will launch the +"gnunet-service-vpn", "gnunet-service-dns" and "gnunet-daemon-pt" +processes. The "gnunet-service-vpn" will create a virtual interface +which will be used as the target for your IP traffic that enters the +VPN. Additionally, a second virtual interface will be created by +the "gnunet-service-dns" for your DNS traffic. You will then need to +specify which traffic you want to tunnel over GNUnet. If your ISP only +provides you with IPv4 or IPv6-access, you may choose to tunnel the +other IP protocol over the GNUnet VPN. If you do not have an ISP +(and are connected to other GNUnet peers via WLAN), you can also +choose to tunnel all IP traffic over GNUnet. This might also provide +you with some anonymity. After you enable the respective options +and restart your peer, your Internet traffic should be tunneled +over the GNUnet VPN. + +The GNUnet VPN uses DNS-ALG to hijack your IP traffic. Whenever an +application resolves a hostname (i.e. 'gnunet.org'), the +"gnunet-daemon-pt" will instruct the "gnunet-service-dns" to intercept +the request (possibly route it over GNUnet as well) and replace the +normal answer with an IP in the range of the VPN's interface. +"gnunet-daemon-pt" will then tell "gnunet-service-vpn" to forward all +traffic it receives on the TUN interface via the VPN to the original +destination. + +For applications that do not use DNS, you can also manually create +such a mapping using the gnunet-vpn command-line tool. Here, you +specfiy the desired address family of the result (i.e. "-4"), and the +intended target IP on the Internet ("-i 131.159.74.67") and +"gnunet-vpn" will tell you which IP address in the range of your +VPN tunnel was mapped. + +@command{gnunet-vpn} can also be used to access "internal" services +offered by GNUnet nodes. So if you happen to know a peer and a +service offered by that peer, you can create an IP tunnel to +that peer by specifying the peer's identity, service name and +protocol (--tcp or --udp) and you will again receive an IP address +that will terminate at the respective peer's service. diff --git a/doc/documentation/chapters/vocabulary.texi b/doc/documentation/chapters/vocabulary.texi new file mode 100644 index 0000000000..85b40b17bf --- /dev/null +++ b/doc/documentation/chapters/vocabulary.texi @@ -0,0 +1,72 @@ +@node Vocabulary +@chapter Vocabulary + +@menu +* Definitions abbreviations and acronyms:: +* Words and characters:: +* Technical Assumptions:: +@end menu + +Throughout this Reference Manual we will use certain words and characters +which are listed in this introductionary chapter. + +@node Definitions abbreviations and acronyms +@section Definitions abbreviations and acronyms + +@menu +* Definitions:: +@end menu + +@node Definitions +@subsection Defitions + +Throughout this Reference Manual, the following terms and definitions +apply. + +@node Words and characters +@section Words and characters + +@enumerate +@item +In chapter Installation Handbook, +``@command{#}'' in example code blocks describes commands executed as root + +@example +# echo "I am root" +I am root +@end example + +@item +However, in the chapter GNUnet C Tutorial +``@command{#}'' in example code blocks describes commands, ie comments. + +@example +# Do the foobar thing: +$ make foobar +@end example + +@item +Dollarsign ``@command{$}'' in example code blocks describes commands you +execute as unprivileged users. + +@example +$ cd foo; ./configure --example-switch +@end example + +@item +Backslash ``@command{\}'' describes linebreaks. + +@example +./configure --foo --bar --baz \ + --short-loop +@end example + +...expands to @code{./configure --foo --bar --baz --short-loop} + +@end enumerate + +@node Technical Assumptions +@section Technical Assumptions + +@c Is it really assuming Bash (ie Bash extensions of POSIX being used)? +The shell on GNU systems is assumed to be Bash. diff --git a/doc/documentation/docstyle.css b/doc/documentation/docstyle.css new file mode 100644 index 0000000000..8719248d03 --- /dev/null +++ b/doc/documentation/docstyle.css @@ -0,0 +1,76 @@ +html, body { + font-size: 1em; + text-align: left; + text-decoration: none; +} +html { background-color: #e7e7e7; } + +body { + max-width: 74.92em; + margin: 0 auto; + padding: .5em 1em 1em 1em; + background-color: white; + border: .1em solid #c0c0c0; +} + +h1, h2, h3, h4 { color: #333; } +h5, h6, dt { color: #222; } + + +a h3 { + color: #005090; +} + +a[href] { color: #005090; } +a[href]:visited { color: #100070; } +a[href]:active, a[href]:hover { + color: #100070; + text-decoration: none; +} + +.linkrow { + margin: 3em 0; +} + +.linkrow { + text-align: center; +} + +div.example { padding: .8em 1.2em .4em; } +pre.example { padding: .8em 1.2em; } +div.example, pre.example { + margin: 1em 0 1em 3% ; + -webkit-border-radius: .3em; + -moz-border-radius: .3em; + border-radius: .3em; + border: 1px solid #d4cbb6; + background-color: #f2efe4; +} +div.example > pre.example { + padding: 0 0 .4em; + margin: 0; + border: none; +} + + +/* This makes the very long tables of contents in Gnulib and other + manuals easier to read. */ +.contents ul, .shortcontents ul { font-weight: bold; } +.contents ul ul, .shortcontents ul ul { font-weight: normal; } +.contents ul { list-style: none; } + +/* For colored navigation bars (Emacs manual): make the bar extend + across the whole width of the page and give it a decent height. */ +.header, .node { margin: 0 -1em; padding: 0 1em; } +.header p, .node p { line-height: 2em; } + +/* For navigation links */ +.node a, .header a { display: inline-block; line-height: 2em; } +.node a:hover, .header a:hover { background: #f2efe4; } + +table.cartouche { + border-collapse: collapse; + border-color: darkred; + border-style: solid; + border-width: 3px; +} diff --git a/doc/fdl-1.3.texi b/doc/documentation/fdl-1.3.texi index cb71f05a17..cb71f05a17 100644 --- a/doc/fdl-1.3.texi +++ b/doc/documentation/fdl-1.3.texi diff --git a/doc/documentation/gendocs.sh b/doc/documentation/gendocs.sh new file mode 100755 index 0000000000..3b71b36a20 --- /dev/null +++ b/doc/documentation/gendocs.sh @@ -0,0 +1,504 @@ +#!/bin/sh -e +# gendocs.sh -- generate a GNU manual in many formats. This script is +# mentioned in maintain.texi. See the help message below for usage details. + +scriptversion=2016-12-31.18 + +# Copyright 2003-2017 Free Software Foundation, Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# +# Original author: Mohit Agarwal. +# Send bug reports and any other correspondence to bug-gnulib@gnu.org. +# +# The latest version of this script, and the companion template, is +# available from the Gnulib repository: +# +# http://git.savannah.gnu.org/cgit/gnulib.git/tree/build-aux/gendocs.sh +# http://git.savannah.gnu.org/cgit/gnulib.git/tree/doc/gendocs_template + +# TODO: +# - image importing was only implemented for HTML generated by +# makeinfo. But it should be simple enough to adjust. +# - images are not imported in the source tarball. All the needed +# formats (PDF, PNG, etc.) should be included. + +prog=`basename "$0"` +srcdir=`pwd` + +scripturl="http://git.savannah.gnu.org/cgit/gnulib.git/plain/build-aux/gendocs.sh" +templateurl="http://git.savannah.gnu.org/cgit/gnulib.git/plain/doc/gendocs_template" + +: ${SETLANG="env LANG= LC_MESSAGES= LC_ALL= LANGUAGE="} +: ${MAKEINFO="makeinfo"} +: ${TEXI2DVI="texi2dvi"} +: ${DOCBOOK2HTML="docbook2html"} +: ${DOCBOOK2PDF="docbook2pdf"} +: ${DOCBOOK2TXT="docbook2txt"} +: ${GENDOCS_TEMPLATE_DIR="."} +: ${PERL='perl'} +: ${TEXI2HTML="texi2html"} +unset CDPATH +unset use_texi2html + +MANUAL_TITLE= +PACKAGE= +EMAIL=webmasters@gnu.org # please override with --email +commonarg= # passed to all makeinfo/texi2html invcations. +dirargs= # passed to all tools (-I dir). +dirs= # -I directories. +htmlarg="--css-ref=/software/gnulib/manual.css -c TOP_NODE_UP_URL=/manual" +infoarg=--no-split +generate_ascii=true +generate_html=true +generate_info=true +generate_tex=true +outdir=manual +source_extra= +split=node +srcfile= +texarg="-t @finalout" + +version="gendocs.sh $scriptversion + +Copyright 2017 Free Software Foundation, Inc. +There is NO warranty. You may redistribute this software +under the terms of the GNU General Public License. +For more information about these matters, see the files named COPYING." + +usage="Usage: $prog [OPTION]... PACKAGE MANUAL-TITLE + +Generate output in various formats from PACKAGE.texinfo (or .texi or +.txi) source. See the GNU Maintainers document for a more extensive +discussion: + http://www.gnu.org/prep/maintain_toc.html + +Options: + --email ADR use ADR as contact in generated web pages; always give this. + + -s SRCFILE read Texinfo from SRCFILE, instead of PACKAGE.{texinfo|texi|txi} + -o OUTDIR write files into OUTDIR, instead of manual/. + -I DIR append DIR to the Texinfo search path. + --common ARG pass ARG in all invocations. + --html ARG pass ARG to makeinfo or texi2html for HTML targets, + instead of '$htmlarg'. + --info ARG pass ARG to makeinfo for Info, instead of --no-split. + --no-ascii skip generating the plain text output. + --no-html skip generating the html output. + --no-info skip generating the info output. + --no-tex skip generating the dvi and pdf output. + --source ARG include ARG in tar archive of sources. + --split HOW make split HTML by node, section, chapter; default node. + --tex ARG pass ARG to texi2dvi for DVI and PDF, instead of -t @finalout. + + --texi2html use texi2html to make HTML target, with all split versions. + --docbook convert through DocBook too (xml, txt, html, pdf). + + --help display this help and exit successfully. + --version display version information and exit successfully. + +Simple example: $prog --email bug-gnu-emacs@gnu.org emacs \"GNU Emacs Manual\" + +Typical sequence: + cd PACKAGESOURCE/doc + wget \"$scripturl\" + wget \"$templateurl\" + $prog --email BUGLIST MANUAL \"GNU MANUAL - One-line description\" + +Output will be in a new subdirectory \"manual\" (by default; +use -o OUTDIR to override). Move all the new files into your web CVS +tree, as explained in the Web Pages node of maintain.texi. + +Please use the --email ADDRESS option so your own bug-reporting +address will be used in the generated HTML pages. + +MANUAL-TITLE is included as part of the HTML <title> of the overall +manual/index.html file. It should include the name of the package being +documented. manual/index.html is created by substitution from the file +$GENDOCS_TEMPLATE_DIR/gendocs_template. (Feel free to modify the +generic template for your own purposes.) + +If you have several manuals, you'll need to run this script several +times with different MANUAL values, specifying a different output +directory with -o each time. Then write (by hand) an overall index.html +with links to them all. + +If a manual's Texinfo sources are spread across several directories, +first copy or symlink all Texinfo sources into a single directory. +(Part of the script's work is to make a tar.gz of the sources.) + +As implied above, by default monolithic Info files are generated. +If you want split Info, or other Info options, use --info to override. + +You can set the environment variables MAKEINFO, TEXI2DVI, TEXI2HTML, +and PERL to control the programs that get executed, and +GENDOCS_TEMPLATE_DIR to control where the gendocs_template file is +looked for. With --docbook, the environment variables DOCBOOK2HTML, +DOCBOOK2PDF, and DOCBOOK2TXT are also consulted. + +By default, makeinfo and texi2dvi are run in the default (English) +locale, since that's the language of most Texinfo manuals. If you +happen to have a non-English manual and non-English web site, see the +SETLANG setting in the source. + +Email bug reports or enhancement requests to bug-gnulib@gnu.org. +" + +while test $# -gt 0; do + case $1 in + -s) shift; srcfile=$1;; + -o) shift; outdir=$1;; + -I) shift; dirargs="$dirargs -I '$1'"; dirs="$dirs $1";; + --common) shift; commonarg=$1;; + --docbook) docbook=yes;; + --email) shift; EMAIL=$1;; + --html) shift; htmlarg=$1;; + --info) shift; infoarg=$1;; + --no-ascii) generate_ascii=false;; + --no-html) generate_ascii=false;; + --no-info) generate_info=false;; + --no-tex) generate_tex=false;; + --source) shift; source_extra=$1;; + --split) shift; split=$1;; + --tex) shift; texarg=$1;; + --texi2html) use_texi2html=1;; + + --help) echo "$usage"; exit 0;; + --version) echo "$version"; exit 0;; + -*) + echo "$0: Unknown option \`$1'." >&2 + echo "$0: Try \`--help' for more information." >&2 + exit 1;; + *) + if test -z "$PACKAGE"; then + PACKAGE=$1 + elif test -z "$MANUAL_TITLE"; then + MANUAL_TITLE=$1 + else + echo "$0: extra non-option argument \`$1'." >&2 + exit 1 + fi;; + esac + shift +done + +# makeinfo uses the dirargs, but texi2dvi doesn't. +commonarg=" $dirargs $commonarg" + +# For most of the following, the base name is just $PACKAGE +base=$PACKAGE + +if test -n "$srcfile"; then + # but here, we use the basename of $srcfile + base=`basename "$srcfile"` + case $base in + *.txi|*.texi|*.texinfo) base=`echo "$base"|sed 's/\.[texinfo]*$//'`;; + esac + PACKAGE=$base +elif test -s "$srcdir/$PACKAGE.texinfo"; then + srcfile=$srcdir/$PACKAGE.texinfo +elif test -s "$srcdir/$PACKAGE.texi"; then + srcfile=$srcdir/$PACKAGE.texi +elif test -s "$srcdir/$PACKAGE.txi"; then + srcfile=$srcdir/$PACKAGE.txi +else + echo "$0: cannot find .texinfo or .texi or .txi for $PACKAGE in $srcdir." >&2 + exit 1 +fi + +if test ! -r $GENDOCS_TEMPLATE_DIR/gendocs_template; then + echo "$0: cannot read $GENDOCS_TEMPLATE_DIR/gendocs_template." >&2 + echo "$0: it is available from $templateurl." >&2 + exit 1 +fi + +# Function to return size of $1 in something resembling kilobytes. +calcsize() +{ + size=`ls -ksl $1 | awk '{print $1}'` + echo $size +} + +# copy_images OUTDIR HTML-FILE... +# ------------------------------- +# Copy all the images needed by the HTML-FILEs into OUTDIR. +# Look for them in . and the -I directories; this is simpler than what +# makeinfo supports with -I, but hopefully it will suffice. +copy_images() +{ + local odir + odir=$1 + shift + $PERL -n -e " +BEGIN { + \$me = '$prog'; + \$odir = '$odir'; + @dirs = qw(. $dirs); +} +" -e ' +/<img src="(.*?)"/g && ++$need{$1}; + +END { + #print "$me: @{[keys %need]}\n"; # for debugging, show images found. + FILE: for my $f (keys %need) { + for my $d (@dirs) { + if (-f "$d/$f") { + use File::Basename; + my $dest = dirname ("$odir/$f"); + # + use File::Path; + -d $dest || mkpath ($dest) + || die "$me: cannot mkdir $dest: $!\n"; + # + use File::Copy; + copy ("$d/$f", $dest) + || die "$me: cannot copy $d/$f to $dest: $!\n"; + next FILE; + } + } + die "$me: $ARGV: cannot find image $f\n"; + } +} +' -- "$@" || exit 1 +} + +case $outdir in + /*) abs_outdir=$outdir;; + *) abs_outdir=$srcdir/$outdir;; +esac + +echo "Making output for $srcfile" +echo " in `pwd`" +mkdir -p "$outdir/" + +# +if $generate_info; then + cmd="$SETLANG $MAKEINFO -o $PACKAGE.info $commonarg $infoarg \"$srcfile\"" + echo "Generating info... ($cmd)" + rm -f $PACKAGE.info* # get rid of any strays + eval "$cmd" + tar czf "$outdir/$PACKAGE.info.tar.gz" $PACKAGE.info* + ls -l "$outdir/$PACKAGE.info.tar.gz" + info_tgz_size=`calcsize "$outdir/$PACKAGE.info.tar.gz"` + # do not mv the info files, there's no point in having them available + # separately on the web. +fi # end info + +# +if $generate_tex; then + cmd="$SETLANG $TEXI2DVI $dirargs $texarg \"$srcfile\"" + printf "\nGenerating dvi... ($cmd)\n" + eval "$cmd" + # compress/finish dvi: + gzip -f -9 $PACKAGE.dvi + dvi_gz_size=`calcsize $PACKAGE.dvi.gz` + mv $PACKAGE.dvi.gz "$outdir/" + ls -l "$outdir/$PACKAGE.dvi.gz" + + cmd="$SETLANG $TEXI2DVI --pdf $dirargs $texarg \"$srcfile\"" + printf "\nGenerating pdf... ($cmd)\n" + eval "$cmd" + pdf_size=`calcsize $PACKAGE.pdf` + mv $PACKAGE.pdf "$outdir/" + ls -l "$outdir/$PACKAGE.pdf" +fi # end tex (dvi + pdf) + +# +if $generate_ascii; then + opt="-o $PACKAGE.txt --no-split --no-headers $commonarg" + cmd="$SETLANG $MAKEINFO $opt \"$srcfile\"" + printf "\nGenerating ascii... ($cmd)\n" + eval "$cmd" + ascii_size=`calcsize $PACKAGE.txt` + gzip -f -9 -c $PACKAGE.txt >"$outdir/$PACKAGE.txt.gz" + ascii_gz_size=`calcsize "$outdir/$PACKAGE.txt.gz"` + mv $PACKAGE.txt "$outdir/" + ls -l "$outdir/$PACKAGE.txt" "$outdir/$PACKAGE.txt.gz" +fi + +# + +if $generate_html; then +# Split HTML at level $1. Used for texi2html. +html_split() +{ + opt="--split=$1 --node-files $commonarg $htmlarg" + cmd="$SETLANG $TEXI2HTML --output $PACKAGE.html $opt \"$srcfile\"" + printf "\nGenerating html by $1... ($cmd)\n" + eval "$cmd" + split_html_dir=$PACKAGE.html + ( + cd ${split_html_dir} || exit 1 + ln -sf ${PACKAGE}.html index.html + tar -czf "$abs_outdir/${PACKAGE}.html_$1.tar.gz" -- *.html + ) + eval html_$1_tgz_size=`calcsize "$outdir/${PACKAGE}.html_$1.tar.gz"` + rm -f "$outdir"/html_$1/*.html + mkdir -p "$outdir/html_$1/" + mv ${split_html_dir}/*.html "$outdir/html_$1/" + rmdir ${split_html_dir} +} + +if test -z "$use_texi2html"; then + opt="--no-split --html -o $PACKAGE.html $commonarg $htmlarg" + cmd="$SETLANG $MAKEINFO $opt \"$srcfile\"" + printf "\nGenerating monolithic html... ($cmd)\n" + rm -rf $PACKAGE.html # in case a directory is left over + eval "$cmd" + html_mono_size=`calcsize $PACKAGE.html` + gzip -f -9 -c $PACKAGE.html >"$outdir/$PACKAGE.html.gz" + html_mono_gz_size=`calcsize "$outdir/$PACKAGE.html.gz"` + copy_images "$outdir/" $PACKAGE.html + mv $PACKAGE.html "$outdir/" + ls -l "$outdir/$PACKAGE.html" "$outdir/$PACKAGE.html.gz" + + # Before Texinfo 5.0, makeinfo did not accept a --split=HOW option, + # it just always split by node. So if we're splitting by node anyway, + # leave it out. + if test "x$split" = xnode; then + split_arg= + else + split_arg=--split=$split + fi + # + opt="--html -o $PACKAGE.html $split_arg $commonarg $htmlarg" + cmd="$SETLANG $MAKEINFO $opt \"$srcfile\"" + printf "\nGenerating html by $split... ($cmd)\n" + eval "$cmd" + split_html_dir=$PACKAGE.html + copy_images $split_html_dir/ $split_html_dir/*.html + ( + cd $split_html_dir || exit 1 + tar -czf "$abs_outdir/$PACKAGE.html_$split.tar.gz" -- * + ) + eval \ + html_${split}_tgz_size=`calcsize "$outdir/$PACKAGE.html_$split.tar.gz"` + rm -rf "$outdir/html_$split/" + mv $split_html_dir "$outdir/html_$split/" + du -s "$outdir/html_$split/" + ls -l "$outdir/$PACKAGE.html_$split.tar.gz" + +else # use texi2html: + opt="--output $PACKAGE.html $commonarg $htmlarg" + cmd="$SETLANG $TEXI2HTML $opt \"$srcfile\"" + printf "\nGenerating monolithic html with texi2html... ($cmd)\n" + rm -rf $PACKAGE.html # in case a directory is left over + eval "$cmd" + html_mono_size=`calcsize $PACKAGE.html` + gzip -f -9 -c $PACKAGE.html >"$outdir/$PACKAGE.html.gz" + html_mono_gz_size=`calcsize "$outdir/$PACKAGE.html.gz"` + mv $PACKAGE.html "$outdir/" + + html_split node + html_split chapter + html_split section +fi +fi # end html + +# +printf "\nMaking .tar.gz for sources...\n" +d=`dirname $srcfile` +( + cd "$d" + srcfiles=`ls -d *.texinfo *.texi *.txi *.eps $source_extra 2>/dev/null` || true + tar czfh "$abs_outdir/$PACKAGE.texi.tar.gz" $srcfiles + ls -l "$abs_outdir/$PACKAGE.texi.tar.gz" +) +texi_tgz_size=`calcsize "$outdir/$PACKAGE.texi.tar.gz"` + +# +# Do everything again through docbook. +if test -n "$docbook"; then + opt="-o - --docbook $commonarg" + cmd="$SETLANG $MAKEINFO $opt \"$srcfile\" >${srcdir}/$PACKAGE-db.xml" + printf "\nGenerating docbook XML... ($cmd)\n" + eval "$cmd" + docbook_xml_size=`calcsize $PACKAGE-db.xml` + gzip -f -9 -c $PACKAGE-db.xml >"$outdir/$PACKAGE-db.xml.gz" + docbook_xml_gz_size=`calcsize "$outdir/$PACKAGE-db.xml.gz"` + mv $PACKAGE-db.xml "$outdir/" + + split_html_db_dir=html_node_db + opt="$commonarg -o $split_html_db_dir" + cmd="$DOCBOOK2HTML $opt \"${outdir}/$PACKAGE-db.xml\"" + printf "\nGenerating docbook HTML... ($cmd)\n" + eval "$cmd" + ( + cd ${split_html_db_dir} || exit 1 + tar -czf "$abs_outdir/${PACKAGE}.html_node_db.tar.gz" -- *.html + ) + html_node_db_tgz_size=`calcsize "$outdir/${PACKAGE}.html_node_db.tar.gz"` + rm -f "$outdir"/html_node_db/*.html + mkdir -p "$outdir/html_node_db" + mv ${split_html_db_dir}/*.html "$outdir/html_node_db/" + rmdir ${split_html_db_dir} + + cmd="$DOCBOOK2TXT \"${outdir}/$PACKAGE-db.xml\"" + printf "\nGenerating docbook ASCII... ($cmd)\n" + eval "$cmd" + docbook_ascii_size=`calcsize $PACKAGE-db.txt` + mv $PACKAGE-db.txt "$outdir/" + + cmd="$DOCBOOK2PDF \"${outdir}/$PACKAGE-db.xml\"" + printf "\nGenerating docbook PDF... ($cmd)\n" + eval "$cmd" + docbook_pdf_size=`calcsize $PACKAGE-db.pdf` + mv $PACKAGE-db.pdf "$outdir/" +fi + +# +printf "\nMaking index.html for $PACKAGE...\n" +if test -z "$use_texi2html"; then + CONDS="/%%IF *HTML_SECTION%%/,/%%ENDIF *HTML_SECTION%%/d;\ + /%%IF *HTML_CHAPTER%%/,/%%ENDIF *HTML_CHAPTER%%/d" +else + # should take account of --split here. + CONDS="/%%ENDIF.*%%/d;/%%IF *HTML_SECTION%%/d;/%%IF *HTML_CHAPTER%%/d" +fi + +curdate=`$SETLANG date '+%B %d, %Y'` +sed \ + -e "s!%%TITLE%%!$MANUAL_TITLE!g" \ + -e "s!%%EMAIL%%!$EMAIL!g" \ + -e "s!%%PACKAGE%%!$PACKAGE!g" \ + -e "s!%%DATE%%!$curdate!g" \ + -e "s!%%HTML_MONO_SIZE%%!$html_mono_size!g" \ + -e "s!%%HTML_MONO_GZ_SIZE%%!$html_mono_gz_size!g" \ + -e "s!%%HTML_NODE_TGZ_SIZE%%!$html_node_tgz_size!g" \ + -e "s!%%HTML_SECTION_TGZ_SIZE%%!$html_section_tgz_size!g" \ + -e "s!%%HTML_CHAPTER_TGZ_SIZE%%!$html_chapter_tgz_size!g" \ + -e "s!%%INFO_TGZ_SIZE%%!$info_tgz_size!g" \ + -e "s!%%DVI_GZ_SIZE%%!$dvi_gz_size!g" \ + -e "s!%%PDF_SIZE%%!$pdf_size!g" \ + -e "s!%%ASCII_SIZE%%!$ascii_size!g" \ + -e "s!%%ASCII_GZ_SIZE%%!$ascii_gz_size!g" \ + -e "s!%%TEXI_TGZ_SIZE%%!$texi_tgz_size!g" \ + -e "s!%%DOCBOOK_HTML_NODE_TGZ_SIZE%%!$html_node_db_tgz_size!g" \ + -e "s!%%DOCBOOK_ASCII_SIZE%%!$docbook_ascii_size!g" \ + -e "s!%%DOCBOOK_PDF_SIZE%%!$docbook_pdf_size!g" \ + -e "s!%%DOCBOOK_XML_SIZE%%!$docbook_xml_size!g" \ + -e "s!%%DOCBOOK_XML_GZ_SIZE%%!$docbook_xml_gz_size!g" \ + -e "s,%%SCRIPTURL%%,$scripturl,g" \ + -e "s!%%SCRIPTNAME%%!$prog!g" \ + -e "$CONDS" \ +$GENDOCS_TEMPLATE_DIR/gendocs_template >"$outdir/index.html" + +echo "Done, see $outdir/ subdirectory for new files." + +# Local variables: +# eval: (add-hook 'write-file-hooks 'time-stamp) +# time-stamp-start: "scriptversion=" +# time-stamp-format: "%:y-%02m-%02d.%02H" +# time-stamp-end: "$" +# End: diff --git a/doc/documentation/gendocs_template b/doc/documentation/gendocs_template new file mode 100644 index 0000000000..178f6cb4c8 --- /dev/null +++ b/doc/documentation/gendocs_template @@ -0,0 +1,91 @@ +<!--#include virtual="/server/header.html" --> +<!-- Parent-Version: 1.77 --> +<title>%%TITLE%% - GNU Project - Free Software Foundation</title> +<!--#include virtual="/server/banner.html" --> +<h2>%%TITLE%%</h2> + +<address>Free Software Foundation</address> +<address>last updated %%DATE%%</address> + +<p>This manual (%%PACKAGE%%) is available in the following formats:</p> + +<ul> +<li><a href="%%PACKAGE%%.html">HTML + (%%HTML_MONO_SIZE%%K bytes)</a> - entirely on one web page.</li> +<li><a href="html_node/index.html">HTML</a> - with one web page per + node.</li> +%%IF HTML_SECTION%% +<li><a href="html_section/index.html">HTML</a> - with one web page per + section.</li> +%%ENDIF HTML_SECTION%% +%%IF HTML_CHAPTER%% +<li><a href="html_chapter/index.html">HTML</a> - with one web page per + chapter.</li> +%%ENDIF HTML_CHAPTER%% +<li><a href="%%PACKAGE%%.html.gz">HTML compressed + (%%HTML_MONO_GZ_SIZE%%K gzipped characters)</a> - entirely on + one web page.</li> +<li><a href="%%PACKAGE%%.html_node.tar.gz">HTML compressed + (%%HTML_NODE_TGZ_SIZE%%K gzipped tar file)</a> - + with one web page per node.</li> +%%IF HTML_SECTION%% +<li><a href="%%PACKAGE%%.html_section.tar.gz">HTML compressed + (%%HTML_SECTION_TGZ_SIZE%%K gzipped tar file)</a> - + with one web page per section.</li> +%%ENDIF HTML_SECTION%% +%%IF HTML_CHAPTER%% +<li><a href="%%PACKAGE%%.html_chapter.tar.gz">HTML compressed + (%%HTML_CHAPTER_TGZ_SIZE%%K gzipped tar file)</a> - + with one web page per chapter.</li> +%%ENDIF HTML_CHAPTER%% +<li><a href="%%PACKAGE%%.info.tar.gz">Info document + (%%INFO_TGZ_SIZE%%K bytes gzipped tar file)</a>.</li> +<li><a href="%%PACKAGE%%.txt">ASCII text + (%%ASCII_SIZE%%K bytes)</a>.</li> +<li><a href="%%PACKAGE%%.txt.gz">ASCII text compressed + (%%ASCII_GZ_SIZE%%K bytes gzipped)</a>.</li> +<li><a href="%%PACKAGE%%.dvi.gz">TeX dvi file + (%%DVI_GZ_SIZE%%K bytes gzipped)</a>.</li> +<li><a href="%%PACKAGE%%.pdf">PDF file + (%%PDF_SIZE%%K bytes)</a>.</li> +<li><a href="%%PACKAGE%%.texi.tar.gz">Texinfo source + (%%TEXI_TGZ_SIZE%%K bytes gzipped tar file).</a></li> +</ul> + +<p>You can <a href="http://shop.fsf.org/">buy printed copies of +some manuals</a> (among other items) from the Free Software Foundation; +this helps support FSF activities.</p> + +<p>(This page generated by the <a href="%%SCRIPTURL%%">%%SCRIPTNAME%% +script</a>.)</p> + +<!-- If needed, change the copyright block at the bottom. In general, + all pages on the GNU web server should have the section about + verbatim copying. Please do NOT remove this without talking + with the webmasters first. + Please make sure the copyright date is consistent with the document + and that it is like this: "2001, 2002", not this: "2001-2002". --> +</div><!-- for id="content", starts in the include above --> +<!--#include virtual="/server/footer.html" --> +<div id="footer"> +<div class="unprintable"> + +<p>Please send general FSF & GNU inquiries to +<a href="mailto:gnu@gnu.org"><gnu@gnu.org></a>. +There are also <a href="/contact/">other ways to contact</a> +the FSF. Broken links and other corrections or suggestions can be sent +to <a href="mailto:%%EMAIL%%"><%%EMAIL%%></a>.</p> +</div> + +<p>Copyright © 2017 Free Software Foundation, Inc.</p> + +<p>This page is licensed under a <a rel="license" +href="http://creativecommons.org/licenses/by-nd/3.0/us/">Creative +Commons Attribution-NoDerivs 3.0 United States License</a>.</p> + +<!--#include virtual="/server/bottom-notes.html" --> + +</div> +</div> +</body> +</html> diff --git a/doc/documentation/gendocs_template_min b/doc/documentation/gendocs_template_min new file mode 100644 index 0000000000..112fa3bfb6 --- /dev/null +++ b/doc/documentation/gendocs_template_min @@ -0,0 +1,93 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" + "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> +<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en"> + +<head> +<title>%%TITLE%% - GNU Project - Free Software Foundation</title> +<meta http-equiv="content-type" content='text/html; charset=utf-8' /> +<link rel="stylesheet" type="text/css" href="/gnu.css" /> +</head> + +<body> + +<h3>%%TITLE%%</h3> + +<address>Free Software Foundation</address> +<address>last updated %%DATE%%</address> +<p> +<a href="/graphics/gnu-head.jpg"> + <img src="/graphics/gnu-head-sm.jpg" + alt=" [image of the head of a GNU] " width="129" height="122"/> +</a> +</p> +<hr /> + +<p>This manual (%%PACKAGE%%) is available in the following formats:</p> + +<ul> +<li><a href="%%PACKAGE%%.html">HTML + (%%HTML_MONO_SIZE%%K bytes)</a> - entirely on one web page.</li> +<li><a href="html_node/index.html">HTML</a> - with one web page per + node.</li> +%%IF HTML_SECTION%% +<li><a href="html_section/index.html">HTML</a> - with one web page per + section.</li> +%%ENDIF HTML_SECTION%% +%%IF HTML_CHAPTER%% +<li><a href="html_chapter/index.html">HTML</a> - with one web page per + chapter.</li> +%%ENDIF HTML_CHAPTER%% +<li><a href="%%PACKAGE%%.html.gz">HTML compressed + (%%HTML_MONO_GZ_SIZE%%K gzipped characters)</a> - entirely on + one web page.</li> +<li><a href="%%PACKAGE%%.html_node.tar.gz">HTML compressed + (%%HTML_NODE_TGZ_SIZE%%K gzipped tar file)</a> - + with one web page per node.</li> +%%IF HTML_SECTION%% +<li><a href="%%PACKAGE%%.html_section.tar.gz">HTML compressed + (%%HTML_SECTION_TGZ_SIZE%%K gzipped tar file)</a> - + with one web page per section.</li> +%%ENDIF HTML_SECTION%% +%%IF HTML_CHAPTER%% +<li><a href="%%PACKAGE%%.html_chapter.tar.gz">HTML compressed + (%%HTML_CHAPTER_TGZ_SIZE%%K gzipped tar file)</a> - + with one web page per chapter.</li> +%%ENDIF HTML_CHAPTER%% +<li><a href="%%PACKAGE%%.info.tar.gz">Info document + (%%INFO_TGZ_SIZE%%K bytes gzipped tar file)</a>.</li> +<li><a href="%%PACKAGE%%.txt">ASCII text + (%%ASCII_SIZE%%K bytes)</a>.</li> +<li><a href="%%PACKAGE%%.txt.gz">ASCII text compressed + (%%ASCII_GZ_SIZE%%K bytes gzipped)</a>.</li> +<li><a href="%%PACKAGE%%.dvi.gz">TeX dvi file + (%%DVI_GZ_SIZE%%K bytes gzipped)</a>.</li> +<li><a href="%%PACKAGE%%.pdf">PDF file + (%%PDF_SIZE%%K bytes)</a>.</li> +<li><a href="%%PACKAGE%%.texi.tar.gz">Texinfo source + (%%TEXI_TGZ_SIZE%%K bytes gzipped tar file).</a></li> +</ul> + +<p>(This page generated by the <a href="%%SCRIPTURL%%">%%SCRIPTNAME%% +script</a>.)</p> + +<div id="footer" class="copyright"> + +<p>Please send general FSF & GNU inquiries to +<a href="mailto:gnu@gnu.org"><gnu@gnu.org></a>. +There are also <a href="/contact/">other ways to contact</a> +the FSF. Broken links and other corrections or suggestions can be sent +to <a href="mailto:%%EMAIL%%"><%%EMAIL%%></a>.</p> +</div> + +<p>Copyright © 2017 Free Software Foundation, Inc.</p> + +<p>This page is licensed under a <a rel="license" +href="http://creativecommons.org/licenses/by-nd/3.0/us/">Creative +Commons Attribution-NoDerivs 3.0 United States License</a>.</p> + +<!--#include virtual="/server/bottom-notes.html" --> + +</div> +</body> +</html> diff --git a/doc/gnunet-c-tutorial.texi b/doc/documentation/gnunet-c-tutorial.texi index 3a4200d7c8..f39c7de642 100644 --- a/doc/gnunet-c-tutorial.texi +++ b/doc/documentation/gnunet-c-tutorial.texi @@ -3,8 +3,12 @@ @setfilename gnunet-c-tutorial.info @documentencoding UTF-8 @settitle GNUnet C Tutorial +@exampleindent 2 @c %**end of header +@c including 'version.texi' makes makeinfo throw errors. +@include version2.texi + @copying Copyright @copyright{} 2001-2017 GNUnet e.V. @@ -27,9 +31,15 @@ A copy of the license is also available from the Free Software Foundation Web site at @url{http://www.gnu.org/licenses/gpl.html}. @end copying +@dircategory Tutorial +@direntry +* GNUnet-C-Tutorial: (gnunet-c-tutorial). C Tutorial for GNunet +@end direntry + + @titlepage @title GNUnet C Tutorial -@subtitle A Tutorial for GNUnet 0.10.x (C version) +@subtitle A Tutorial for GNUnet @value{VERSION} (C version) @author The GNUnet Developers @page @@ -48,10 +58,14 @@ Foundation Web site at @url{http://www.gnu.org/licenses/gpl.html}. @node Top @top Introduction -This tutorials explains how to install GNUnet on a GNU/Linux system and gives an introduction on how -GNUnet can be used to develop a Peer-to-Peer application. Detailed installation instructions for -various operating systems and a detailed list of all dependencies can be found on our website at -@uref{https://gnunet.org/installation}. +This tutorials explains how to install GNUnet on a +GNU/Linux system and gives an introduction on how +GNUnet can be used to develop a Peer-to-Peer application. +Detailed installation instructions for +various operating systems and a detailed list of all +dependencies can be found on our website at +@uref{https://gnunet.org/installation} and in our +Reference Documentation (GNUnet Handbook). Please read this tutorial carefully since every single step is important and do not hesitate to contact the GNUnet team if you have @@ -103,14 +117,15 @@ Developing Applications @node Installing GNUnet @chapter Installing GNUnet -First of all you have to install a current version of GNUnet. You can download a -tarball of a stable version from GNU FTP mirrors or obtain the latest development -version from our Git repository. +First of all you have to install a current version of GNUnet. +You can download a tarball of a stable version from GNU FTP mirrors +or obtain the latest development version from our Git repository. -Most of the time you should prefer to download the stable version since with the -latest development version things can be broken, functionality can be changed or tests -can fail. You should only use the development version if you know that you require a -certain feature or a certain issue has been fixed since the last release. +Most of the time you should prefer to download the stable version +since with the latest development version things can be broken, +functionality can be changed or tests can fail. You should only use +the development version if you know that you require a certain +feature or a certain issue has been fixed since the last release. @menu * Obtaining a stable version:: @@ -123,67 +138,109 @@ certain feature or a certain issue has been fixed since the last release. @node Obtaining a stable version @section Obtaining a stable version -You can download the latest stable version of GNUnet from GNU FTP mirrors: -@uref{https://ftp.gnu.org/gnu/gnunet/gnunet-0.10.x.tar.gz} -You should also download the signature file and verify the integrity of the tarball. -@uref{https://ftp.gnu.org/gnu/gnunet/gnunet-0.10.x.tar.gz.sig} -To verify the signature you should first import the GPG key used to sign the tarball +Download the tarball from +@indicateurl{https://ftp.gnu.org/gnu/gnunet/gnunet-@value{VERSION}.tar.gz}. + +Make sure to download the associated @file{.sig} file and to verify the +authenticity of the tarball against it, like this: + @example -$ gpg --keyserver keys.gnupg.net --recv-keys 48426C7E +$ wget https://ftp.gnu.org/gnu/gnunet/gnunet-@value{VERSION}.tar.gz.sig +$ gpg --verify-files gnunet-@value{VERSION}.tar.gz.sig @end example -And use this key to verify the tarball's signature + +@noindent +If this command fails because you do not have the required public key, +then you need to run this command to import it: + @example -$ gpg --verify gnunet-0.10.x.tar.gz.sig gnunet-0.10.x.tar.gz +$ gpg --keyserver keys.gnupg.net --recv-keys 48426C7E @end example -After successfully verifying the integrity you can extract the tarball using + +@noindent +and rerun the @code{gpg --verify-files} command. + +@b{Note:}@ +@b{The pub key to sign the 0.10.1 release has been +revoked}. You will get an error message stating that +@b{there is no known public key or that it has been revoked}. +The next release of GNUnet will have a valid signature +again. We are sorry for the inconvenience this causes. +Another possible source you could use is our +"gnunet" git repository which has mandatory signed commits +by every developer. + +Now you can extract the tarball and rename the resulting +directory to @file{gnunet} which we will be using in the +remainder of this document. + @example -$ tar xvzf gnunet-0.10.x.tar.gz -## we will use the directory "gnunet" in the remainder of this document -$ mv gnunet-0.10.x gnunet +$ tar xvzf gnunet-@value{VERSION}.tar.gz +$ mv gnunet-@value{VERSION} gnunet $ cd gnunet @end example -However, please note that stable versions can be very outdated, as a developer -you are strongly encouraged to use the version from @uref{https://gnunet.org/git/}. +@noindent +However, please note that stable versions can be very outdated. +As a developer you are @b{strongly} encouraged to use the version +from @uref{https://gnunet.org/git/, git}. @node Installing Build Tool Chain and Dependencies @section Installing Build Tool Chain and Dependencies -To successfully compile GNUnet you need the tools to build GNUnet and the required dependencies. -Please have a look at @uref{https://gnunet.org/dependencies} for a list of required dependencies -and @uref{https://gnunet.org/generic_installation} for specific instructions for your operating system. - -Please check the notes at the end of the configure process about required dependencies. - -For GNUnet bootstrapping support and the http(s) plugin you should install libgnurl. -For the filesharing service you should install at least one of the datastore backends mysql, -sqlite or postgresql. +To successfully compile GNUnet, you need the tools to build GNUnet and +the required dependencies. Please take a look at the +GNUnet Reference Documentation +(@pxref{Dependencies, The GNUnet Reference Documentation,, gnunet, The GNUnet Reference Documentation}) +for a list of required dependencies +and +(@pxref{Generic installation instructions, The GNUnet Reference Documentation,, gnunet, The GNUnet Reference Documentation}) +read its Installation chapter for specific instructions for +your operating system. +Please check the notes at the end of the configure process about +required dependencies. + +For GNUnet bootstrapping support and the HTTP(S) plugin you should +install @uref{https://gnunet.org/gnurl, libgnurl}. +For the filesharing service you should install at least one of the +datastore backends. MySQL, SQlite and PostgreSQL are supported. @node Obtaining the latest version from Git @section Obtaining the latest version from Git -The latest development version can obtained from our Git repository. To obtain -the code you need Git installed and checkout the repository using: +The latest development version can obtained from our Git repository. +To obtain the code you need to have @code{Git} installed, which is +required for obtaining the repository via: + @example $ git clone https://gnunet.org/git/gnunet @end example -After cloning the repository you have to execute + +@noindent +After cloning the repository you have to execute the @file{bootstrap} +script in the new directory: + @example $ cd gnunet $ ./bootstrap @end example -The remainder of this tutorial assumes that you have Git branch ``master'' checked out. +@noindent +The remainder of this tutorial will assume that you have the +Git branch ``master'' checked out. @node Compiling and Installing GNUnet @section Compiling and Installing GNUnet -First, you need to install at least libgnupgerror version 1.27 -@uref{https://www.gnupg.org/ftp/gcrypt/libgpg-error/libgpg-error-1.27.tar.bz2} -and libgcrypt version 1.7.6 @uref{https://www.gnupg.org/ftp/gcrypt/libgcrypt/libgcrypt-1.7.6.tar.bz2}. +Note: This section is a duplication of the more in depth +@pxref{GNUnet Installation Handbook, The GNUnet Reference Documentation,, gnunet, The GNUnet Reference Documentation}. + +First, you need to install libgnupgerror @geq{} 1.27 and +libgcrypt @geq{} 1.7.6: @example -$ wget https://www.gnupg.org/ftp/gcrypt/libgpg-error/libgpg-error-1.27.tar.bz2 +$ export GNUPGFTP="https://www.gnupg.org/ftp/gcrypt" +$ wget $GNUPGFTP/libgpg-error/libgpg-error-1.27.tar.bz2 $ tar xf libgpg-error-1.27.tar.bz2 $ cd libgpg-error-1.27 $ ./configure @@ -192,7 +249,8 @@ $ cd .. @end example @example -$ wget https://www.gnupg.org/ftp/gcrypt/libgcrypt/libgcrypt-1.7.6.tar.bz2 +$ export GNUPGFTP="https://www.gnupg.org/ftp/gcrypt" +$ wget $GNUPGFTP/libgcrypt/libgcrypt-1.7.6.tar.bz2 $ tar xf libgcrypt-1.7.6.tar.bz2 $ cd libgcrypt-1.7.6 $ ./configure @@ -220,10 +278,12 @@ $ make $ make install @end example -After installing GNUnet you have to add your GNUnet installation to your path -environmental variable. In addition you have to create the @file{.config} -directory in your home directory (unless it already exists) where GNUnet stores -its data and an empty GNUnet configuration file: +@noindent +After installing GNUnet you have to add your GNUnet installation +to your path environmental variable. In addition you have to +create the @file{.config} directory in your home directory +(unless it already exists) where GNUnet stores its data and an +empty GNUnet configuration file: @example $ export PATH=$PATH:$PREFIX/bin @@ -238,23 +298,34 @@ $ touch ~/.config/gnunet.conf You should check your installation to ensure that installing GNUnet was successful up to this point. You should be able to access GNUnet's binaries and run GNUnet's self check. + @example $ which gnunet-arm @end example -should return $PREFIX/bin/gnunet-arm. It should be -located in your GNUnet installation and the output should not be -empty. If you see an output like: + +@noindent +should return $PREFIX/bin/gnunet-arm. It should be located in your +GNUnet installation and the output should not be empty. +If you see an output like: + @example $ which gnunet-arm @end example -check your PATH variable to ensure GNUnet's @file{bin} directory is included. + +@noindent +check your PATH variable to ensure GNUnet's @file{bin} directory is +included. GNUnet provides tests for all of its subcomponents. Run + @example $ make check @end example -to execute tests for all components. make check traverses all subdirectories in src. -For every subdirectory you should get a message like this: + +@noindent +to execute tests for all components. @command{make check} traverses all +subdirectories in @file{src}. For every subdirectory you should +get a message like this: @example make[2]: Entering directory `/home/$USER/gnunet/contrib' @@ -269,24 +340,29 @@ PASS: test_gnunet_prefix GNUnet is organized in layers and services. Each service is composed of a main service implementation and a client library for other programs to use -the service's functionality, described by an API. This approach is shown in -@c** FIXME: enable this once the commented block below works: -@c** figure~\ref{fig:service}. -Some services provide an additional command line tool to enable the user to -interact with the service. - -Very often it is other GNUnet services that will use these APIs to build the -higher layers of GNUnet on top of the lower ones. Each layer expands or extends -the functionality of the service below (for instance, to build a mesh on top of -a DHT). -@c** FXIME: See comment above. -@c** See figure ~\ref{fig:interaction} for an illustration of this approach. - -@c ** @image{filename[, width[, height[, alttext[, extension]]]]} - -@image{images/gnunet-tutorial-service,,5in,Service with API and network protocol,.png} - -@image{images/gnunet-tutorial-system,,5in,The layered system architecture of GNUnet,.png} +the service's functionality, described by an API. +@c This approach is shown in +@c FIXME: enable this once the commented block below works: +@c figure~\ref fig:service. +Some services provide an additional command line tool to enable the user +to interact with the service. + +Very often it is other GNUnet services that will use these APIs to build +the higher layers of GNUnet on top of the lower ones. Each layer expands +or extends the functionality of the service below (for instance, to build +a mesh on top of a DHT). +@c FXIME: See comment above. +@c See figure ~\ref fig:interaction for an illustration of this approach. + +@c ** @image filename[, width[, height[, alttext[, extension]]]] +@c FIXME: Texlive (?) 20112 makes the assumption that this means +@c 'images/OBJECTNAME.txt' but later versions of it (2017) use this +@c syntax as described below. +@c TODO: Checkout the makedoc script Guile uses. + +@c FIXME!!! +@c @image{images/gnunet-tutorial-service,,5in,Service with API and network protocol,.png} +@c @image{images/gnunet-tutorial-system,,5in,The layered system architecture of GNUnet,.png} @c \begin{figure}[!h] @c \begin{center} @@ -308,11 +384,11 @@ a DHT). @c \caption{GNUnet's layered system architecture} @c \end{figure} -The main service implementation runs as a standalone process in the operating -system and the client code runs as part of the client program, so crashes of a -client do not affect the service process or other clients. The service and the -clients communicate via a message protocol to be defined and implemented by -the programmer. +The main service implementation runs as a standalone process in the +operating system and the client code runs as part of the client program, +so crashes of a client do not affect the service process or other clients. +The service and the clients communicate via a message protocol to be +defined and implemented by the programmer. @node First Steps with GNUnet @chapter First Steps with GNUnet @@ -328,43 +404,51 @@ the programmer. @node Configure your peer @section Configure your peer -First of all we need to configure your peer. Each peer is started with a configuration -containing settings for GNUnet itself and its services. This configuration is based on the -default configuration shipped with GNUnet and can be modified. The default configuration -is located in the @file{$PREFIX/share/gnunet/config.d} directory. When starting a peer, you -can specify a customized configuration using the the @command{-c} command line switch when -starting the ARM service and all other services. When using a modified configuration the -default values are loaded and only values specified in the configuration file will replace -the default values. - -Since we want to start additional peers later, we need some modifications from the default -configuration. We need to create a separate service home and a file containing our -modifications for this peer: +First of all we need to configure your peer. Each peer is started with +a configuration containing settings for GNUnet itself and its services. +This configuration is based on the default configuration shipped with +GNUnet and can be modified. The default configuration is located in the +@file{$PREFIX/share/gnunet/config.d} directory. When starting a peer, you +can specify a customized configuration using the the @command{-c} command +line switch when starting the ARM service and all other services. When +using a modified configuration the default values are loaded and only +values specified in the configuration file will replace the default +values. + +Since we want to start additional peers later, we need some modifications +from the default configuration. We need to create a separate service +home and a file containing our modifications for this peer: + @example $ mkdir ~/gnunet1/ $ touch peer1.conf @end example -Now add the following lines to @file{peer1.conf} to use this directory. For -simplified usage we want to prevent the peer to connect to the GNUnet +@noindent +Now add the following lines to @file{peer1.conf} to use this directory. +For simplified usage we want to prevent the peer to connect to the GNUnet network since this could lead to confusing output. This modifications will replace the default settings: + @example [PATHS] -GNUNET_HOME = ~/gnunet1/ # Use this directory to store GNUnet data +# Use this directory to store GNUnet data +GNUNET_HOME = ~/gnunet1/ [hostlist] -SERVERS = # prevent bootstrapping +# prevent bootstrapping +SERVERS = @end example @node Start a peer @section Start a peer Each GNUnet instance (called peer) has an identity (peer ID) based on a -cryptographic public private key pair. The peer ID is the printable hash of the -public key. +cryptographic public private key pair. The peer ID is the printable hash +of the public key. -GNUnet services are controlled by a master service, the so called @dfn{Automatic Restart Manager} (ARM). -ARM starts, stops and even restarts services automatically or on demand when a client connects. -You interact with the ARM service using the gnunet-arm tool. +GNUnet services are controlled by a master service, the so called +@dfn{Automatic Restart Manager} (ARM). ARM starts, stops and even +restarts services automatically or on demand when a client connects. +You interact with the ARM service using the @command{gnunet-arm} tool. GNUnet can then be started with @command{gnunet-arm -s} and stopped with @command{gnunet-arm -e}. An additional service not automatically started can be started using @command{gnunet-arm -i <service name>} and stopped @@ -372,11 +456,16 @@ using @command{gnunet-arm -k <servicename>}. Once you have started your peer, you can use many other GNUnet commands to interact with it. For example, you can run: + @example $ gnunet-peerinfo -s @end example + +@noindent to obtain the public key of your peer. + You should see an output containing the peer ID similar to: + @example I am peer `0PA02UVRKQTS2C .. JL5Q78F6H0B1ACPV1CJI59MEQUMQCC5G'. @end example @@ -384,33 +473,45 @@ I am peer `0PA02UVRKQTS2C .. JL5Q78F6H0B1ACPV1CJI59MEQUMQCC5G'. @node Monitor a peer @section Monitor a peer -In this section, we will monitor the behaviour of our peer's DHT service with respect to a -specific key. First we will start GNUnet and then start the DHT service and use the DHT monitor tool -to monitor the PUT and GET commands we issue ussing the @command{gnunet-dht-put} and -@command{gnunet-dht-get} commands. Using the ``monitor'' line given below, you can observe -the behavior of your own peer's DHT with respect to the specified KEY: +In this section, we will monitor the behaviour of our peer's DHT +service with respect to a specific key. First we will start +GNUnet and then start the DHT service and use the DHT monitor tool +to monitor the PUT and GET commands we issue ussing the +@command{gnunet-dht-put} and @command{gnunet-dht-get} commands. +Using the ``monitor'' line given below, you can observe the behavior +of your own peer's DHT with respect to the specified KEY: @example -$ gnunet-arm -c ~/peer1.conf -s # start gnunet with all default services -$ gnunet-arm -c ~/peer1.conf -i dht # start DHT service +# start gnunet with all default services: +$ gnunet-arm -c ~/peer1.conf -s +# start DHT service: +$ gnunet-arm -c ~/peer1.conf -i dht $ cd ~/gnunet/src/dht; $ ./gnunet-dht-monitor -c ~/peer1.conf -k KEY @end example -Now open a separate terminal and change again to the @file{gnunet/src/dht} directory: + +@noindent +Now open a separate terminal and change again to +the @file{gnunet/src/dht} directory: + @example $ cd ~/gnunet/src/dht -$ ./gnunet-dht-put -c ~/peer1.conf -k KEY -d VALUE # put VALUE under KEY in the DHT -$ ./gnunet/src/dht/gnunet-dht-get -c ~/peer1.conf -k KEY # get key KEY from the DHT -$ gnunet-statistics -c ~/peer1.conf # print statistics about current GNUnet state -$ gnunet-statistics -c ~/peer1.conf -s dht # print statistics about DHT service +# put VALUE under KEY in the DHT: +$ ./gnunet-dht-put -c ~/peer1.conf -k KEY -d VALUE +# get key KEY from the DHT: +$ ./gnunet/src/dht/gnunet-dht-get -c ~/peer1.conf -k KEY +# print statistics about current GNUnet state: +$ gnunet-statistics -c ~/peer1.conf +# print statistics about DHT service: +$ gnunet-statistics -c ~/peer1.conf -s dht @end example @node Starting Two Peers by Hand @section Starting Two Peers by Hand This section describes how to start two peers on the same machine by hand. -The process is rather painful, but the description is somewhat instructive. -In practice, you might prefer the automated method +The process is rather painful, but the description is somewhat +instructive. In practice, you might prefer the automated method (@pxref{Starting Peers Using the Testbed Service}). @menu @@ -424,29 +525,37 @@ In practice, you might prefer the automated method We will now start a second peer on your machine. For the second peer, you will need to manually create a modified configuration file to avoid conflicts with ports and directories. -A peers configuration file is by default located in @file{~/.gnunet/gnunet.conf}. -This file is typically very short or even empty as only the differences to the -defaults need to be specified. The defaults are located in -many files in the @file{$PREFIX/share/gnunet/config.d} directory. +A peers configuration file is by default located +in @file{~/.gnunet/gnunet.conf}. This file is typically very short +or even empty as only the differences to the defaults need to be +specified. The defaults are located in many files in the +@file{$PREFIX/share/gnunet/config.d} directory. To configure the second peer, use the files @file{$PREFIX/share/gnunet/config.d} as a template for your main configuration file: + @example $ cat $PREFIX/share/gnunet/config.d/*.conf > peer2.conf @end example + +@noindent Now you have to edit @file{peer2.conf} and change: + @itemize @item @code{GNUNET\_TEST\_HOME} under @code{PATHS} @item Every (uncommented) value for ``@code{PORT}'' (add 10000) in any section (the option may be commented out if @code{PORT} is prefixed by "\#", in this case, UNIX domain sockets are used and the PORT option does not need to be touched) -@item Every value for ``@code{UNIXPATH}'' in any section (e.g. by adding a "-p2" suffix) +@item Every value for ``@code{UNIXPATH}'' in any section +(e.g. by adding a "-p2" suffix) @end itemize -to a fresh, unique value. Make sure that the PORT numbers stay below 65536. -From now on, whenever you interact with the second peer, you need to specify -@command{-c peer2.conf} as an additional command line argument. + +to a fresh, unique value. Make sure that the PORT numbers stay +below 65536. From now on, whenever you interact with the second peer, +you need to specify @command{-c peer2.conf} as an additional +command line argument. Now, generate the 2nd peer's private key: @@ -454,6 +563,7 @@ Now, generate the 2nd peer's private key: $ gnunet-peerinfo -s -c peer2.conf @end example +@noindent This may take a while, generate entropy using your keyboard or mouse as needed. Also, make sure the output is different from the gnunet-peerinfo output for the first peer (otherwise you made an @@ -463,34 +573,46 @@ error in the configuration). @subsection Start the second peer and connect the peers Then, you can start a second peer using: + @example $ gnunet-arm -c peer2.conf -s $ gnunet-arm -c peer2.conf -i dht $ ~/gnunet/src/dht/gnunet-dht-put -c peer2.conf -k KEY -d VALUE $ ~/gnunet/src/dht/gnunet-dht-get -c peer2.conf -k KEY @end example + If you want the two peers to connect, you have multiple options: + @itemize @item UDP neighbour discovery (automatic) @item Setup a bootstrap server @item Connect manually @end itemize + To setup peer 1 as bootstrapping server change the configuration of the first one to be a hostlist server by adding the following lines to @file{peer1.conf} to enable bootstrapping server: + @example [hostlist] OPTIONS = -p @end example -Then change @file{peer2.conf} and replace the ``@code{SERVERS}'' line in the ``@code{[hostlist]}'' section with +@noindent +Then change @file{peer2.conf} and replace the ``@code{SERVERS}'' +line in the ``@code{[hostlist]}'' section with ``@code{http://localhost:8080/}''. Restart both peers using: + @example -$ gnunet-arm -c peer1.conf -e # stop first peer -$ gnunet-arm -c peer1.conf -s # start first peer -$ gnunet-arm -c peer2.conf -s # start second peer +# stop first peer +$ gnunet-arm -c peer1.conf -e +# start first peer +$ gnunet-arm -c peer1.conf -s +# start second peer +$ gnunet-arm -c peer2.conf -s @end example +@noindent Note that if you start your peers without changing these settings, they will use the ``global'' hostlist servers of the GNUnet P2P network and likely connect to those peers. At that point, debugging might become @@ -501,16 +623,23 @@ by you. @node How to connect manually @subsection How to connect manually -If you want to use the @code{peerinfo} tool to connect your peers, you should: +If you want to use the @code{peerinfo} tool to connect your +peers, you should: + @itemize -@item Set @code{FORCESTART = NO} in section @code{hostlist} (to not connect to the global GNUnet) -@item Start both peers running @command{gnunet-arm -c peer1.conf -s} and @command{gnunet-arm -c peer2.conf -s} -@item Get @code{HELLO} message of the first peer running @command{gnunet-peerinfo -c peer1.conf -g} -@item Give the output to the second peer by running @command{gnunet-peerinfo -c peer2.conf -p '<output>'} +@item Set @code{FORCESTART = NO} in section @code{hostlist} +(to not connect to the global GNUnet) +@item Start both peers running @command{gnunet-arm -c peer1.conf -s} +and @command{gnunet-arm -c peer2.conf -s} +@item Get @code{HELLO} message of the first peer running +@command{gnunet-peerinfo -c peer1.conf -g} +@item Give the output to the second peer by running +@command{gnunet-peerinfo -c peer2.conf -p '<output>'} @end itemize Check that they are connected using @command{gnunet-core -c peer1.conf}, which should give you the other peer's peer identity: + @example $ gnunet-core -c peer1.conf Peer `9TVUCS8P5A7ILLBGO6 [...shortened...] 1KNBJ4NGCHP3JPVULDG' @@ -520,90 +649,110 @@ Peer `9TVUCS8P5A7ILLBGO6 [...shortened...] 1KNBJ4NGCHP3JPVULDG' @section Starting Peers Using the Testbed Service @c \label{sec:testbed} -GNUnet's testbed service is used for testing scenarios where a number of peers -are to be started. The testbed can manage peers on a single host or on multiple -hosts in a distributed fashion. On a single affordable computer, it should be -possible to run around tens of peers without drastically increasing the load on the +GNUnet's testbed service is used for testing scenarios where +a number of peers are to be started. The testbed can manage peers +on a single host or on multiple hosts in a distributed fashion. +On a single affordable computer, it should be possible to run +around tens of peers without drastically increasing the load on the system. The testbed service can be access through its API -@file{include/gnunet\_testbed\_service.h}. The API provides many routines for -managing a group of peers. It also provides a helper function -@code{GNUNET\_TESTBED\_test\_run()} to quickly setup a minimalistic testing -environment on a single host. - -This function takes a configuration file which will be used as a template -configuration for the peers. The testbed takes care of modifying relevant -options in the peers' configuration such as @code{SERVICEHOME}, @code{PORT}, @code{UNIXPATH} to -unique values so that peers run without running into conflicts. It also checks +@file{include/gnunet\_testbed\_service.h}. The API provides many +routines for managing a group of peers. It also provides a helper +function @code{GNUNET\_TESTBED\_test\_run()} to quickly setup a +minimalistic testing environment on a single host. + +This function takes a configuration file which will be used as a +template configuration for the peers. The testbed takes care of +modifying relevant options in the peers' configuration such as +@code{SERVICEHOME}, @code{PORT}, @code{UNIXPATH} to unique values +so that peers run without running into conflicts. It also checks and assigns the ports in configurations only if they are free. -Additionally, the testbed service also reads its options from the same -configuration file. Various available options and details about them can be -found in the testbed default configuration file @file{src/testbed/testbed.conf}. +Additionally, the testbed service also reads its options from the +same configuration file. Various available options and details +about them can be found in the testbed default configuration file +@file{src/testbed/testbed.conf}. With the testbed API, a sample test case can be structured as follows: + @example @verbatiminclude testbed_test.c @end example + +@noindent The source code for the above listing can be found at -@uref{https://gnunet.org/git/gnunet.git/tree/doc/testbed_test.c} -or in the @file{doc/} folder of your repository check-out. +@uref{https://gnunet.org/git/gnunet.git/tree/doc/ +documentation/testbed_test.c} +or in the @file{doc/documentation/} folder of your repository check-out. After installing GNUnet, the above source code can be compiled as: + @example $ export CPPFLAGS="-I/path/to/gnunet/headers" $ export LDFLAGS="-L/path/to/gnunet/libraries" -$ gcc $CPPFLAGS $LDFLAGS -o testbed-test testbed_test.c -lgnunettestbed -lgnunetdht -lgnunetutil -$ touch template.conf # Generate (empty) configuration -$ ./testbed-test # run it (press CTRL-C to stop) +$ gcc $CPPFLAGS $LDFLAGS -o testbed-test testbed_test.c \ + -lgnunettestbed -lgnunetdht -lgnunetutil +# Generate (empty) configuration +$ touch template.conf +# run it (press CTRL-C to stop) +$ ./testbed-test @end example -The @code{CPPFLAGS} and @code{LDFLAGS} are necessary if GNUnet is installed -into a different directory other than @file{/usr/local}. - -All of testbed API's peer management functions treat management actions as -operations and return operation handles. It is expected that the operations -begin immediately, but they may get delayed (to balance out load on the system). -The program using the API then has to take care of marking the operation as -``done'' so that its associated resources can be freed immediately and other -waiting operations can be executed. Operations will be canceled if they are + +@noindent +The @code{CPPFLAGS} and @code{LDFLAGS} are necessary if GNUnet +is installed into a different directory other than @file{/usr/local}. + +All of testbed API's peer management functions treat management +actions as operations and return operation handles. It is expected +that the operations begin immediately, but they may get delayed (to +balance out load on the system). The program using the API then has +to take care of marking the operation as ``done'' so that its +associated resources can be freed immediately and other waiting +operations can be executed. Operations will be canceled if they are marked as ``done'' before their completion. -An operation is treated as completed when it succeeds or fails. Completion of -an operation is either conveyed as events through @i{controller event callback} -or through respective operation completion callbacks. In functions -which support completion notification through both controller event callback and -operation completion callback, first the controller event callback will be -called. If the operation is not marked as done in that callback or if the -callback is given as NULL when creating the operation, the operation completion -callback will be called. The API documentation shows which event are to be -expected in the controller event notifications. It also documents any -exceptional behaviour. - -Once the peers are started, test cases often need to connect some of the peers' -services. Normally, opening a connect to a peer's service requires the peer's -configuration. While using testbed, the testbed automatically generates -per-peer configuration. Accessing those configurations directly through file -system is discouraged as their locations are dynamically created and will be -different among various runs of testbed. To make access to these configurations -easy, testbed API provides the function -@code{GNUNET\_TESTBED\_service\_connect()}. This function fetches the -configuration of a given peer and calls the @i{Connect Adapter}. -In the example code, it is the @code{dht\_ca}. A connect adapter is expected -to open the connection to the needed service by using the provided configuration -and return the created service connection handle. Successful connection to the -needed service is signaled through @code{service\_connect\_comp\_cb}. - -A dual to connect adapter is the @i{Disconnect Adapter}. This callback is -called after the connect adapter has been called when the operation from -@code{GNUNET\_TESTBED\_service\_connect()} is marked as ``done''. It has to -disconnect from the service with the provided service handle (@code{op\_result}). +An operation is treated as completed when it succeeds or fails. +Completion of an operation is either conveyed as events through +@dfn{controller event callback} or through respective +@dfn{operation completion callbacks}. +In functions which support completion notification +through both controller event callback and operation +completion callback, first the controller event callback will be +called. If the operation is not marked as done in that callback +or if the callback is given as NULL when creating the operation, +the operation completion callback will be called. The API +documentation shows which event are to be expected in the +controller event notifications. It also documents any exceptional +behaviour. + +Once the peers are started, test cases often need to connect +some of the peers' services. Normally, opening a connect to +a peer's service requires the peer's configuration. While using +testbed, the testbed automatically generates per-peer configuration. +Accessing those configurations directly through file system is +discouraged as their locations are dynamically created and will be +different among various runs of testbed. To make access to these +configurations easy, testbed API provides the function +@code{GNUNET\_TESTBED\_service\_connect()}. This function fetches +the configuration of a given peer and calls the @dfn{Connect Adapter}. +In the example code, it is the @code{dht\_ca}. A connect adapter is +expected to open the connection to the needed service by using the +provided configuration and return the created service connection handle. +Successful connection to the needed service is signaled through +@code{service\_connect\_comp\_cb}. + +A dual to connect adapter is the @dfn{Disconnect Adapter}. This callback +is called after the connect adapter has been called when the operation +from @code{GNUNET\_TESTBED\_service\_connect()} is marked as ``done''. +It has to disconnect from the service with the provided service +handle (@code{op\_result}). Exercise: Find out how many peers you can run on your system. Exercise: Find out how to create a 2D torus topology by changing the -options in the configuration file. See @uref{https://gnunet.org/supported-topologies} -Then use the DHT API to store and retrieve values in the -network. +options in the configuration file. +@xref{Supported Topologies, The GNUnet Reference Documentation ,, gnunet, The GNUnet Reference Documentation}, +then use the DHT API to store and retrieve values in the network. @node Developing Applications @chapter Developing Applications @@ -635,17 +784,21 @@ $ make install $ make check @end example -The GNUnet ext template includes examples and a working buildsystem for a new GNUnet service. -A common GNUnet service consists of the following parts which will be discussed in detail in the -remainder of this document. The functionality of a GNUnet service is implemented in: +@noindent +The GNUnet ext template includes examples and a working buildsystem +for a new GNUnet service. A common GNUnet service consists of the +following parts which will be discussed in detail in the remainder +of this document. The functionality of a GNUnet service is implemented in: @itemize @item the GNUnet service (gnunet-ext/src/ext/gnunet-service-ext.c) @item the client API (gnunet-ext/src/ext/ext_api.c) -@item the client application using the service API (gnunet-ext/src/ext/gnunet-ext.c) +@item the client application using the service API +(gnunet-ext/src/ext/gnunet-ext.c) @end itemize The interfaces for these entities are defined in: + @itemize @item client API interface (gnunet-ext/src/ext/ext.h) @item the service interface (gnunet-ext/src/include/gnunet_service_SERVICE.h) @@ -654,9 +807,11 @@ The interfaces for these entities are defined in: In addition the ext systems provides: + @itemize @item a test testing the API (gnunet-ext/src/ext/test_ext_api.c) -@item a configuration template for the service (gnunet-ext/src/ext/ext.conf.in) +@item a configuration template for the service +(gnunet-ext/src/ext/ext.conf.in) @end itemize @node Adapting the Template @@ -664,22 +819,24 @@ In addition the ext systems provides: The first step for writing any extension with a new service is to ensure that the @file{ext.conf.in} file contains entries for the -@code{UNIXPATH}, @code{PORT} and @code{BINARY} for the service in a section named after -the service. +@code{UNIXPATH}, @code{PORT} and @code{BINARY} for the service in a +section named after the service. -If you want to adapt the template rename the @file{ext.conf.in} to match your -services name, you have to modify the @code{AC\_OUTPUT} section in @file{configure.ac} -in the @file{gnunet-ext} root. +If you want to adapt the template rename the @file{ext.conf.in} to +match your services name, you have to modify the @code{AC\_OUTPUT} +section in @file{configure.ac} in the @file{gnunet-ext} root. @node Writing a Client Application @section Writing a Client Application When writing any client application (for example, a command-line -tool), the basic structure is to start with the @code{GNUNET\_PROGRAM\_run} -function. This function will parse command-line options, setup the scheduler -and then invoke the @code{run} function (with the remaining non-option arguments) -and a handle to the parsed configuration (and the configuration file name that was -used, which is typically not needed): +tool), the basic structure is to start with the +@code{GNUNET\_PROGRAM\_run} function. This function will parse +command-line options, setup the scheduler and then invoke the +@code{run} function (with the remaining non-option arguments) +and a handle to the parsed configuration (and the configuration +file name that was used, which is typically not needed): + @example @verbatiminclude tutorial-examples/001.c @end example @@ -697,6 +854,7 @@ Options can then be added easily by adding global variables and expanding the @code{options} array. For example, the following would add a string-option and a binary flag (defaulting to @code{NULL} and @code{GNUNET\_NO} respectively): + @example @verbatiminclude tutorial-examples/002.c @end example @@ -753,10 +911,12 @@ file). Before a client library can implement the application-specific protocol with the service, a connection must be created: + @example @verbatiminclude tutorial-examples/003.c @end example +@noindent As a result a @code{GNUNET\_MQ\_Handle} is returned which can to used henceforth to transmit messages to the service. The complete MQ API can be found in @file{gnunet\_mq\_lib.h}. @@ -769,27 +929,35 @@ there are errors communicating with the service. @node Sending messages @subsubsection Sending messages -In GNUnet, messages are always sent beginning with a @code{struct GNUNET\_MessageHeader} -in big endian format. This header defines the size and the type of the +In GNUnet, messages are always sent beginning with a +@code{struct GNUNET\_MessageHeader} in big endian format. +This header defines the size and the type of the message, the payload follows after this header. + @example @verbatiminclude tutorial-examples/004.c @end example +@noindent Existing message types are defined in @file{gnunet\_protocols.h}. A common way to create a message is with an envelope: + @example @verbatiminclude tutorial-examples/005.c @end example +@noindent Exercise: Define a message struct that includes a 32-bit unsigned integer in addition to the standard GNUnet MessageHeader. Add a C struct and define a fresh protocol number for your message. -Protocol numbers in gnunet-ext are defined in @file{gnunet-ext/src/include/gnunet_protocols_ext.h} +Protocol numbers in gnunet-ext are defined +in @file{gnunet-ext/src/include/gnunet_protocols_ext.h} -Exercise: Find out how you can determine the number of messages in a message queue. +Exercise: Find out how you can determine the number of messages +in a message queue. -Exercise: Find out how you can determine when a message you have queued was actually transmitted. +Exercise: Find out how you can determine when a message you +have queued was actually transmitted. Exercise: Define a helper function to transmit a 32-bit unsigned integer (as payload) to a service using some given client @@ -808,16 +976,19 @@ to actually process the message. Fixed size messages are fully checked by the MQ-logic, and thus only need to provide the handler to process the message. Note that the prefixes @code{check\_} and @code{handle\_} are mandatory. + @example @verbatiminclude tutorial-examples/006.c @end example +@noindent Exercise: Expand your helper function to receive a response message (for example, containing just the @code{struct GNUnet MessageHeader} without any payload). Upon receiving the service's response, you should call a callback provided to your helper function's API. -Exercise: Figure out where you can pass values to the closures (@code{cls}). +Exercise: Figure out where you can pass values to the +closures (@code{cls}). @node Writing a user interface @subsection Writing a user interface @@ -834,8 +1005,8 @@ command-line to the service. @node Writing a Service @section Writing a Service -Before you can test the client you've written so far, you'll need to also -implement the corresponding service. +Before you can test the client you've written so far, you'll +need to also implement the corresponding service. @menu * Code Placement:: @@ -845,21 +1016,25 @@ implement the corresponding service. @node Code Placement @subsection Code Placement -New services are placed in their own subdirectory under @file{gnunet/src}. -This subdirectory should contain the API implementation file @file{SERVICE\_api.c}, -the description of the client-service protocol @file{SERVICE.h} and P2P protocol +New services are placed in their own subdirectory under +@file{gnunet/src}. This subdirectory should contain the API +implementation file @file{SERVICE\_api.c}, the description of +the client-service protocol @file{SERVICE.h} and P2P protocol @file{SERVICE\_protocol.h}, the implementation of the service itself -@file{gnunet-service-SERVICE.h} and several files for tests, including test code -and configuration files. +@file{gnunet-service-SERVICE.h} and several files for tests, +including test code and configuration files. @node Starting a Service @subsection Starting a Service -The key API definition for creating a service is the @code{GNUNET\_SERVICE\_MAIN} macro: +The key API definition for creating a service is the +@code{GNUNET\_SERVICE\_MAIN} macro: + @example @verbatiminclude tutorial-examples/007.c @end example +@noindent In addition to the service name and flags, the macro takes three functions, typically called @code{run}, @code{client\_connect\_cb} and @code{client\_disconnect\_cb} as well as an array of message handlers @@ -867,10 +1042,12 @@ that will be called for incoming messages from clients. A minimal version of the three central service funtions would look like this: + @example @verbatiminclude tutorial-examples/008.c @end example +@noindent Exercise: Write a stub service that processes no messages at all in your code. Create a default configuration for it, integrate it with the build system and start the service from @@ -883,8 +1060,9 @@ Exercise: Figure out how to send messages from the service back to the client. Each handler function in the service @b{must} eventually (possibly in some -asynchronous continuation) call @code{GNUNET\_SERVICE\_client\_continue()}. -Only after this call additional messages from the same client may +asynchronous continuation) call +@code{GNUNET\_SERVICE\_client\_continue()}. Only after this call +additional messages from the same client may be processed. This way, the service can throttle processing messages from the same client. @@ -900,8 +1078,9 @@ FIXME: This section still needs to be updated to the lastest API! One of the most important services in GNUnet is the @code{CORE} service managing connections between peers and handling encryption between peers. -One of the first things any service that extends the P2P protocol typically does -is connect to the @code{CORE} service using: +One of the first things any service that extends the P2P protocol +typically does is connect to the @code{CORE} service using: + @example @verbatiminclude tutorial-examples/009.c @end example @@ -916,15 +1095,18 @@ is connect to the @code{CORE} service using: @node New P2P connections @subsection New P2P connections -Before any traffic with a different peer can be exchanged, the peer must be -known to the service. This is notified by the @code{CORE} @code{connects} callback, -which communicates the identity of the new peer to the service: +Before any traffic with a different peer can be exchanged, the peer must +be known to the service. This is notified by the @code{CORE} +@code{connects} callback, which communicates the identity of the new +peer to the service: + @example @verbatiminclude tutorial-examples/010.c @end example +@noindent Note that whatever you return from @code{connects} is given as the -@i{cls} argument to the message handlers for messages from +@code{cls} argument to the message handlers for messages from the respective peer. Exercise: Create a service that connects to the @code{CORE}. Then @@ -935,7 +1117,7 @@ callback is invoked. @subsection Receiving P2P Messages To receive messages from @code{CORE}, you pass the desired -@i{handlers} to the @code{GNUNET\_CORE\_connect()} function, +@code{handlers} to the @code{GNUNET\_CORE\_connect()} function, just as we showed for services. It is your responsibility to process messages fast enough or @@ -951,8 +1133,8 @@ the two peers are connected? Why? @node Sending P2P Messages @subsection Sending P2P Messages -You can transmit messages to other peers using the @i{mq} you were -given during the @code{connect} callback. Note that the @i{mq} +You can transmit messages to other peers using the @code{mq} you were +given during the @code{connect} callback. Note that the @code{mq} automatically is released upon @code{disconnect} and that you must not use it afterwards. @@ -969,23 +1151,27 @@ you stop the peer that is receiving your messages? @node End of P2P connections @subsection End of P2P connections -If a message handler returns @code{GNUNET\_SYSERR}, the remote peer shuts down or -there is an unrecoverable network disconnection, CORE notifies the service that -the peer disconnected. After this notification no more messages will be received -from the peer and the service is no longer allowed to send messages to the peer. +If a message handler returns @code{GNUNET\_SYSERR}, the remote +peer shuts down or there is an unrecoverable network +disconnection, CORE notifies the service that the peer disconnected. +After this notification no more messages will be received from the +peer and the service is no longer allowed to send messages to the peer. The disconnect callback looks like the following: + @example @verbatiminclude tutorial-examples/011.c @end example +@noindent Exercise: Fix your service to handle peer disconnects. @node Storing peer-specific data using the PEERSTORE service @section Storing peer-specific data using the PEERSTORE service -GNUnet's PEERSTORE service offers a persistorage for arbitrary peer-specific data. -Other GNUnet services can use the PEERSTORE to store, retrieve and monitor data records. -Each data record stored with PEERSTORE contains the following fields: +GNUnet's PEERSTORE service offers a persistorage for arbitrary +peer-specific data. Other GNUnet services can use the PEERSTORE +to store, retrieve and monitor data records. Each data record +stored with PEERSTORE contains the following fields: @itemize @item subsystem: Name of the subsystem responsible for the record. @@ -1000,8 +1186,8 @@ The first step is to start a connection to the PEERSTORE service: @verbatiminclude tutorial-examples/012.c @end example -The service handle @code{peerstore_handle} will be needed for all subsequent -PEERSTORE operations. +The service handle @code{peerstore_handle} will be needed for +all subsequent PEERSTORE operations. @menu * Storing records:: @@ -1014,36 +1200,46 @@ PEERSTORE operations. @subsection Storing records To store a new record, use the following function: + @example @verbatiminclude tutorial-examples/013.c @end example -The @code{options} parameter can either be @code{GNUNET_PEERSTORE_STOREOPTION_MULTIPLE} -which means that multiple values can be stored under the same key combination (subsystem, peerid, key), -or @code{GNUNET_PEERSTORE_STOREOPTION_REPLACE} which means that PEERSTORE will replace any -existing values under the given key combination (subsystem, peerid, key) with the new given value. +@noindent +The @code{options} parameter can either be +@code{GNUNET_PEERSTORE_STOREOPTION_MULTIPLE} which means that multiple +values can be stored under the same key combination +(subsystem, peerid, key), or @code{GNUNET_PEERSTORE_STOREOPTION_REPLACE} +which means that PEERSTORE will replace any existing values under the +given key combination (subsystem, peerid, key) with the new given value. -The continuation function @code{cont} will be called after the store request is successfully -sent to the PEERSTORE service. This does not guarantee that the record is successfully stored, only -that it was received by the service. +The continuation function @code{cont} will be called after the store +request is successfully sent to the PEERSTORE service. This does not +guarantee that the record is successfully stored, only that it was +received by the service. + +The @code{GNUNET_PEERSTORE_store} function returns a handle to the store +operation. This handle can be used to cancel the store operation only +before the continuation function is called: -The @code{GNUNET_PEERSTORE_store} function returns a handle to the store operation. This handle -can be used to cancel the store operation only before the continuation function is called: @example -void -GNUNET_PEERSTORE_store_cancel (struct GNUNET_PEERSTORE_StoreContext *sc); +@verbatiminclude tutorial-examples/013.1.c @end example @node Retrieving records @subsection Retrieving records To retrieve stored records, use the following function: + @example @verbatiminclude tutorial-examples/014.c @end example -The values of @code{peer} and @code{key} can be @code{NULL}. This allows the -iteration over values stored under any of the following key combinations: +@noindent +The values of @code{peer} and @code{key} can be @code{NULL}. This +allows the iteration over values stored under any of the following +key combinations: + @itemize @item (subsystem) @item (subsystem, peerid) @@ -1051,25 +1247,32 @@ iteration over values stored under any of the following key combinations: @item (subsystem, peerid, key) @end itemize -The @code{callback} function will be called once with each retrieved record and once -more with a @code{NULL} record to signal the end of results. +The @code{callback} function will be called once with each retrieved +record and once more with a @code{NULL} record to signal the end of +results. -The @code{GNUNET_PEERSTORE_iterate} function returns a handle to the iterate operation. This -handle can be used to cancel the iterate operation only before the callback function is called with -a @code{NULL} record. +The @code{GNUNET_PEERSTORE_iterate} function returns a handle to the +iterate operation. This handle can be used to cancel the iterate +operation only before the callback function is called with a +@code{NULL} record. @node Monitoring records @subsection Monitoring records -PEERSTORE offers the functionality of monitoring for new records stored under a specific key -combination (subsystem, peerid, key). To start the monitoring, use the following function: +PEERSTORE offers the functionality of monitoring for new records +stored under a specific key combination (subsystem, peerid, key). +To start the monitoring, use the following function: + @example @verbatiminclude tutorial-examples/015.c @end example -Whenever a new record is stored under the given key combination, the @code{callback} function -will be called with this new record. This will continue until the connection to the PEERSTORE service -is broken or the watch operation is canceled: +@noindent +Whenever a new record is stored under the given key combination, +the @code{callback} function will be called with this new +record. This will continue until the connection to the PEERSTORE +service is broken or the watch operation is canceled: + @example @verbatiminclude tutorial-examples/016.c @end example @@ -1077,15 +1280,18 @@ is broken or the watch operation is canceled: @node Disconnecting from PEERSTORE @subsection Disconnecting from PEERSTORE -When the connection to the PEERSTORE service is no longer needed, disconnect using the following -function: +When the connection to the PEERSTORE service is no longer needed, +disconnect using the following function: + @example @verbatiminclude tutorial-examples/017.c @end example -If the @code{sync_first} flag is set to @code{GNUNET_YES}, the API will delay the -disconnection until all store requests are received by the PEERSTORE service. Otherwise, -it will disconnect immediately. +@noindent +If the @code{sync_first} flag is set to @code{GNUNET_YES}, +the API will delay the disconnection until all store requests +are received by the PEERSTORE service. Otherwise, it will +disconnect immediately. @node Using the DHT @section Using the DHT @@ -1094,10 +1300,12 @@ The DHT allows to store data so other peers in the P2P network can access it and retrieve data stored by any peers in the network. This section will explain how to use the DHT. Of course, the first thing to do is to connect to the DHT service: + @example @verbatiminclude tutorial-examples/018.c @end example +@noindent The second parameter indicates how many requests in parallel to expect. It is not a hard limit, but a good approximation will make the DHT more efficient. @@ -1113,27 +1321,32 @@ efficient. @subsection Storing data in the DHT Since the DHT is a dynamic environment (peers join and leave frequently) the data that we put in the DHT does not stay there indefinitely. It is -important to ``refresh'' the data periodically by simply storing it again, -in order to make sure other peers can access it. +important to ``refresh'' the data periodically by simply storing it +again, in order to make sure other peers can access it. The put API call offers a callback to signal that the PUT request has been sent. This does not guarantee that the data is accessible to others peers, or even that is has been stored, only that the service has requested to a neighboring peer the retransmission of the PUT request towards its final destination. Currently there is no feedback about whether or not the data -has been sucessfully stored or where it has been stored. In order to improve -the availablilty of the data and to compensate for possible errors, peers leaving -and other unfavorable events, just make several PUT requests! +has been sucessfully stored or where it has been stored. In order to +improve the availablilty of the data and to compensate for possible +errors, peers leaving and other unfavorable events, just make several +PUT requests! + @example @verbatiminclude tutorial-examples/019.c @end example -Exercise: Store a value in the DHT periodically to make sure it is available -over time. You might consider using the function @code{GNUNET\_SCHEDULER\_add\_delayed} -and call @code{GNUNET\_DHT\_put} from inside a helper function. +@noindent +Exercise: Store a value in the DHT periodically to make sure it +is available over time. You might consider using the function +@code{GNUNET\_SCHEDULER\_add\_delayed} and call +@code{GNUNET\_DHT\_put} from inside a helper function. @node Obtaining data from the DHT @subsection Obtaining data from the DHT + As we saw in the previous example, the DHT works in an asynchronous mode. Each request to the DHT is executed ``in the background'' and the API calls return immediately. In order to receive results from the DHT, the @@ -1143,16 +1356,20 @@ duplicates) until the timeout expires or we explicitly stop the request. It is possible to give a ``forever'' timeout with @code{GNUNET\_TIME\_UNIT\_FOREVER\_REL}. -If we give a route option @code{GNUNET\_DHT\_RO\_RECORD\_ROUTE} the callback -will get a list of all the peers the data has travelled, both on the PUT -path and on the GET path. +If we give a route option @code{GNUNET\_DHT\_RO\_RECORD\_ROUTE} +the callback will get a list of all the peers the data has travelled, +both on the PUT path and on the GET path. + @example @verbatiminclude tutorial-examples/020.c @end example -Exercise: Store a value in the DHT and after a while retrieve it. Show the IDs of all -the peers the requests have gone through. In order to convert a peer ID to a string, use -the function @code{GNUNET\_i2s}. Pay attention to the route option parameters in both calls! +@noindent +Exercise: Store a value in the DHT and after a while retrieve it. +Show the IDs of all the peers the requests have gone through. +In order to convert a peer ID to a string, use the function +@code{GNUNET\_i2s}. Pay attention to the route option parameters +in both calls! @node Implementing a block plugin @subsection Implementing a block plugin @@ -1187,10 +1404,12 @@ the @code{xquery} argument is application-specific. Applications that do not use an extended query should check that the @code{xquery\_size} is zero. The block group is typically used to filter duplicate replies. + @example @verbatiminclude tutorial-examples/021.c @end example +@noindent Note that it is mandatory to detect duplicate replies in this function and return the respective status code. Duplicate detection is typically done using the Bloom filter block group provided by @@ -1206,6 +1425,7 @@ function is used to obtain the key of a block --- for example, by means of hashing. If deriving the key is not possible, the function should simply return @code{GNUNET\_SYSERR} (the DHT will still work just fine with such blocks). + @example @verbatiminclude tutorial-examples/022.c @end example @@ -1218,6 +1438,7 @@ an initialization function which should initialize the plugin. The initialization function specifies what block types the plugin cares about and returns a struct with the functions that are to be used for validation and obtaining keys (the ones just defined above). + @example @verbatiminclude tutorial-examples/023.c @end example @@ -1228,6 +1449,7 @@ validation and obtaining keys (the ones just defined above). Following GNUnet's general plugin API concept, the plugin must export a second function for cleaning up. It usually does very little. + @example @verbatiminclude tutorial-examples/024.c @end example @@ -1237,28 +1459,34 @@ little. In order to compile the plugin, the @file{Makefile.am} file for the service SERVICE should contain a rule similar to this: -@c* Actually this is a Makefile not C. But the whole structure of examples -@c* must be improved. +@c Actually this is a Makefile not C. But the whole structure of examples +@c must be improved. + @example @verbatiminclude tutorial-examples/025.c @end example +@noindent Exercise: Write a block plugin that accepts all queries and all replies but prints information about queries and replies when the respective validation hooks are called. @node Monitoring the DHT @subsection Monitoring the DHT -It is possible to monitor the functioning of the local DHT service. When monitoring -the DHT, the service will alert the monitoring program of any events, -both started locally or received for routing from another peer. The are three different -types of events possible: a GET request, a PUT request or a response (a reply to -a GET). - -Since the different events have different associated data, the API gets 3 -different callbacks (one for each message type) and optional type and key parameters, -to allow for filtering of messages. When an event happens, the appropiate callback -is called with all the information about the event. + +It is possible to monitor the functioning of the local +DHT service. When monitoring the DHT, the service will +alert the monitoring program of any events, both started +locally or received for routing from another peer. +The are three different types of events possible: a +GET request, a PUT request or a response (a reply to a GET). + +Since the different events have different associated data, +the API gets 3 different callbacks (one for each message type) +and optional type and key parameters, to allow for filtering of +messages. When an event happens, the appropiate callback is +called with all the information about the event. + @example @verbatiminclude tutorial-examples/026.c @end example @@ -1266,17 +1494,20 @@ is called with all the information about the event. @node Debugging with gnunet-arm @section Debugging with gnunet-arm -Even if services are managed by @command{gnunet-arm}, you can start them with -@command{gdb} or @command{valgrind}. For example, you could add the following lines -to your configuration file to start the DHT service in a @command{gdb} session in a -fresh @command{xterm}: +Even if services are managed by @command{gnunet-arm}, you can +start them with @command{gdb} or @command{valgrind}. For +example, you could add the following lines to your +configuration file to start the DHT service in a @command{gdb} +session in a fresh @command{xterm}: @example [dht] PREFIX=xterm -e gdb --args @end example -Alternatively, you can stop a service that was started via ARM and run it manually: +@noindent +Alternatively, you can stop a service that was started via +ARM and run it manually: @example $ gnunet-arm -k dht @@ -1284,20 +1515,23 @@ $ gdb --args gnunet-service-dht -L DEBUG $ valgrind gnunet-service-dht -L DEBUG @end example -Assuming other services are well-written, they will automatically re-integrate the -restarted service with the peer. +@noindent +Assuming other services are well-written, they will automatically +re-integrate the restarted service with the peer. -GNUnet provides a powerful logging mechanism providing log levels @code{ERROR}, -@code{WARNING}, @code{INFO} and @code{DEBUG}. The current log level is -configured using the @code{$GNUNET_FORCE_LOG} environmental variable. -The @code{DEBUG} level is only available if @command{--enable-logging=verbose} was used when -running @command{configure}. More details about logging can be found under +GNUnet provides a powerful logging mechanism providing log +levels @code{ERROR}, @code{WARNING}, @code{INFO} and @code{DEBUG}. +The current log level is configured using the @code{$GNUNET_FORCE_LOG} +environmental variable. The @code{DEBUG} level is only available if +@command{--enable-logging=verbose} was used when running +@command{configure}. More details about logging can be found under @uref{https://gnunet.org/logging}. You should also probably enable the creation of core files, by setting -@code{ulimit}, and echo'ing @code{1} into @file{/proc/sys/kernel/core\_uses\_pid}. -Then you can investigate the core dumps with @command{gdb}, which is often -the fastest method to find simple errors. +@code{ulimit}, and echo'ing @code{1} into +@file{/proc/sys/kernel/core\_uses\_pid}. Then you can investigate the +core dumps with @command{gdb}, which is often the fastest method to +find simple errors. Exercise: Add a memory leak to your service and obtain a trace pointing to the leak using @command{valgrind} while running the service diff --git a/doc/gnunet.texi b/doc/documentation/gnunet.texi index 7803e52d3d..35eed54b6a 100644 --- a/doc/gnunet.texi +++ b/doc/documentation/gnunet.texi @@ -5,15 +5,17 @@ @setfilename gnunet.info @documentencoding UTF-8 @settitle GNUnet Reference Manual +@exampleindent 2 +@urefbreakstyle before @c %**end of header @include version.texi @c Set Versions which might be used in more than one place: -@set GNUNET-DIST-URL https://ftp.gnu.org/gnu/gnunet/ -@set GNUNET-VERSION 0.10.1 +@set GNUFTP-URL https://ftp.gnu.org/gnu/gnunet +@set PYPI-URL https://pypi.python.org/packages/source @set GNURL-VERSION-CURRENT 7.55.1 -@set GNURL-DIST-URL https://gnunet.org/sites/default/files/ +@set GNUNET-DIST-URL https://gnunet.org/sites/default/files/ @c @set OPENPGP-SIGNING-KEY-ID @copying @@ -38,6 +40,13 @@ A copy of the license is also available from the Free Software Foundation Web site at @url{http://www.gnu.org/licenses/gpl.html}. @end copying +@c TODO: Improve this and improve https://directory.fsf.org/wiki/Gnunet + +@dircategory Networking +@direntry +* GNUnet: (gnunet). Framework for secure peer-to-peer networking +@end direntry + @titlepage @title GNUnet Reference Manual @subtitle Installing, configuring, using and contributing to GNUnet @@ -51,60 +60,27 @@ Edition @value{EDITION} @* @insertcopying @end titlepage +@summarycontents @contents -@c ********************************************************************* -@node Top -@top Contributing to GNUnet -@c ********************************************************************* - -This document describes GNUnet version @value{VERSION}. - - -GNUnet is a @uref{http://www.gnu.org/, GNU} package. All code contributions -must thus be put under the -@uref{http://www.gnu.org/copyleft/gpl.html, GNU Public License (GPL)}. -All documentation should be put under FSF approved licenses -(see @uref{http://www.gnu.org/copyleft/fdl.html, fdl}). - -By submitting documentation, translations, comments and other content to this -website you automatically grant the right to publish code under the -GNU Public License and documentation under either or both the GNU -Public License or the GNU Free Documentation License. When contributing -to the GNUnet project, GNU standards and the -@uref{http://www.gnu.org/philosophy/philosophy.html, GNU philosophy} -should be adhered to. - -Note that we do now require a formal copyright assignment for GNUnet -contributors to GNUnet e.V.; nevertheless, we do allow pseudonymous -contributions. By signing the copyright agreement and submitting your -code (or documentation) to us, you agree to share the rights to your -code with GNUnet e.V.; GNUnet e.V. receives non-exclusive ownership -rights, and in particular is allowed to dual-license the code. You -retain non-exclusive rights to your contributions, so you can also -share your contributions freely with other projects. - -GNUnet e.V. will publish all accepted contributions under the GPLv3 or any -later version. The association may decide to publish contributions under -additional licenses (dual-licensing). - -We do not intentionally remove your name from your contributions; however, -due to extensive editing it is not always trivial to attribute contributors -properly. If you find that you significantly contributed to a file (or the -project as a whole) and are not listed in the respective authors file or -section, please do let us know. +@node Top +@top Introduction +This document is the Reference Manual for GNUnet version @value{VERSION}. @menu * Philosophy:: About GNUnet +* Vocabulary:: Vocabulary * GNUnet Installation Handbook:: How to install GNUnet * Using GNUnet:: Using GNUnet +* Configuration Handbook:: Configuring GNUnet +* GNUnet Contributors Handbook:: Contributing to GNUnet * GNUnet Developer Handbook:: Developing GNUnet -* GNU Free Documentation License:: The license of this manual. -* GNU General Public License:: The license of this manual. -* Concept Index:: Concepts. -* Programming Index:: Data types, functions, and variables. +* GNU Free Documentation License:: The license of this manual +* GNU General Public License:: The license of this manual +* Concept Index:: Concepts +* Programming Index:: Data types, functions, and variables @detailmenu --- The Detailed Node Listing --- @@ -112,7 +88,7 @@ section, please do let us know. Philosophy * Design Goals:: -* Security & Privacy:: +* Security and Privacy:: * Versatility:: * Practicality:: * Key Concepts:: @@ -127,10 +103,86 @@ Philosophy * Backup of Identities and Egos:: * Revocation:: -Installation +Vocabulary + +* Definitions abbreviations and acronyms:: +* Words and characters:: +* Technical Assumptions:: + +GNUnet Installation Handbook * Dependencies:: -* External dependencies:: +* Pre-installation notes:: +* Generic installation instructions:: +* Build instructions for Ubuntu 12.04 using Git:: +* Build instructions for software builds from source:: +* Build Instructions for Microsoft Windows Platforms:: +* Build instructions for Debian 7.5:: +* Installing GNUnet from Git on Ubuntu 14.4:: +* Build instructions for Debian 8:: +* Outdated build instructions for previous revisions:: +@c * Portable GNUnet:: +* The graphical configuration interface:: +* How to start and stop a GNUnet peer:: + +Using GNUnet + +* Checking the Installation:: +* First steps - File-sharing:: +* First steps - Using the GNU Name System:: +* First steps - Using GNUnet Conversation:: +* First steps - Using the GNUnet VPN:: +* File-sharing:: +* The GNU Name System:: +* Using the Virtual Public Network:: + +Configuration Handbook + +GNUnet Contributors Handbook + +* Contributing to GNUnet:: +* Licenses of contributions:: +* Copyright Assignment:: +* Contributing to the Reference Manual:: + +GNUnet Developer Handbook + +* Developer Introduction:: +* Code overview:: +* System Architecture:: +* Subsystem stability:: +* Naming conventions and coding style guide:: +* Build-system:: +* Developing extensions for GNUnet using the gnunet-ext template:: +* Writing testcases:: +* TESTING library:: +* Performance regression analysis with Gauger:: +* TESTBED Subsystem:: +* libgnunetutil:: +* Automatic Restart Manager (ARM):: +* TRANSPORT Subsystem:: +* NAT library:: +* Distance-Vector plugin:: +* SMTP plugin:: +* Bluetooth plugin:: +* WLAN plugin:: +* ATS Subsystem:: +* CORE Subsystem:: +* CADET Subsystem:: +* NSE Subsystem:: +* HOSTLIST Subsystem:: +* IDENTITY Subsystem:: +* NAMESTORE Subsystem:: +* PEERINFO Subsystem:: +* PEERSTORE Subsystem:: +* SET Subsystem:: +* STATISTICS Subsystem:: +* Distributed Hash Table (DHT):: +* GNU Name System (GNS):: +* GNS Namecache:: +* REVOCATION Subsystem:: +* File-sharing (FS) Subsystem:: +* REGEX Subsystem:: @end detailmenu @end menu @@ -139,6 +191,8 @@ Installation @include chapters/philosophy.texi @c ********************************************************************* +@include chapters/vocabulary.texi + @c ********************************************************************* @include chapters/installation.texi @c ********************************************************************* @@ -147,11 +201,15 @@ Installation @include chapters/user.texi @c ********************************************************************* +@include chapters/configuration.texi + +@include chapters/contributing.texi + @c ********************************************************************* @include chapters/developer.texi +@c @include gnunet-c-tutorial.texi @c ********************************************************************* - @c ********************************************************************* @node GNU Free Documentation License @appendix GNU Free Documentation License diff --git a/doc/gpl-3.0.texi b/doc/documentation/gpl-3.0.texi index 0e2e212acb..0e2e212acb 100644 --- a/doc/gpl-3.0.texi +++ b/doc/documentation/gpl-3.0.texi diff --git a/doc/documentation/htmlxref.cnf b/doc/documentation/htmlxref.cnf new file mode 100644 index 0000000000..a4928f6fee --- /dev/null +++ b/doc/documentation/htmlxref.cnf @@ -0,0 +1,668 @@ +# htmlxref.cnf - reference file for free Texinfo manuals on the web. +# Modified by Ludovic Courtès <ludo@gnu.org> for the GNU Guix manual. +# Modified by ng0 <ng0@gnunet.org> for the GNUnet manual. + +htmlxrefversion=2017-10-26.06; # UTC + +# Copyright 2010, 2011, 2012, 2013, 2014, 2015 Free Software Foundation, Inc. +# +# Copying and distribution of this file, with or without modification, +# are permitted in any medium without royalty provided the copyright +# notice and this notice are preserved. +# +# The latest version of this file is available at +# http://ftpmirror.gnu.org/texinfo/htmlxref.cnf. +# Email corrections or additions to bug-texinfo@gnu.org. +# The primary goal is to list all relevant GNU manuals; +# other free manuals are also welcome. +# +# To be included in this list, a manual must: +# +# - have a generic url, e.g., no version numbers; +# - have a unique file name (e.g., manual identifier), i.e., be related to the +# package name. Things like "refman" or "tutorial" don't work. +# - follow the naming convention for nodes described at +# http://www.gnu.org/software/texinfo/manual/texinfo/html_node/HTML-Xref.html +# This is what makeinfo and texi2html implement. +# +# Unless the above criteria are met, it's not possible to generate +# reliable cross-manual references. +# +# For information on automatically generating all the useful formats for +# a manual to put on the web, see +# http://www.gnu.org/prep/maintain/html_node/Manuals-on-Web-Pages.html. + +# For people editing this file: when a manual named foo is related to a +# package named bar, the url should contain a variable reference ${BAR}. +# Otherwise, the gnumaint scripts have no way of knowing they are +# associated, and thus gnu.org/manual can't include them. + +# shorten references to manuals on www.gnu.org. +G = https://www.gnu.org +GS = ${G}/software + +3dldf mono ${GS}/3dldf/manual/user_ref/3DLDF.html +3dldf node ${GS}/3dldf/manual/user_ref/ + +alive mono ${GS}/alive/manual/alive.html +alive node ${GS}/alive/manual/html_node/ + +anubis chapter ${GS}/anubis/manual/html_chapter/ +anubis section ${GS}/anubis/manual/html_section/ +anubis node ${GS}/anubis/manual/html_node/ + +artanis mono ${GS}/artanis/manual/artanis.html +artanis node ${GS}/artanis/manual/html_node/ + +aspell section http://aspell.net/man-html/index.html + +auctex mono ${GS}/auctex/manual/auctex.html +auctex node ${GS}/auctex/manual/auctex/ + +autoconf mono ${GS}/autoconf/manual/autoconf.html +autoconf node ${GS}/autoconf/manual/html_node/ + +autogen mono ${GS}/autogen/manual/html_mono/autogen.html +autogen chapter ${GS}/autogen/manual/html_chapter/ +autogen node ${GS}/autoconf/manual/html_node/ + +automake mono ${GS}/automake/manual/automake.html +automake node ${GS}/automake/manual/html_node/ + +avl node http://www.stanford.edu/~blp/avl/libavl.html/ + +bash mono ${GS}/bash/manual/bash.html +bash node ${GS}/bash/manual/html_node/ + +BINUTILS = http://sourceware.org/binutils/docs +binutils node ${BINUTILS}/binutils/ + as node ${BINUTILS}/as/ + bfd node ${BINUTILS}/bfd/ + gprof node ${BINUTILS}/gprof/ + ld node ${BINUTILS}/ld/ + +bison mono ${GS}/bison/manual/bison.html +bison node ${GS}/bison/manual/html_node/ + +bpel2owfn mono ${GS}/bpel2owfn/manual/2.0.x/bpel2owfn.html + +ccd2cue mono ${GS}/ccd2cue/manual/ccd2cue.html +ccd2cue node ${GS}/ccd2cue/manual/html_node/ + +cflow mono ${GS}/cflow/manual/cflow.html +cflow node ${GS}/cflow/manual/html_node/ + +chess mono ${GS}/chess/manual/gnuchess.html +chess node ${GS}/chess/manual/html_node/ + +combine mono ${GS}/combine/manual/combine.html +combine chapter ${GS}/combine/manual/html_chapter/ +combine section ${GS}/combine/manual/html_section/ +combine node ${GS}/combine/manual/html_node/ + +complexity mono ${GS}/complexity/manual/complexity.html +complexity node ${GS}/complexity/manual/html_node/ + +coreutils mono ${GS}/coreutils/manual/coreutils +coreutils node ${GS}/coreutils/manual/html_node/ + +cpio mono ${GS}/cpio/manual/cpio +cpio node ${GS}/cpio/manual/html_node/ + +cssc node ${GS}/cssc/manual/ + +#cvs cannot be handled here; see http://ximbiot.com/cvs/manual. + +ddd mono ${GS}/ddd/manual/html_mono/ddd.html + +ddrescue mono ${GS}/ddrescue/manual/ddrescue_manual.html + +DICO = http://puszcza.gnu.org.ua/software/dico/manual +dico mono ${DICO}/dico.html +dico chapter ${DICO}/html_chapter/ +dico section ${DICO}/html_section/ +dico node ${DICO}/html_node/ + +diffutils mono ${GS}/diffutils/manual/diffutils +diffutils node ${GS}/diffutils/manual/html_node/ + +ed mono ${GS}/ed/manual/ed_manual.html + +EMACS = ${GS}/emacs/manual +emacs mono ${EMACS}/html_mono/emacs.html +emacs node ${EMACS}/html_node/emacs/ + # + ada-mode mono ${EMACS}/html_mono/ada-mode.html + ada-mode node ${EMACS}/html_node/ada-mode/ + # + autotype mono ${EMACS}/html_mono/autotype.html + autotype node ${EMACS}/html_node/autotype/ + # + ccmode mono ${EMACS}/html_mono/ccmode.html + ccmode node ${EMACS}/html_node/ccmode/ + # + cl mono ${EMACS}/html_mono/cl.html + cl node ${EMACS}/html_node/cl/ + # + ebrowse mono ${EMACS}/html_mono/ebrowse.html + ebrowse node ${EMACS}/html_node/ebrowse/ + # + ediff mono ${EMACS}/html_mono/ediff.html + ediff node ${EMACS}/html_node/ediff/ + # + eieio mono ${EMACS}/html_mono/eieio.html + eieio node ${EMACS}/html_node/eieio/ + # + elisp mono ${EMACS}/html_mono/elisp.html + elisp node ${EMACS}/html_node/elisp/ + # + epa mono ${EMACS}/html_mono/epa.html + epa node ${EMACS}/html_node/epa/ + # + erc mono ${EMACS}/html_mono/erc.html + erc node ${EMACS}/html_node/erc/ + # + dired-x mono ${EMACS}/html_mono/dired-x.html + dired-x node ${EMACS}/html_node/dired-x/ + # + eshell mono ${EMACS}/html_mono/eshell.html + eshell node ${EMACS}/html_node/eshell/ + # + flymake mono ${EMACS}/html_mono/flymake.html + flymake node ${EMACS}/html_node/flymake/ + # + gnus mono ${EMACS}/html_mono/gnus.html + gnus node ${EMACS}/html_node/gnus/ + # + idlwave mono ${EMACS}/html_mono/idlwave.html + idlwave node ${EMACS}/html_node/idlwave/ + # + message mono ${EMACS}/html_mono/message.html + message node ${EMACS}/html_node/message/ + # + mh-e mono ${EMACS}/html_mono/mh-e.html + mh-e node ${EMACS}/html_node/mh-e/ + # + nxml-mode mono ${EMACS}/html_mono/nxml-mode.html + nxml-mode node ${EMACS}/html_node/nxml-mode/ + # + org mono ${EMACS}/html_mono/org.html + org node ${EMACS}/html_node/org/ + # + pcl-cvs mono ${EMACS}/html_mono/pcl-cvs.html + pcl-cvs node ${EMACS}/html_node/pcl-cvs/ + # + rcirc mono ${EMACS}/html_mono/rcirc.html + rcirc node ${EMACS}/html_node/rcirc/ + # + semantic mono ${EMACS}/html_mono/semantic.html + semantic node ${EMACS}/html_node/semantic/ + # + smtp mono ${EMACS}/html_mono/smtpmail.html + smtp node ${EMACS}/html_node/smtpmail/ + # + speedbar mono ${EMACS}/html_mono/speedbar.html + speedbar node ${EMACS}/html_node/speedbar/ + # + tramp mono ${EMACS}/html_mono/tramp.html + tramp node ${EMACS}/html_node/tramp/ + # + vip mono ${EMACS}/html_mono/vip.html + vip node ${EMACS}/html_node/vip/ + # + viper mono ${EMACS}/html_mono/viper.html + viper node ${EMACS}/html_node/viper/ + # + woman mono ${EMACS}/html_mono/woman.html + woman node ${EMACS}/html_node/woman/ + # (end emacs manuals) + +easejs mono ${GS}/easejs/manual/easejs.html +easejs node ${GS}/easejs/manual/ + +EMACS_GUIX = https://alezost.github.io/guix.el/manual/latest +emacs-guix mono ${EMACS_GUIX}/emacs-guix.html +emacs-guix node ${EMACS_GUIX}/html_node/ + +emacs-muse node ${GS}/emacs-muse/manual/muse.html +emacs-muse node ${GS}/emacs-muse/manual/html_node/ + +emms node ${GS}/emms/manual/ + +# The file is called 'find.info' but the package is 'findutils'. +find mono ${GS}/findutils/manual/html_mono/find.html +find node ${GS}/findutils/manual/html_node/find_html +findutils mono ${GS}/findutils/manual/html_mono/find.html +findutils node ${GS}/findutils/manual/html_node/find_html + +FLEX = http://flex.sourceforge.net +flex node ${FLEX}/manual/ + +gama mono ${GS}/gama/manual/gama.html +gama node ${GS}/gama/manual/html_node/ + +GAWK = ${GS}/gawk/manual +gawk mono ${GAWK}/gawk.html +gawk node ${GAWK}/html_node/ + gawkinet mono ${GAWK}/gawkinet/gawkinet.html + gawkinet node ${GAWK}/gawkinet/html_node/ + +gcal mono ${GS}/gcal/manual/gcal.html +gcal node ${GS}/gcal/manual/html_node/ + +GCC = http://gcc.gnu.org/onlinedocs +gcc node ${GCC}/gcc/ + cpp node ${GCC}/cpp/ + gcj node ${GCC}/gcj/ + gfortran node ${GCC}/gfortran/ + gnat_rm node ${GCC}/gnat_rm/ + gnat_ugn_unw node ${GCC}/gnat_ugn_unw/ + libgomp node ${GCC}/libgomp/ + libstdc++ node ${GCC}/libstdc++/ + # + gccint node ${GCC}/gccint/ + cppinternals node ${GCC}/cppinternals/ + gfc-internals node ${GCC}/gfc-internals/ + gnat-style node ${GCC}/gnat-style/ + libiberty node ${GCC}/libiberty/ + +GDB = http://sourceware.org/gdb/current/onlinedocs +gdb node ${GDB}/gdb/ + stabs node ${GDB}/stabs/ + +GDBM = http://www.gnu.org.ua/software/gdbm/manual +gdbm mono ${GDBM}/gdbm.html +gdbm chapter ${GDBM}/html_chapter/ +gdbm section ${GDBM}/html_section/ +gdbm node ${GDBM}/html_node/ + +gettext mono ${GS}/gettext/manual/gettext.html +gettext node ${GS}/gettext/manual/html_node/ + +gforth node http://www.complang.tuwien.ac.at/forth/gforth/Docs-html/ + +global mono ${GS}/global/manual/global.html + +gmediaserver node ${GS}/gmediaserver/manual/ + +gmp node http://www.gmplib.org/manual/ + +gnu-arch node ${GS}/gnu-arch/tutorial/ + +gnu-c-manual mono ${GS}/gnu-c-manual/gnu-c-manual.html + +gnu-crypto node ${GS}/gnu-crypto/manual/ + +gnubg mono ${GS}/gnubg/manual/gnubg.html +gnubg node ${GS}/gnubg/manual/html_node/ + +gnubik mono ${GS}/gnubik/manual/gnubik.html +gnubik node ${GS}/gnubik/manual/html_node/ + +gnulib mono ${GS}/gnulib/manual/gnulib.html +gnulib node ${GS}/gnulib/manual/html_node/ + +GNUN = ${GS}/trans-coord/manual +gnun mono ${GNUN}/gnun/gnun.html +gnun node ${GNUN}/gnun/html_node/ + web-trans mono ${GNUN}/web-trans/web-trans.html + web-trans node ${GNUN}/web-trans/html_node/ + +GNUNET = https://docs.gnunet.org/manuals +gnunet node ${GNUNET}/gnunet/ + gnunet-c-tutorial node ${GNUNET}/gnunet-c-tutorial/ + gnunet-java-tutorial node ${GNUNET}/gnunet-java-tutorial/ + +GNUPG = http://www.gnupg.org/documentation/manuals +gnupg node ${GNUPG}/gnupg/ + dirmngr node ${GNUPG}/dirmngr/ + gcrypt node ${GNUPG}/gcrypt/ + libgcrypt node ${GNUPG}/gcrypt/ + ksba node ${GNUPG}/ksba/ + assuan node ${GNUPG}/assuan/ + gpgme node ${GNUPG}/gpgme/ + +gnuprologjava node ${GS}/gnuprologjava/manual/ + +gnuschool mono ${GS}/gnuschool/gnuschool.html + +GNUSTANDARDS = ${G}/prep + maintain mono ${GNUSTANDARDS}/maintain/maintain.html + maintain node ${GNUSTANDARDS}/maintain/html_node/ + # + standards mono ${GNUSTANDARDS}/standards/standards.html + standards node ${GNUSTANDARDS}/standards/html_node/ + +gnutls mono http://gnutls.org/manual/gnutls.html +gnutls node http://gnutls.org/manual/html_node/ + +gnutls-guile mono http://gnutls.org/manual/gnutls-guile.html +gnutls-guile node http://gnutls.org/manual/gnutls-guile/ + +gperf mono ${GS}/gperf/manual/gperf.html +gperf node ${GS}/gperf/manual/html_node/ + +grep mono ${GS}/grep/manual/grep.html +grep node ${GS}/grep/manual/html_node/ + +groff node ${GS}/groff/manual/html_node/ + +GRUB = ${GS}/grub/manual + grub mono ${GRUB}/grub.html + grub node ${GRUB}/html_node/ + # + multiboot mono ${GRUB}/multiboot/multiboot.html + multiboot node ${GRUB}/multiboot/html_node/ + +gsasl mono ${GS}/gsasl/manual/gsasl.html +gsasl node ${GS}/gsasl/manual/html_node/ + +gsl node ${GS}/gsl/manual/html_node/ + +gsrc mono ${GS}/gsrc/manual/gsrc.html +gsrc node ${GS}/gsrc/manual/html_node/ + +gss mono ${GS}/gss/manual/gss.html +gss node ${GS}/gss/manual/html_node/ + +gtypist mono ${GS}/gtypist/doc/ + +guile mono ${GS}/guile/manual/guile.html +guile node ${GS}/guile/manual/html_node/ + +guile-avahi mono http://nongnu.org/guile-avahi/doc/guile-avahi.html + +GUILE_GNOME = ${GS}/guile-gnome/docs + gobject node ${GUILE_GNOME}/gobject/html/ + glib node ${GUILE_GNOME}/glib/html/ + atk node ${GUILE_GNOME}/atk/html/ + pango node ${GUILE_GNOME}/pango/html/ + pangocairo node ${GUILE_GNOME}/pangocairo/html/ + gdk node ${GUILE_GNOME}/gdk/html/ + gtk node ${GUILE_GNOME}/gtk/html/ + libglade node ${GUILE_GNOME}/libglade/html/ + gnome-vfs node ${GUILE_GNOME}/gnome-vfs/html/ + libgnomecanvas node ${GUILE_GNOME}/libgnomecanvas/html/ + gconf node ${GUILE_GNOME}/gconf/html/ + libgnome node ${GUILE_GNOME}/libgnome/html/ + libgnomeui node ${GUILE_GNOME}/libgnomeui/html/ + corba node ${GUILE_GNOME}/corba/html/ + clutter node ${GUILE_GNOME}/clutter/html/ + clutter-glx node ${GUILE_GNOME}/clutter-glx/html/ + +guile-gtk node ${GS}/guile-gtk/docs/guile-gtk/ + +guile-rpc mono ${GS}/guile-rpc/manual/guile-rpc.html +guile-rpc node ${GS}/guile-rpc/manual/html_node/ + +guix mono ${GS}/guix/manual/guix.html +guix node ${GS}/guix/manual/html_node/ + +gv mono ${GS}/gv/manual/gv.html +gv node ${GS}/gv/manual/html_node/ + +gzip mono ${GS}/gzip/manual/gzip.html +gzip node ${GS}/gzip/manual/html_node/ + +hello mono ${GS}/hello/manual/hello.html +hello node ${GS}/hello/manual/html_node/ + +help2man mono ${GS}/help2man/help2man.html + +idutils mono ${GS}/idutils/manual/idutils.html +idutils node ${GS}/idutils/manual/html_node/ + +inetutils mono ${GS}/inetutils/manual/inetutils.html +inetutils node ${GS}/inetutils/manual/html_node/ + +jwhois mono ${GS}/jwhois/manual/jwhois.html +jwhois node ${GS}/jwhois/manual/html_node/ + +libc mono ${GS}/libc/manual/html_mono/libc.html +libc node ${GS}/libc/manual/html_node/ + +LIBCDIO = ${GS}/libcdio + libcdio mono ${LIBCDIO}/libcdio.html + cd-text mono ${LIBCDIO}/cd-text-format.html + +libextractor mono ${GS}/libextractor/manual/libextractor.html +libextractor node ${GS}/libextractor/manual/html_node/ + +libidn mono ${GS}/libidn/manual/libidn.html +libidn node ${GS}/libidn/manual/html_node/ + +librejs mono ${GS}/librejs/manual/librejs.html +librejs node ${GS}/librejs/manual/html_node/ + +libmatheval mono ${GS}/libmatheval/manual/libmatheval.html + +LIBMICROHTTPD = ${GS}/libmicrohttpd +libmicrohttpd mono ${LIBMICROHTTPD}/manual/libmicrohttpd.html +libmicrohttpd node ${LIBMICROHTTPD}/manual/html_node/ + microhttpd-tutorial mono ${LIBMICROHTTPD}/tutorial.html + +libtasn1 mono ${GS}/libtasn1/manual/libtasn1.html +libtasn1 node ${GS}/libtasn1/manual/html_node/ + +libtool mono ${GS}/libtool/manual/libtool.html +libtool node ${GS}/libtool/manual/html_node/ + +lightning mono ${GS}/lightning/manual/lightning.html +lightning node ${GS}/lightning/manual/html_node/ + +# The stable/ url redirects immediately, but that's ok. +# The .html extension is omitted on their web site, but it works if given. +LILYPOND = http://lilypond.org/doc/stable/Documentation + lilypond-internals node ${LILYPOND}/internals/ + lilypond-learning node ${LILYPOND}/learning/ + lilypond-notation node ${LILYPOND}/notation/ + lilypond-snippets node ${LILYPOND}/snippets/ + lilypond-usage node ${LILYPOND}/usage/ + lilypond-web node ${LILYPOND}/web/ + music-glossary node ${LILYPOND}/music-glossary/ + +liquidwar6 mono ${GS}/liquidwar6/manual/liquidwar6.html +liquidwar6 node ${GS}/liquidwar6/manual/html_node/ + +lispintro mono ${GS}/emacs/emacs-lisp-intro/html_mono/emacs-lisp-intro.html +lispintro node ${GS}/emacs/emacs-lisp-intro/html_node/index.html + +LSH = http://www.lysator.liu.se/~nisse/lsh + lsh mono ${LSH}/lsh.html + +m4 mono ${GS}/m4/manual/m4.html +m4 node ${GS}/m4/manual/html_node/ + +mailutils mono ${GS}/mailutils/manual/mailutils.html +mailutils chapter ${GS}/mailutils/manual/html_chapter/ +mailutils section ${GS}/mailutils/manual/html_section/ +mailutils node ${GS}/mailutils/manual/html_node/ + +make mono ${GS}/make/manual/make.html +make node ${GS}/make/manual/html_node/ + +mcron mono ${GS}/mcron/manual/mcron.html +mcron node ${GS}/mcron/manual/html_node/ + +mdk mono ${GS}/mdk/manual/mdk.html +mdk node ${GS}/mdk/manual/html_node/ + +METAEXCHANGE = http://ftp.gwdg.de/pub/gnu2/iwfmdh/doc/texinfo + iwf_mh node ${METAEXCHANGE}/iwf_mh.html + scantest node ${METAEXCHANGE}/scantest.html + +MIT_SCHEME = ${GS}/mit-scheme/documentation + mit-scheme-ref node ${MIT_SCHEME}/mit-scheme-ref/ + mit-scheme-user node ${MIT_SCHEME}/mit-scheme-user/ + sos node ${MIT_SCHEME}/mit-scheme-sos/ + mit-scheme-imail node ${MIT_SCHEME}/mit-scheme-imail/ + +moe mono ${GS}/moe/manual/moe_manual.html + +motti node ${GS}/motti/manual/ + +mpc node http://www.multiprecision.org/index.php?prog=mpc&page=html + +mpfr mono http://www.mpfr.org/mpfr-current/mpfr.html + +mtools mono ${GS}/mtools/manual/mtools.html + +myserver node http://www.myserverproject.net/documentation/ + +nano mono http://www.nano-editor.org/dist/latest/nano.html + +nettle chapter http://www.lysator.liu.se/~nisse/nettle/nettle.html + +ocrad mono ${GS}/ocrad/manual/ocrad_manual.html + +parted mono ${GS}/parted/manual/parted.html +parted node ${GS}/parted/manual/html_node/ + +pascal mono http://www.gnu-pascal.de/gpc/ + +# can't use pcb since url's contain dates --30nov10 + +perl mono ${GS}/perl/manual/perldoc-all.html + +PIES = http://www.gnu.org.ua/software/pies/manual +pies mono ${PIES}/pies.html +pies chapter ${PIES}/html_chapter/ +pies section ${PIES}/html_section/ +pies node ${PIES}/html_node/ + +plotutils mono ${GS}/plotutils/manual/en/plotutils.html +plotutils node ${GS}/plotutils/manual/en/html_node/ + +proxyknife mono ${GS}/proxyknife/manual/proxyknife.html +proxyknife node ${GS}/proxyknife/manual/html_node/ + +pspp mono ${GS}/pspp/manual/pspp.html +pspp node ${GS}/pspp/manual/html_node/ + +pyconfigure mono ${GS}/pyconfigure/manual/pyconfigure.html +pyconfigure node ${GS}/pyconfigure/manual/html_node/ + +R = http://cran.r-project.org/doc/manuals + R-intro mono ${R}/R-intro.html + R-lang mono ${R}/R-lang.html + R-exts mono ${R}/R-exts.html + R-data mono ${R}/R-data.html + R-admin mono ${R}/R-admin.html + R-ints mono ${R}/R-ints.html + +rcs mono ${GS}/rcs/manual/rcs.html +rcs node ${GS}/rcs/manual/html_node/ + +READLINE = http://cnswww.cns.cwru.edu/php/chet/readline +readline mono ${READLINE}/readline.html + rluserman mono ${READLINE}/rluserman.html + history mono ${READLINE}/history.html + +recode mono http://recode.progiciels-bpi.ca/manual/index.html + +recutils mono ${GS}/recutils/manual/recutils.html +recutils node ${GS}/recutils/manual/html_node/ + +reftex mono ${GS}/auctex/manual/reftex.html +reftex node ${GS}/auctex/manual/reftex/ + +remotecontrol mono ${GS}/remotecontrol/manual/remotecontrol.html +remotecontrol node ${GS}/remotecontrol/manual/html_node/ + +rottlog mono ${GS}/rottlog/manual/rottlog.html +rottlog node ${GS}/rottlog/manual/html_node/ + +RUSH = http://www.gnu.org.ua/software/rush/manual +rush mono ${RUSH}/rush.html +rush chapter ${RUSH}/html_chapter/ +rush section ${RUSH}/html_section/ +rush node ${RUSH}/html_node/ + +screen mono ${GS}/screen/manual/screen.html +screen node ${GS}/screen/manual/html_node/ + +sed mono ${GS}/sed/manual/sed.html +sed node ${GS}/sed/manual/html_node/ + +sharutils mono ${GS}/sharutils/manual/html_mono/sharutils.html +sharutils chapter ${GS}/sharutils/manual/html_chapter/ +sharutils node ${GS}/sharutils/manual/html_node/ + +shepherd mono ${GS}/shepherd/manual/shepherd.html +shepherd node ${GS}/shepherd/manual/html_node/ + +# can't use mono files since they have generic names +SMALLTALK = ${GS}/smalltalk +smalltalk node ${SMALLTALK}/manual/html_node/ + smalltalk-base node ${SMALLTALK}/manual-base/html_node/ + smalltalk-libs node ${SMALLTALK}/manual-libs/html_node/ + +sourceinstall mono ${GS}/sourceinstall/manual/sourceinstall.html +sourceinstall node ${GS}/sourceinstall/manual/html_node/ + +sqltutor mono ${GS}/sqltutor/manual/sqltutor.html +sqltutor node ${GS}/sqltutor/manual/html_node/ + +src-highlite mono ${GS}/src-highlite/source-highlight.html + +swbis mono ${GS}/swbis/manual.html + +tar mono ${GS}/tar/manual/tar.html +tar chapter ${GS}/tar/manual/html_chapter/ +tar section ${GS}/tar/manual/html_section/ +tar node ${GS}/autoconf/manual/html_node/ + +teseq mono ${GS}/teseq/teseq.html +teseq node ${GS}/teseq/html_node/ + +TEXINFO = ${GS}/texinfo/manual +texinfo mono ${TEXINFO}/texinfo/texinfo.html +texinfo node ${TEXINFO}/texinfo/html_node/ + # + info mono ${TEXINFO}/info/info.html + info node ${TEXINFO}/info/html_node/ + # + info-stnd mono ${TEXINFO}/info-stnd/info-stnd.html + info-stnd node ${TEXINFO}/info-stnd/html_node/ + +thales node ${GS}/thales/manual/ + +units mono ${GS}/units/manual/units.html +units node ${GS}/units/manual/html_node/ + +vc-dwim mono ${GS}/vc-dwim/manual/vc-dwim.html +vc-dwim node ${GS}/vc-dwim/manual/html_node/ + +wdiff mono ${GS}/wdiff/manual/wdiff.html +wdiff node ${GS}/wdiff/manual/html_node/ + +websocket4j mono ${GS}/websocket4j/manual/websocket4j.html +websocket4j node ${GS}/websocket4j/manual/html_node/ + +wget mono ${GS}/wget/manual/wget.html +wget node ${GS}/wget/manual/html_node/ + +xboard mono ${GS}/xboard/manual/xboard.html +xboard node ${GS}/xboard/manual/html_node/ + +# emacs-page +# Free TeX-related Texinfo manuals on tug.org. + +T = http://tug.org/texinfohtml + +dvipng mono ${T}/dvipng.html +dvips mono ${T}/dvips.html +eplain mono ${T}/eplain.html +kpathsea mono ${T}/kpathsea.html +latex2e mono ${T}/latex2e.html +tlbuild mono ${T}/tlbuild.html +web2c mono ${T}/web2c.html + + +# Local Variables: +# eval: (add-hook 'write-file-hooks 'time-stamp) +# time-stamp-start: "htmlxrefversion=" +# time-stamp-format: "%:y-%02m-%02d.%02H" +# time-stamp-time-zone: "UTC" +# time-stamp-end: "; # UTC" +# End: diff --git a/doc/images/daemon_lego_block.png b/doc/documentation/images/daemon_lego_block.png Binary files differindex 5a088b5321..5a088b5321 100644 --- a/doc/images/daemon_lego_block.png +++ b/doc/documentation/images/daemon_lego_block.png diff --git a/doc/images/daemon_lego_block.svg b/doc/documentation/images/daemon_lego_block.svg index 38ad90d134..38ad90d134 100644 --- a/doc/images/daemon_lego_block.svg +++ b/doc/documentation/images/daemon_lego_block.svg diff --git a/doc/images/gnunet-0-10-peerinfo.png b/doc/documentation/images/gnunet-0-10-peerinfo.png Binary files differindex c5e711aff5..c5e711aff5 100644 --- a/doc/images/gnunet-0-10-peerinfo.png +++ b/doc/documentation/images/gnunet-0-10-peerinfo.png diff --git a/doc/images/gnunet-fs-gtk-0-10-star-tab.png b/doc/documentation/images/gnunet-fs-gtk-0-10-star-tab.png Binary files differindex d7993cc468..d7993cc468 100644 --- a/doc/images/gnunet-fs-gtk-0-10-star-tab.png +++ b/doc/documentation/images/gnunet-fs-gtk-0-10-star-tab.png diff --git a/doc/images/gnunet-gtk-0-10-download-area.png b/doc/documentation/images/gnunet-gtk-0-10-download-area.png Binary files differindex 8500d46c99..8500d46c99 100644 --- a/doc/images/gnunet-gtk-0-10-download-area.png +++ b/doc/documentation/images/gnunet-gtk-0-10-download-area.png diff --git a/doc/images/gnunet-gtk-0-10-fs-menu.png b/doc/documentation/images/gnunet-gtk-0-10-fs-menu.png Binary files differindex dc20c45a95..dc20c45a95 100644 --- a/doc/images/gnunet-gtk-0-10-fs-menu.png +++ b/doc/documentation/images/gnunet-gtk-0-10-fs-menu.png diff --git a/doc/images/gnunet-gtk-0-10-fs-publish-editing.png b/doc/documentation/images/gnunet-gtk-0-10-fs-publish-editing.png Binary files differindex 6f9f75ea6d..6f9f75ea6d 100644 --- a/doc/images/gnunet-gtk-0-10-fs-publish-editing.png +++ b/doc/documentation/images/gnunet-gtk-0-10-fs-publish-editing.png diff --git a/doc/images/gnunet-gtk-0-10-fs-publish-select.png b/doc/documentation/images/gnunet-gtk-0-10-fs-publish-select.png Binary files differindex 50672e379a..50672e379a 100644 --- a/doc/images/gnunet-gtk-0-10-fs-publish-select.png +++ b/doc/documentation/images/gnunet-gtk-0-10-fs-publish-select.png diff --git a/doc/images/gnunet-gtk-0-10-fs-publish-with-file.png b/doc/documentation/images/gnunet-gtk-0-10-fs-publish-with-file.png Binary files differindex b465425630..b465425630 100644 --- a/doc/images/gnunet-gtk-0-10-fs-publish-with-file.png +++ b/doc/documentation/images/gnunet-gtk-0-10-fs-publish-with-file.png diff --git a/doc/images/gnunet-gtk-0-10-fs-publish-with-file_0.png b/doc/documentation/images/gnunet-gtk-0-10-fs-publish-with-file_0.png Binary files differindex b465425630..b465425630 100644 --- a/doc/images/gnunet-gtk-0-10-fs-publish-with-file_0.png +++ b/doc/documentation/images/gnunet-gtk-0-10-fs-publish-with-file_0.png diff --git a/doc/images/gnunet-gtk-0-10-fs-publish.png b/doc/documentation/images/gnunet-gtk-0-10-fs-publish.png Binary files differindex 033b38fa5b..033b38fa5b 100644 --- a/doc/images/gnunet-gtk-0-10-fs-publish.png +++ b/doc/documentation/images/gnunet-gtk-0-10-fs-publish.png diff --git a/doc/images/gnunet-gtk-0-10-fs-published.png b/doc/documentation/images/gnunet-gtk-0-10-fs-published.png Binary files differindex fbd3dd6a35..fbd3dd6a35 100644 --- a/doc/images/gnunet-gtk-0-10-fs-published.png +++ b/doc/documentation/images/gnunet-gtk-0-10-fs-published.png diff --git a/doc/images/gnunet-gtk-0-10-fs-search.png b/doc/documentation/images/gnunet-gtk-0-10-fs-search.png Binary files differindex bb64ab92e8..bb64ab92e8 100644 --- a/doc/images/gnunet-gtk-0-10-fs-search.png +++ b/doc/documentation/images/gnunet-gtk-0-10-fs-search.png diff --git a/doc/images/gnunet-gtk-0-10-fs.png b/doc/documentation/images/gnunet-gtk-0-10-fs.png Binary files differindex c7a2948782..c7a2948782 100644 --- a/doc/images/gnunet-gtk-0-10-fs.png +++ b/doc/documentation/images/gnunet-gtk-0-10-fs.png diff --git a/doc/images/gnunet-gtk-0-10-gns-a-done.png b/doc/documentation/images/gnunet-gtk-0-10-gns-a-done.png Binary files differindex f8231b3a62..f8231b3a62 100644 --- a/doc/images/gnunet-gtk-0-10-gns-a-done.png +++ b/doc/documentation/images/gnunet-gtk-0-10-gns-a-done.png diff --git a/doc/images/gnunet-gtk-0-10-gns-a.png b/doc/documentation/images/gnunet-gtk-0-10-gns-a.png Binary files differindex 39858d72c1..39858d72c1 100644 --- a/doc/images/gnunet-gtk-0-10-gns-a.png +++ b/doc/documentation/images/gnunet-gtk-0-10-gns-a.png diff --git a/doc/images/gnunet-gtk-0-10-gns.png b/doc/documentation/images/gnunet-gtk-0-10-gns.png Binary files differindex c71a2bd7be..c71a2bd7be 100644 --- a/doc/images/gnunet-gtk-0-10-gns.png +++ b/doc/documentation/images/gnunet-gtk-0-10-gns.png diff --git a/doc/images/gnunet-gtk-0-10-identity.png b/doc/documentation/images/gnunet-gtk-0-10-identity.png Binary files differindex d0b426098a..d0b426098a 100644 --- a/doc/images/gnunet-gtk-0-10-identity.png +++ b/doc/documentation/images/gnunet-gtk-0-10-identity.png diff --git a/doc/images/gnunet-gtk-0-10-search-selected.png b/doc/documentation/images/gnunet-gtk-0-10-search-selected.png Binary files differindex da1ad4d31e..da1ad4d31e 100644 --- a/doc/images/gnunet-gtk-0-10-search-selected.png +++ b/doc/documentation/images/gnunet-gtk-0-10-search-selected.png diff --git a/doc/images/gnunet-gtk-0-10-traffic.png b/doc/documentation/images/gnunet-gtk-0-10-traffic.png Binary files differindex 76458f717a..76458f717a 100644 --- a/doc/images/gnunet-gtk-0-10-traffic.png +++ b/doc/documentation/images/gnunet-gtk-0-10-traffic.png diff --git a/doc/images/gnunet-gtk-0-10.png b/doc/documentation/images/gnunet-gtk-0-10.png Binary files differindex 3615849a79..3615849a79 100644 --- a/doc/images/gnunet-gtk-0-10.png +++ b/doc/documentation/images/gnunet-gtk-0-10.png diff --git a/doc/images/gnunet-namestore-gtk-phone.png b/doc/documentation/images/gnunet-namestore-gtk-phone.png Binary files differindex 3bb859629a..3bb859629a 100644 --- a/doc/images/gnunet-namestore-gtk-phone.png +++ b/doc/documentation/images/gnunet-namestore-gtk-phone.png diff --git a/doc/images/gnunet-namestore-gtk-vpn.png b/doc/documentation/images/gnunet-namestore-gtk-vpn.png Binary files differindex c716729ba2..c716729ba2 100644 --- a/doc/images/gnunet-namestore-gtk-vpn.png +++ b/doc/documentation/images/gnunet-namestore-gtk-vpn.png diff --git a/doc/images/gnunet-setup-exit.png b/doc/documentation/images/gnunet-setup-exit.png Binary files differindex 66bd972bc0..66bd972bc0 100644 --- a/doc/images/gnunet-setup-exit.png +++ b/doc/documentation/images/gnunet-setup-exit.png diff --git a/doc/images/gnunet-tutorial-service.png b/doc/documentation/images/gnunet-tutorial-service.png Binary files differindex 6daed2f35f..6daed2f35f 100644 --- a/doc/images/gnunet-tutorial-service.png +++ b/doc/documentation/images/gnunet-tutorial-service.png diff --git a/doc/images/gnunet-tutorial-system.png b/doc/documentation/images/gnunet-tutorial-system.png Binary files differindex 8b54e16cf2..8b54e16cf2 100644 --- a/doc/images/gnunet-tutorial-system.png +++ b/doc/documentation/images/gnunet-tutorial-system.png diff --git a/doc/images/iceweasel-preferences.png b/doc/documentation/images/iceweasel-preferences.png Binary files differindex e62c2c4d99..e62c2c4d99 100644 --- a/doc/images/iceweasel-preferences.png +++ b/doc/documentation/images/iceweasel-preferences.png diff --git a/doc/images/iceweasel-proxy.png b/doc/documentation/images/iceweasel-proxy.png Binary files differindex 9caad4508d..9caad4508d 100644 --- a/doc/images/iceweasel-proxy.png +++ b/doc/documentation/images/iceweasel-proxy.png diff --git a/doc/images/lego_stack.svg b/doc/documentation/images/lego_stack.svg index a0e8017c39..a0e8017c39 100644 --- a/doc/images/lego_stack.svg +++ b/doc/documentation/images/lego_stack.svg diff --git a/doc/images/service_lego_block.png b/doc/documentation/images/service_lego_block.png Binary files differindex 56caf6b9cc..56caf6b9cc 100644 --- a/doc/images/service_lego_block.png +++ b/doc/documentation/images/service_lego_block.png diff --git a/doc/images/service_lego_block.svg b/doc/documentation/images/service_lego_block.svg index ef0d0234fd..ef0d0234fd 100644 --- a/doc/images/service_lego_block.svg +++ b/doc/documentation/images/service_lego_block.svg diff --git a/doc/images/service_stack.png b/doc/documentation/images/service_stack.png Binary files differindex 747d087b20..747d087b20 100644 --- a/doc/images/service_stack.png +++ b/doc/documentation/images/service_stack.png diff --git a/doc/images/structure.dot b/doc/documentation/images/structure.dot index a53db90b80..a53db90b80 100644 --- a/doc/images/structure.dot +++ b/doc/documentation/images/structure.dot diff --git a/doc/documentation/index.html b/doc/documentation/index.html new file mode 100644 index 0000000000..0c3b04e9db --- /dev/null +++ b/doc/documentation/index.html @@ -0,0 +1,35 @@ +<title>GNUnet - GNUnet Manuals and Handbooks</title> +<h2>GNUnet - GNUnet Manuals and Handbooks</h2> + +<address>GNUnet e.V.</address> +<address>Fakultät für Informatik -- I8</address> +<address>Technische Universität München</address> +<address>Boltzmannstraße 3</address> +<address>85748 Garching</address> +<address>GERMANY</address> + +<p>The following handbooks and manuals are available:</p> + +<ul> +<li><a href="gnunet/index.html">GNUnet Reference Manual</li> +<li><a href="gnunet-c-tutorial/index.html">GNUnet C Tutorial</li> +</ul> + +<div id="footer"> +<div class="unprintable"> + +<p>Please send general FSF & GNU inquiries to +<a href="mailto:gnu@gnu.org"><gnu@gnu.org></a>. +There are also <a href="/contact/">other ways to contact</a> +the FSF. Broken links and other corrections or suggestions can be sent +to <a href="mailto:gnunet-developers@gnu.org"><gnunet-developers@gnu.org></a>.</p> +</div> + +<p>Copyright © 2001 - 2017 GNUnet e.V.</p> + +<p>This page is licensed under a FIXME License.</p> + +</div> +</div> +</body> +</html> diff --git a/doc/documentation/run-gendocs.sh b/doc/documentation/run-gendocs.sh new file mode 100755 index 0000000000..d025701772 --- /dev/null +++ b/doc/documentation/run-gendocs.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +make version.texi +make version2.texi +./gendocs.sh --email gnunet-developers@gnu.org gnunet-c-tutorial "GNUnet C Tutorial" -o "manual/gnunet-c-tutorial" +#cd manual +#mkdir gnunet-c-tutorial +#mv * gnunet-c-tutorial/ +#cd .. +./gendocs.sh --email gnunet-developers@gnu.org gnunet "GNUnet Reference Manual" -o "manual/gnunet" +#cd manual +#mkdir handbook +#mkdir ../tmp-gnunet +#mv gnunet ../tmp-gnunet +#mv * handbook/ +#mv ../tmp-gnunet gnunet +cp "index.html" manual/ +printf "Success" diff --git a/doc/testbed_test.c b/doc/documentation/testbed_test.c index 1696234b05..1696234b05 100644 --- a/doc/testbed_test.c +++ b/doc/documentation/testbed_test.c diff --git a/doc/tutorial-examples/001.c b/doc/documentation/tutorial-examples/001.c index 7f6699dd22..7f6699dd22 100644 --- a/doc/tutorial-examples/001.c +++ b/doc/documentation/tutorial-examples/001.c diff --git a/doc/tutorial-examples/002.c b/doc/documentation/tutorial-examples/002.c index 02233fd619..02233fd619 100644 --- a/doc/tutorial-examples/002.c +++ b/doc/documentation/tutorial-examples/002.c diff --git a/doc/documentation/tutorial-examples/003.c b/doc/documentation/tutorial-examples/003.c new file mode 100644 index 0000000000..d158d7e756 --- /dev/null +++ b/doc/documentation/tutorial-examples/003.c @@ -0,0 +1,11 @@ +struct GNUNET_MQ_MessageHandlers handlers[] = { + // ... + GNUNET_MQ_handler_end () +}; +struct GNUNET_MQ_Handle *mq; + +mq = GNUNET_CLIENT_connect (cfg, + "service-name", + handlers, + &error_cb, + NULL); diff --git a/doc/tutorial-examples/004.c b/doc/documentation/tutorial-examples/004.c index 0ef0079070..0ef0079070 100644 --- a/doc/tutorial-examples/004.c +++ b/doc/documentation/tutorial-examples/004.c diff --git a/doc/tutorial-examples/005.c b/doc/documentation/tutorial-examples/005.c index 0c459f509d..0c459f509d 100644 --- a/doc/tutorial-examples/005.c +++ b/doc/documentation/tutorial-examples/005.c diff --git a/doc/tutorial-examples/006.c b/doc/documentation/tutorial-examples/006.c index 944d2b18c7..944d2b18c7 100644 --- a/doc/tutorial-examples/006.c +++ b/doc/documentation/tutorial-examples/006.c diff --git a/doc/tutorial-examples/007.c b/doc/documentation/tutorial-examples/007.c index 096539e432..096539e432 100644 --- a/doc/tutorial-examples/007.c +++ b/doc/documentation/tutorial-examples/007.c diff --git a/doc/tutorial-examples/008.c b/doc/documentation/tutorial-examples/008.c index 2dffe2cf91..2dffe2cf91 100644 --- a/doc/tutorial-examples/008.c +++ b/doc/documentation/tutorial-examples/008.c diff --git a/doc/tutorial-examples/009.c b/doc/documentation/tutorial-examples/009.c index 26d918fb02..26d918fb02 100644 --- a/doc/tutorial-examples/009.c +++ b/doc/documentation/tutorial-examples/009.c diff --git a/doc/tutorial-examples/010.c b/doc/documentation/tutorial-examples/010.c index 33494490db..33494490db 100644 --- a/doc/tutorial-examples/010.c +++ b/doc/documentation/tutorial-examples/010.c diff --git a/doc/tutorial-examples/011.c b/doc/documentation/tutorial-examples/011.c index 23bc051de1..23bc051de1 100644 --- a/doc/tutorial-examples/011.c +++ b/doc/documentation/tutorial-examples/011.c diff --git a/doc/tutorial-examples/012.c b/doc/documentation/tutorial-examples/012.c index cb21d78ab2..cb21d78ab2 100644 --- a/doc/tutorial-examples/012.c +++ b/doc/documentation/tutorial-examples/012.c diff --git a/doc/documentation/tutorial-examples/013.1.c b/doc/documentation/tutorial-examples/013.1.c new file mode 100644 index 0000000000..fa52128685 --- /dev/null +++ b/doc/documentation/tutorial-examples/013.1.c @@ -0,0 +1,3 @@ +void +GNUNET_PEERSTORE_store_cancel (struct GNUNET_PEERSTORE_StoreContext + *sc); diff --git a/doc/tutorial-examples/013.c b/doc/documentation/tutorial-examples/013.c index 6792417e19..6792417e19 100644 --- a/doc/tutorial-examples/013.c +++ b/doc/documentation/tutorial-examples/013.c diff --git a/doc/tutorial-examples/014.c b/doc/documentation/tutorial-examples/014.c index ce204f7956..ce204f7956 100644 --- a/doc/tutorial-examples/014.c +++ b/doc/documentation/tutorial-examples/014.c diff --git a/doc/tutorial-examples/015.c b/doc/documentation/tutorial-examples/015.c index 0dd267e8e3..0dd267e8e3 100644 --- a/doc/tutorial-examples/015.c +++ b/doc/documentation/tutorial-examples/015.c diff --git a/doc/tutorial-examples/016.c b/doc/documentation/tutorial-examples/016.c index d8db4b3b8c..d169da16d0 100644 --- a/doc/tutorial-examples/016.c +++ b/doc/documentation/tutorial-examples/016.c @@ -1,3 +1,4 @@ void -GNUNET_PEERSTORE_watch_cancel (struct GNUNET_PEERSTORE_WatchContext *wc); +GNUNET_PEERSTORE_watch_cancel (struct GNUNET_PEERSTORE_WatchContext + *wc); diff --git a/doc/documentation/tutorial-examples/017.c b/doc/documentation/tutorial-examples/017.c new file mode 100644 index 0000000000..c86fbcd1ff --- /dev/null +++ b/doc/documentation/tutorial-examples/017.c @@ -0,0 +1,4 @@ +void +GNUNET_PEERSTORE_disconnect (struct GNUNET_PEERSTORE_Handle *h, + int sync_first); + diff --git a/doc/tutorial-examples/018.c b/doc/documentation/tutorial-examples/018.c index 3fc22584c8..3fc22584c8 100644 --- a/doc/tutorial-examples/018.c +++ b/doc/documentation/tutorial-examples/018.c diff --git a/doc/tutorial-examples/019.c b/doc/documentation/tutorial-examples/019.c index d016d381bb..aaf0015169 100644 --- a/doc/tutorial-examples/019.c +++ b/doc/documentation/tutorial-examples/019.c @@ -1,4 +1,5 @@ -message_sent_cont (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +message_sent_cont (void *cls, + const struct GNUNET_SCHEDULER_TaskContext *tc) { // Request has left local node } @@ -8,7 +9,9 @@ GNUNET_DHT_put (struct GNUNET_DHT_Handle *handle, const struct GNUNET_HashCode *key, uint32_t desired_replication_level, enum GNUNET_DHT_RouteOption options, - enum GNUNET_BLOCK_Type type, size_t size, const void *data, + enum GNUNET_BLOCK_Type type, + size_t size, + const void *data, struct GNUNET_TIME_Absolute exp, struct GNUNET_TIME_Relative timeout, GNUNET_DHT_PutContinuation cont, void *cont_cls) diff --git a/doc/tutorial-examples/020.c b/doc/documentation/tutorial-examples/020.c index 5ecba1c169..596db30693 100644 --- a/doc/tutorial-examples/020.c +++ b/doc/documentation/tutorial-examples/020.c @@ -5,7 +5,8 @@ get_result_iterator (void *cls, struct GNUNET_TIME_Absolute expiration, unsigned int get_path_length, const struct GNUNET_PeerIdentity *put_path, unsigned int put_path_length, - enum GNUNET_BLOCK_Type type, size_t size, const void *data) + enum GNUNET_BLOCK_Type type, size_t size, + const void *data) { // Optionally: GNUNET_DHT_get_stop (get_handle); diff --git a/doc/tutorial-examples/021.c b/doc/documentation/tutorial-examples/021.c index 688a31fe0e..688a31fe0e 100644 --- a/doc/tutorial-examples/021.c +++ b/doc/documentation/tutorial-examples/021.c diff --git a/doc/tutorial-examples/022.c b/doc/documentation/tutorial-examples/022.c index a373619bd3..a373619bd3 100644 --- a/doc/tutorial-examples/022.c +++ b/doc/documentation/tutorial-examples/022.c diff --git a/doc/tutorial-examples/023.c b/doc/documentation/tutorial-examples/023.c index 820c38b10a..820c38b10a 100644 --- a/doc/tutorial-examples/023.c +++ b/doc/documentation/tutorial-examples/023.c diff --git a/doc/tutorial-examples/024.c b/doc/documentation/tutorial-examples/024.c index 2e84b5905c..2e84b5905c 100644 --- a/doc/tutorial-examples/024.c +++ b/doc/documentation/tutorial-examples/024.c diff --git a/doc/tutorial-examples/025.c b/doc/documentation/tutorial-examples/025.c index 66d4f80eca..66d4f80eca 100644 --- a/doc/tutorial-examples/025.c +++ b/doc/documentation/tutorial-examples/025.c diff --git a/doc/tutorial-examples/026.c b/doc/documentation/tutorial-examples/026.c index 264e0b6b97..264e0b6b97 100644 --- a/doc/tutorial-examples/026.c +++ b/doc/documentation/tutorial-examples/026.c diff --git a/doc/hacks.el b/doc/hacks.el new file mode 100644 index 0000000000..9f271b3afa --- /dev/null +++ b/doc/hacks.el @@ -0,0 +1,17 @@ +;;;; hacks.el --- a few functions to help me work on the manual +;;;; Jim Blandy <jimb@red-bean.com> --- October 1998 +;;;; -- imported from https://git.savannah.gnu.org/cgit/guile.git/tree/doc/hacks.el + +(defun jh-exemplify-region (start end) + (interactive "r") + (save-excursion + (save-restriction + (narrow-to-region start end) + + ;; Texinfo doesn't handle tabs well. + (untabify (point-min) (point-max)) + + ;; Quote any characters special to texinfo. + (goto-char (point-min)) + (while (re-search-forward "[{}@]" nil t) + (replace-match "@\\&"))))) diff --git a/doc/man/gnunet-ecc.1 b/doc/man/gnunet-ecc.1 index a91a2ac2f7..910687f1f3 100644 --- a/doc/man/gnunet-ecc.1 +++ b/doc/man/gnunet-ecc.1 @@ -19,11 +19,11 @@ Create COUNT public-private key pairs and write them to FILENAME. Used for crea .IP "\-p, \-\-print-public-key" Print the corresponding public key to stdout. This is the value used for PKEY records in GNS. .B -.IP "\-p, \-\-print-hex" -Print the corresponding public key to stdout in HEX format. Useful for comparing to Ed25519 keys in X.509 tools. +.IP "\-P, \-\-print-private-key" +Print the corresponding private key to stdout. This is the value used for PKEY records in GNS. .B -.IP "\-P, \-\-print-peer-identity" -Print the corresponding peer identity (hash of the public key) to stdout. This hash is used for the name of peers. +.IP "\-x, \-\-print-hex" +Print the corresponding public key to stdout in HEX format. Useful for comparing to Ed25519 keys in X.509 tools. .B .IP "\-c FILENAME, \-\-config=FILENAME" Use the configuration file FILENAME. diff --git a/doc/tutorial-examples/003.c b/doc/tutorial-examples/003.c deleted file mode 100644 index d13681ca60..0000000000 --- a/doc/tutorial-examples/003.c +++ /dev/null @@ -1,7 +0,0 @@ - struct GNUNET_MQ_MessageHandlers handlers[] = { - // ... - GNUNET_MQ_handler_end () - }; - struct GNUNET_MQ_Handle *mq; - - mq = GNUNET_CLIENT_connect (cfg, "service-name", handlers, &error_cb, NULL); diff --git a/doc/tutorial-examples/017.c b/doc/tutorial-examples/017.c deleted file mode 100644 index c4acbc0883..0000000000 --- a/doc/tutorial-examples/017.c +++ /dev/null @@ -1,3 +0,0 @@ -void -GNUNET_PEERSTORE_disconnect (struct GNUNET_PEERSTORE_Handle *h, int sync_first); - diff --git a/po/POTFILES.in b/po/POTFILES.in index 3444e321bf..01c197fcd4 100644 --- a/po/POTFILES.in +++ b/po/POTFILES.in @@ -196,9 +196,10 @@ src/hello/hello.c src/hostlist/gnunet-daemon-hostlist.c src/hostlist/gnunet-daemon-hostlist_client.c src/hostlist/gnunet-daemon-hostlist_server.c +src/identity-attribute/identity_attribute.c +src/identity-attribute/plugin_identity_attribute_gnuid.c src/identity-provider/gnunet-idp.c src/identity-provider/gnunet-service-identity-provider.c -src/identity-provider/identity_attribute.c src/identity-provider/identity_provider_api.c src/identity-provider/jwt.c src/identity-provider/plugin_gnsrecord_identity_provider.c diff --git a/src/Makefile.am b/src/Makefile.am index e4d7d8924f..68878b5a01 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -17,6 +17,7 @@ endif if HAVE_JSON if HAVE_MHD + ATTRIBUTE_DIR = identity-attribute PROVIDER_DIR = identity-provider endif endif @@ -131,6 +132,7 @@ SUBDIRS = \ social \ $(AUCTION_DIR) \ $(EXP_DIR) \ + $(ATTRIBUTE_DIR) \ $(PROVIDER_DIR) endif diff --git a/src/arm/Makefile.am b/src/arm/Makefile.am index 373847fde2..b1706a4793 100644 --- a/src/arm/Makefile.am +++ b/src/arm/Makefile.am @@ -92,7 +92,8 @@ test_gnunet_service_arm_SOURCES = \ do_subst = $(SED) -e 's,[@]PYTHON[@],$(PYTHON),g' -%.py: %.py.in Makefile +SUFFIXES = .py.in .py +.py.in.py: $(do_subst) < $(srcdir)/$< > $@ chmod +x $@ diff --git a/src/ats-tool/gnunet-ats.c b/src/ats-tool/gnunet-ats.c index 5ec7693b1a..d88e6d523a 100644 --- a/src/ats-tool/gnunet-ats.c +++ b/src/ats-tool/gnunet-ats.c @@ -872,8 +872,8 @@ run (void *cls, for (c = 0; c < strlen (opt_type_str); c++) { - if (isupper (opt_type_str[c])) - opt_type_str[c] = tolower (opt_type_str[c]); + if (isupper ((unsigned char) opt_type_str[c])) + opt_type_str[c] = tolower ((unsigned char) opt_type_str[c]); } if (0 == strcasecmp ("latency", opt_type_str)) @@ -974,7 +974,7 @@ main (int argc, gettext_noop ("set preference for the given peer"), &opt_set_pref), - GNUNET_GETOPT_option_flag ('q', + GNUNET_GETOPT_option_flag ('q', "quotas", gettext_noop ("print all configured quotas"), &opt_print_quotas), diff --git a/src/cadet/gnunet-service-cadet_paths.c b/src/cadet/gnunet-service-cadet_paths.c index 13752643c3..79eed0dccd 100644 --- a/src/cadet/gnunet-service-cadet_paths.c +++ b/src/cadet/gnunet-service-cadet_paths.c @@ -179,7 +179,7 @@ GCPP_del_connection (struct CadetPeerPath *path, GCC_2s (cc), GCPP_2s (path), off); - GNUNET_assert (off < path->entries_length); + GNUNET_assert (off < path->entries_length); /* FIXME: This assertion fails sometimes! */ entry = path->entries[off]; GNUNET_assert (cc == entry->cc); entry->cc = NULL; @@ -498,8 +498,8 @@ GCPP_try_path_from_dht (const struct GNUNET_PeerIdentity *get_path, const struct GNUNET_PeerIdentity *pid; pid = (off < get_path_length) - ? &get_path[get_path_length - off] - : &put_path[get_path_length + put_path_length - off]; + ? &get_path[get_path_length - off - 1] + : &put_path[get_path_length + put_path_length - off - 1]; cpath[off - skip] = GCP_get (pid, GNUNET_YES); /* Check that no peer is twice on the path */ diff --git a/src/conversation/gnunet-conversation.c b/src/conversation/gnunet-conversation.c index 8f9ddec257..00ab65680d 100644 --- a/src/conversation/gnunet-conversation.c +++ b/src/conversation/gnunet-conversation.c @@ -1091,7 +1091,7 @@ handle_command_string (char *message, strlen (commands[i].command)))) i++; ptr = &message[strlen (commands[i].command)]; - while (isspace ((int) *ptr)) + while (isspace ((unsigned char) *ptr)) ptr++; if ('\0' == *ptr) ptr = NULL; diff --git a/src/core/gnunet-service-core_sessions.c b/src/core/gnunet-service-core_sessions.c index 034f2e8838..5d34b7c26c 100644 --- a/src/core/gnunet-service-core_sessions.c +++ b/src/core/gnunet-service-core_sessions.c @@ -975,6 +975,7 @@ GSC_SESSIONS_set_typemap (const struct GNUNET_PeerIdentity *peer, session = find_session (peer); if (NULL == session) { + GSC_TYPEMAP_destroy (nmap); GNUNET_break (0); return; } diff --git a/src/core/test_core_api_reliability.c b/src/core/test_core_api_reliability.c index 900c9f7327..528093c994 100644 --- a/src/core/test_core_api_reliability.c +++ b/src/core/test_core_api_reliability.c @@ -381,6 +381,7 @@ process_hello (void *cls, GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Received (my) `%s' from transport service\n", "HELLO"); GNUNET_assert (message != NULL); + GNUNET_free_non_null (p->hello); p->hello = GNUNET_copy_message (message); if ((p == &p1) && (NULL == p2.oh)) p2.oh = GNUNET_TRANSPORT_offer_hello (p2.cfg, @@ -518,6 +519,8 @@ main (int argc, &ok); stop_arm (&p1); stop_arm (&p2); + GNUNET_free_non_null (p1.hello); + GNUNET_free_non_null (p2.hello); GNUNET_DISK_directory_remove ("/tmp/test-gnunet-core-peer-1"); GNUNET_DISK_directory_remove ("/tmp/test-gnunet-core-peer-2"); diff --git a/src/datastore/datastore_api.c b/src/datastore/datastore_api.c index 31f7a997f7..2ad864987f 100644 --- a/src/datastore/datastore_api.c +++ b/src/datastore/datastore_api.c @@ -651,6 +651,46 @@ process_queue (struct GNUNET_DATASTORE_Handle *h) } +/** + * Get the entry at the head of the message queue. + * + * @param h handle to the datastore + * @param response_type the expected response type + * @return the queue entry + */ +static struct GNUNET_DATASTORE_QueueEntry * +get_queue_head (struct GNUNET_DATASTORE_Handle *h, + uint16_t response_type) +{ + struct GNUNET_DATASTORE_QueueEntry *qe; + + if (h->skip_next_messages > 0) + { + h->skip_next_messages--; + process_queue (h); + return NULL; + } + qe = h->queue_head; + if (NULL == qe) + { + GNUNET_break (0); + do_disconnect (h); + return NULL; + } + if (NULL != qe->env) + { + GNUNET_break (0); + do_disconnect (h); + return NULL; + } + if (response_type != qe->response_type) + { + GNUNET_break (0); + do_disconnect (h); + return NULL; + } + return qe; +} /** @@ -702,30 +742,10 @@ handle_status (void *cls, const char *emsg; int32_t status = ntohl (sm->status); - if (h->skip_next_messages > 0) - { - h->skip_next_messages--; - process_queue (h); - return; - } - if (NULL == (qe = h->queue_head)) - { - GNUNET_break (0); - do_disconnect (h); - return; - } - if (NULL != qe->env) - { - GNUNET_break (0); - do_disconnect (h); - return; - } - if (GNUNET_MESSAGE_TYPE_DATASTORE_STATUS != qe->response_type) - { - GNUNET_break (0); - do_disconnect (h); + qe = get_queue_head (h, + GNUNET_MESSAGE_TYPE_DATASTORE_STATUS); + if (NULL == qe) return; - } rc = qe->qc.sc; free_queue_entry (qe); if (ntohs (sm->header.size) > sizeof (struct StatusMessage)) @@ -785,30 +805,10 @@ handle_data (void *cls, struct GNUNET_DATASTORE_QueueEntry *qe; struct ResultContext rc; - if (h->skip_next_messages > 0) - { - process_queue (h); - return; - } - qe = h->queue_head; + qe = get_queue_head (h, + GNUNET_MESSAGE_TYPE_DATASTORE_DATA); if (NULL == qe) - { - GNUNET_break (0); - do_disconnect (h); - return; - } - if (NULL != qe->env) - { - GNUNET_break (0); - do_disconnect (h); - return; - } - if (GNUNET_MESSAGE_TYPE_DATASTORE_DATA != qe->response_type) - { - GNUNET_break (0); - do_disconnect (h); return; - } #if INSANE_STATISTICS GNUNET_STATISTICS_update (h->stats, gettext_noop ("# Results received"), @@ -854,31 +854,10 @@ handle_data_end (void *cls, struct GNUNET_DATASTORE_QueueEntry *qe; struct ResultContext rc; - if (h->skip_next_messages > 0) - { - h->skip_next_messages--; - process_queue (h); - return; - } - qe = h->queue_head; + qe = get_queue_head (h, + GNUNET_MESSAGE_TYPE_DATASTORE_DATA); if (NULL == qe) - { - GNUNET_break (0); - do_disconnect (h); return; - } - if (NULL != qe->env) - { - GNUNET_break (0); - do_disconnect (h); - return; - } - if (GNUNET_MESSAGE_TYPE_DATASTORE_DATA != qe->response_type) - { - GNUNET_break (0); - do_disconnect (h); - return; - } rc = qe->qc.rc; free_queue_entry (qe); LOG (GNUNET_ERROR_TYPE_DEBUG, diff --git a/src/dht/Makefile.am b/src/dht/Makefile.am index 00ce0e9344..4a78ea4c79 100644 --- a/src/dht/Makefile.am +++ b/src/dht/Makefile.am @@ -213,7 +213,8 @@ endif do_subst = $(SED) -e 's,[@]PYTHON[@],$(PYTHON),g' -e 's,[@]bindir[@],$(bindir),g' -%.py: %.py.in Makefile +SUFFIXES = .py.in .py +.py.in.py: $(do_subst) < $(srcdir)/$< > $@ chmod +x $@ diff --git a/src/dns/dnsparser.c b/src/dns/dnsparser.c index 36b4c36f17..30d9245ff7 100644 --- a/src/dns/dnsparser.c +++ b/src/dns/dnsparser.c @@ -1278,8 +1278,8 @@ GNUNET_DNSPARSER_hex_to_bin (const char *hex, in[2] = '\0'; for (off = 0; off < data_size; off++) { - in[0] = tolower ((int) hex[off * 2]); - in[1] = tolower ((int) hex[off * 2 + 1]); + in[0] = tolower ((unsigned char) hex[off * 2]); + in[1] = tolower ((unsigned char) hex[off * 2 + 1]); if (1 != sscanf (in, "%x", &h)) return off; idata[off] = (uint8_t) h; diff --git a/src/fs/fs_misc.c b/src/fs/fs_misc.c index bcb8620cfa..b26de431cc 100644 --- a/src/fs/fs_misc.c +++ b/src/fs/fs_misc.c @@ -1,6 +1,6 @@ /* This file is part of GNUnet. - Copyright (C) 2010, 2011 GNUnet e.V. + Copyright (C) 2010, 2011, 2017 GNUnet e.V. GNUnet is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published @@ -43,6 +43,8 @@ GNUNET_FS_meta_data_suggest_filename (const struct GNUNET_CONTAINER_MetaData {"application/gnunet-directory", ".gnd"}, {"application/java", ".class"}, {"application/msword", ".doc"}, + {"application/nar", ".nar"}, + {"application/narinfo", ".narinfo"}, {"application/ogg", ".ogg"}, {"application/pdf", ".pdf"}, {"application/pgp-keys", ".key"}, @@ -53,8 +55,8 @@ GNUNET_FS_meta_data_suggest_filename (const struct GNUNET_CONTAINER_MetaData {"application/xml", ".xml"}, {"application/x-debian-package", ".deb"}, {"application/x-dvi", ".dvi"}, - {"applixation/x-flac", ".flac"}, - {"applixation/x-gzip", ".gz"}, + {"application/x-flac", ".flac"}, + {"application/x-gzip", ".gz"}, {"application/x-java-archive", ".jar"}, {"application/x-java-vm", ".class"}, {"application/x-python-code", ".pyc"}, diff --git a/src/fs/fs_publish_ublock.c b/src/fs/fs_publish_ublock.c index e21443ccbf..189a6909a2 100644 --- a/src/fs/fs_publish_ublock.c +++ b/src/fs/fs_publish_ublock.c @@ -301,6 +301,7 @@ GNUNET_FS_publish_ublock_ (struct GNUNET_FS_Handle *h, uc->task = GNUNET_SCHEDULER_add_now (&run_cont, uc); } + GNUNET_free (ub_enc); return uc; } diff --git a/src/identity-attribute/Makefile.am b/src/identity-attribute/Makefile.am new file mode 100644 index 0000000000..b84ad34926 --- /dev/null +++ b/src/identity-attribute/Makefile.am @@ -0,0 +1,45 @@ +# This Makefile.am is in the public domain +AM_CPPFLAGS = -I$(top_srcdir)/src/include + +plugindir = $(libdir)/gnunet + +pkgcfgdir= $(pkgdatadir)/config.d/ + +libexecdir= $(pkglibdir)/libexec/ + +if MINGW + WINFLAGS = -Wl,--no-undefined -Wl,--export-all-symbols +endif + +if USE_COVERAGE + AM_CFLAGS = --coverage -O0 + XLIBS = -lgcov +endif + +lib_LTLIBRARIES = \ + libgnunetidentityattribute.la + +libgnunetidentityattribute_la_SOURCES = \ + identity_attribute.c \ + jwt.c +libgnunetidentityattribute_la_LIBADD = \ + $(top_builddir)/src/util/libgnunetutil.la \ + $(GN_LIBINTL) +libgnunetidentityattribute_la_LDFLAGS = \ + $(GN_LIB_LDFLAGS) $(WINFLAGS) \ + -version-info 0:0:0 + + +plugin_LTLIBRARIES = \ + libgnunet_plugin_identity_attribute_gnuid.la + + +libgnunet_plugin_identity_attribute_gnuid_la_SOURCES = \ + plugin_identity_attribute_gnuid.c +libgnunet_plugin_identity_attribute_gnuid_la_LIBADD = \ + $(top_builddir)/src/util/libgnunetutil.la \ + $(LTLIBINTL) +libgnunet_plugin_identity_attribute_gnuid_la_LDFLAGS = \ + $(GN_PLUGIN_LDFLAGS) + + diff --git a/src/identity-attribute/identity_attribute.c b/src/identity-attribute/identity_attribute.c new file mode 100644 index 0000000000..a8aae6ced7 --- /dev/null +++ b/src/identity-attribute/identity_attribute.c @@ -0,0 +1,421 @@ +/* + This file is part of GNUnet + Copyright (C) 2010-2015 GNUnet e.V. + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + Boston, MA 02110-1301, USA. + */ + +/** + * @file identity-provider/identity_attribute.c + * @brief helper library to manage identity attributes + * @author Martin Schanzenbach + */ +#include "platform.h" +#include "gnunet_util_lib.h" +#include "identity_attribute.h" +#include "gnunet_identity_attribute_plugin.h" + +/** + * Handle for a plugin + */ +struct Plugin +{ + /** + * Name of the plugin + */ + char *library_name; + + /** + * Plugin API + */ + struct GNUNET_IDENTITY_ATTRIBUTE_PluginFunctions *api; +}; + +/** + * Plugins + */ +static struct Plugin **attr_plugins; + +/** + * Number of plugins + */ +static unsigned int num_plugins; + +/** + * Init canary + */ +static int initialized; + +/** + * Add a plugin + */ +static void +add_plugin (void* cls, + const char *library_name, + void *lib_ret) +{ + struct GNUNET_IDENTITY_ATTRIBUTE_PluginFunctions *api = lib_ret; + struct Plugin *plugin; + + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Loading attribute plugin `%s'\n", + library_name); + plugin = GNUNET_new (struct Plugin); + plugin->api = api; + plugin->library_name = GNUNET_strdup (library_name); + GNUNET_array_append (attr_plugins, num_plugins, plugin); +} + +/** + * Load plugins + */ +static void +init() +{ + if (GNUNET_YES == initialized) + return; + initialized = GNUNET_YES; + GNUNET_PLUGIN_load_all ("libgnunet_plugin_identity_attribute_", NULL, + &add_plugin, NULL); +} + +/** + * Convert a type name to the corresponding number + * + * @param typename name to convert + * @return corresponding number, UINT32_MAX on error + */ +uint32_t +GNUNET_IDENTITY_ATTRIBUTE_typename_to_number (const char *typename) +{ + unsigned int i; + struct Plugin *plugin; + uint32_t ret; + + init (); + for (i = 0; i < num_plugins; i++) + { + plugin = attr_plugins[i]; + if (UINT32_MAX != (ret = plugin->api->typename_to_number (plugin->api->cls, + typename))) + return ret; + } + return UINT32_MAX; +} + +/** + * Convert a type number to the corresponding type string + * + * @param type number of a type + * @return corresponding typestring, NULL on error + */ +const char* +GNUNET_IDENTITY_ATTRIBUTE_number_to_typename (uint32_t type) +{ + unsigned int i; + struct Plugin *plugin; + const char *ret; + + init (); + for (i = 0; i < num_plugins; i++) + { + plugin = attr_plugins[i]; + if (NULL != (ret = plugin->api->number_to_typename (plugin->api->cls, + type))) + return ret; + } + return NULL; +} + +/** + * Convert human-readable version of a 'claim' of an attribute to the binary + * representation + * + * @param type type of the claim + * @param s human-readable string + * @param data set to value in binary encoding (will be allocated) + * @param data_size set to number of bytes in @a data + * @return #GNUNET_OK on success + */ +int +GNUNET_IDENTITY_ATTRIBUTE_string_to_value (uint32_t type, + const char *s, + void **data, + size_t *data_size) +{ + unsigned int i; + struct Plugin *plugin; + + init (); + for (i = 0; i < num_plugins; i++) + { + plugin = attr_plugins[i]; + if (GNUNET_OK == plugin->api->string_to_value (plugin->api->cls, + type, + s, + data, + data_size)) + return GNUNET_OK; + } + return GNUNET_SYSERR; +} + +/** + * Convert the 'claim' of an attribute to a string + * + * @param type the type of attribute + * @param data claim in binary encoding + * @param data_size number of bytes in @a data + * @return NULL on error, otherwise human-readable representation of the claim + */ +char * +GNUNET_IDENTITY_ATTRIBUTE_value_to_string (uint32_t type, + const void* data, + size_t data_size) +{ + unsigned int i; + struct Plugin *plugin; + char *ret; + + init(); + for (i = 0; i < num_plugins; i++) + { + plugin = attr_plugins[i]; + if (NULL != (ret = plugin->api->value_to_string (plugin->api->cls, + type, + data, + data_size))) + return ret; + } + return NULL; +} + +/** + * Create a new attribute. + * + * @param name the attribute name + * @param type the attribute type + * @param data the attribute value + * @param data_size the attribute value size + * @return the new attribute + */ +struct GNUNET_IDENTITY_ATTRIBUTE_Claim * +GNUNET_IDENTITY_ATTRIBUTE_claim_new (const char* attr_name, + uint32_t attr_type, + const void* data, + size_t data_size) +{ + struct GNUNET_IDENTITY_ATTRIBUTE_Claim *attr; + char *write_ptr; + + attr = GNUNET_malloc (sizeof (struct GNUNET_IDENTITY_ATTRIBUTE_Claim) + + strlen (attr_name) + 1 + + data_size); + attr->type = attr_type; + attr->data_size = data_size; + attr->version = 0; + write_ptr = (char*)&attr[1]; + GNUNET_memcpy (write_ptr, + attr_name, + strlen (attr_name) + 1); + attr->name = write_ptr; + write_ptr += strlen (attr->name) + 1; + GNUNET_memcpy (write_ptr, + data, + data_size); + attr->data = write_ptr; + return attr; +} + +size_t +GNUNET_IDENTITY_ATTRIBUTE_list_serialize_get_size (const struct GNUNET_IDENTITY_ATTRIBUTE_ClaimList *attrs) +{ + struct GNUNET_IDENTITY_ATTRIBUTE_ClaimListEntry *le; + size_t len = 0; + for (le = attrs->list_head; NULL != le; le = le->next) + len += GNUNET_IDENTITY_ATTRIBUTE_serialize_get_size (le->claim); + return len; +} + +size_t +GNUNET_IDENTITY_ATTRIBUTE_list_serialize (const struct GNUNET_IDENTITY_ATTRIBUTE_ClaimList *attrs, + char *result) +{ + struct GNUNET_IDENTITY_ATTRIBUTE_ClaimListEntry *le; + size_t len; + size_t total_len; + char* write_ptr; + + write_ptr = result; + total_len = 0; + for (le = attrs->list_head; NULL != le; le = le->next) + { + len = GNUNET_IDENTITY_ATTRIBUTE_serialize (le->claim, + write_ptr); + total_len += len; + write_ptr += len; + } + return total_len; +} + +struct GNUNET_IDENTITY_ATTRIBUTE_ClaimList * +GNUNET_IDENTITY_ATTRIBUTE_list_deserialize (const char* data, + size_t data_size) +{ + struct GNUNET_IDENTITY_ATTRIBUTE_ClaimList *attrs; + struct GNUNET_IDENTITY_ATTRIBUTE_ClaimListEntry *le; + size_t attr_len; + const char* read_ptr; + + if (data_size < sizeof (struct Attribute)) + return NULL; + + attrs = GNUNET_new (struct GNUNET_IDENTITY_ATTRIBUTE_ClaimList); + read_ptr = data; + while (((data + data_size) - read_ptr) >= sizeof (struct Attribute)) + { + + le = GNUNET_new (struct GNUNET_IDENTITY_ATTRIBUTE_ClaimListEntry); + le->claim = GNUNET_IDENTITY_ATTRIBUTE_deserialize (read_ptr, + data_size - (read_ptr - data)); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Deserialized attribute %s\n", le->claim->name); + GNUNET_CONTAINER_DLL_insert (attrs->list_head, + attrs->list_tail, + le); + attr_len = GNUNET_IDENTITY_ATTRIBUTE_serialize_get_size (le->claim); + read_ptr += attr_len; + } + return attrs; +} + +struct GNUNET_IDENTITY_ATTRIBUTE_ClaimList* +GNUNET_IDENTITY_ATTRIBUTE_list_dup (const struct GNUNET_IDENTITY_ATTRIBUTE_ClaimList *attrs) +{ + struct GNUNET_IDENTITY_ATTRIBUTE_ClaimListEntry *le; + struct GNUNET_IDENTITY_ATTRIBUTE_ClaimListEntry *result_le; + struct GNUNET_IDENTITY_ATTRIBUTE_ClaimList *result; + size_t len; + + result = GNUNET_new (struct GNUNET_IDENTITY_ATTRIBUTE_ClaimList); + for (le = attrs->list_head; NULL != le; le = le->next) + { + result_le = GNUNET_new (struct GNUNET_IDENTITY_ATTRIBUTE_ClaimListEntry); + len = sizeof (struct GNUNET_IDENTITY_ATTRIBUTE_Claim) + le->claim->data_size; + result_le->claim = GNUNET_malloc (len); + GNUNET_memcpy (result_le->claim, + le->claim, + len); + result_le->claim->name = (const char*)&result_le->claim[1]; + GNUNET_CONTAINER_DLL_insert (result->list_head, + result->list_tail, + result_le); + } + return result; +} + + +void +GNUNET_IDENTITY_ATTRIBUTE_list_destroy (struct GNUNET_IDENTITY_ATTRIBUTE_ClaimList *attrs) +{ + struct GNUNET_IDENTITY_ATTRIBUTE_ClaimListEntry *le; + struct GNUNET_IDENTITY_ATTRIBUTE_ClaimListEntry *tmp_le; + + for (le = attrs->list_head; NULL != le;) + { + GNUNET_free (le->claim); + tmp_le = le; + le = le->next; + GNUNET_free (tmp_le); + } + GNUNET_free (attrs); + +} + +size_t +GNUNET_IDENTITY_ATTRIBUTE_serialize_get_size (const struct GNUNET_IDENTITY_ATTRIBUTE_Claim *attr) +{ + return sizeof (struct Attribute) + + strlen (attr->name) + + attr->data_size; +} + +size_t +GNUNET_IDENTITY_ATTRIBUTE_serialize (const struct GNUNET_IDENTITY_ATTRIBUTE_Claim *attr, + char *result) +{ + size_t data_len_ser; + size_t name_len; + struct Attribute *attr_ser; + char* write_ptr; + + attr_ser = (struct Attribute*)result; + attr_ser->attribute_type = htons (attr->type); + attr_ser->attribute_version = htonl (attr->version); + name_len = strlen (attr->name); + attr_ser->name_len = htons (name_len); + write_ptr = (char*)&attr_ser[1]; + GNUNET_memcpy (write_ptr, attr->name, name_len); + write_ptr += name_len; + //TODO plugin-ize + //data_len_ser = plugin->serialize_attribute_value (attr, + // &attr_ser[1]); + data_len_ser = attr->data_size; + GNUNET_memcpy (write_ptr, attr->data, attr->data_size); + attr_ser->data_size = htons (data_len_ser); + + return sizeof (struct Attribute) + strlen (attr->name) + attr->data_size; +} + +struct GNUNET_IDENTITY_ATTRIBUTE_Claim * +GNUNET_IDENTITY_ATTRIBUTE_deserialize (const char* data, + size_t data_size) +{ + struct GNUNET_IDENTITY_ATTRIBUTE_Claim *attr; + struct Attribute *attr_ser; + size_t data_len; + size_t name_len; + char* write_ptr; + + if (data_size < sizeof (struct Attribute)) + return NULL; + + attr_ser = (struct Attribute*)data; + data_len = ntohs (attr_ser->data_size); + name_len = ntohs (attr_ser->name_len); + attr = GNUNET_malloc (sizeof (struct GNUNET_IDENTITY_ATTRIBUTE_Claim) + + data_len + name_len + 1); + attr->type = ntohs (attr_ser->attribute_type); + attr->version = ntohl (attr_ser->attribute_version); + attr->data_size = ntohs (attr_ser->data_size); + + write_ptr = (char*)&attr[1]; + GNUNET_memcpy (write_ptr, + &attr_ser[1], + name_len); + write_ptr[name_len] = '\0'; + attr->name = write_ptr; + + write_ptr += name_len + 1; + GNUNET_memcpy (write_ptr, + (char*)&attr_ser[1] + name_len, + attr->data_size); + attr->data = write_ptr; + return attr; + +} + +/* end of identity_attribute.c */ diff --git a/src/identity-attribute/identity_attribute.h b/src/identity-attribute/identity_attribute.h new file mode 100644 index 0000000000..0463218078 --- /dev/null +++ b/src/identity-attribute/identity_attribute.h @@ -0,0 +1,56 @@ +/* + This file is part of GNUnet. + Copyright (C) 2012-2015 GNUnet e.V. + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + Boston, MA 02110-1301, USA. + */ +/** + * @author Martin Schanzenbach + * @file identity-provider/identity_attribute.h + * @brief GNUnet Identity Provider library + * + */ +#ifndef IDENTITY_ATTRIBUTE_H +#define IDENTITY_ATTRIBUTE_H + +#include "gnunet_identity_provider_service.h" + +struct Attribute +{ + /** + * Attribute type + */ + uint32_t attribute_type; + + /** + * Attribute version + */ + uint32_t attribute_version; + + /** + * Name length + */ + uint32_t name_len; + + /** + * Data size + */ + uint32_t data_size; + + //followed by data_size Attribute value data +}; + +#endif diff --git a/src/identity-provider/jwt.c b/src/identity-attribute/jwt.c index c8bc67806b..935e0a79d1 100644 --- a/src/identity-provider/jwt.c +++ b/src/identity-attribute/jwt.c @@ -26,7 +26,7 @@ #include "platform.h" #include "gnunet_util_lib.h" #include "gnunet_signatures.h" -#include "identity_attribute.h" +#include "gnunet_identity_attribute_lib.h" #include <jansson.h> @@ -55,18 +55,20 @@ create_jwt_header(void) } /** - * Create a JWT from a ticket and attributes + * Create a JWT from attributes * - * @param ticket the ticket + * @param sub_key the public of the subject * @param attrs the attribute list + * @param priv_key the key used to sign the JWT * @return a new base64-encoded JWT string. */ char* -jwt_create (const struct GNUNET_IDENTITY_PROVIDER_Ticket *ticket, - const struct GNUNET_IDENTITY_PROVIDER_AttributeList *attrs, - const struct GNUNET_CRYPTO_EcdsaPrivateKey *priv_key) +GNUNET_IDENTITY_ATTRIBUTE_jwt_create_from_list (const struct GNUNET_CRYPTO_EcdsaPublicKey *sub_key, + const struct GNUNET_IDENTITY_ATTRIBUTE_ClaimList *attrs, + const struct GNUNET_CRYPTO_EcdsaPrivateKey *priv_key) { - struct GNUNET_IDENTITY_PROVIDER_AttributeListEntry *le; + struct GNUNET_IDENTITY_ATTRIBUTE_ClaimListEntry *le; + struct GNUNET_CRYPTO_EcdsaPublicKey iss_key; struct GNUNET_CRYPTO_EcdsaSignature signature; struct GNUNET_CRYPTO_EccSignaturePurpose *purpose; char* audience; @@ -79,12 +81,14 @@ jwt_create (const struct GNUNET_IDENTITY_PROVIDER_Ticket *ticket, char* body_base64; char* signature_target; char* signature_base64; + char* attr_val_str; json_t* body; + GNUNET_CRYPTO_ecdsa_key_get_public (priv_key, &iss_key); /* TODO maybe we should use a local identity here */ - issuer = GNUNET_STRINGS_data_to_string_alloc (&ticket->identity, + issuer = GNUNET_STRINGS_data_to_string_alloc (&iss_key, sizeof (struct GNUNET_CRYPTO_EcdsaPublicKey)); - audience = GNUNET_STRINGS_data_to_string_alloc (&ticket->audience, + audience = GNUNET_STRINGS_data_to_string_alloc (sub_key, sizeof (struct GNUNET_CRYPTO_EcdsaPublicKey)); header = create_jwt_header (); body = json_object (); @@ -103,9 +107,13 @@ jwt_create (const struct GNUNET_IDENTITY_PROVIDER_Ticket *ticket, * calls the Attribute plugins to create a * json representation for its value */ + attr_val_str = GNUNET_IDENTITY_ATTRIBUTE_value_to_string (le->claim->type, + le->claim->data, + le->claim->data_size); json_object_set_new (body, - le->attribute->name, - json_string (le->attribute->data)); + le->claim->name, + json_string (attr_val_str)); + GNUNET_free (attr_val_str); } body_str = json_dumps (body, JSON_INDENT(0)); json_decref (body); diff --git a/src/identity-attribute/plugin_identity_attribute_gnuid.c b/src/identity-attribute/plugin_identity_attribute_gnuid.c new file mode 100644 index 0000000000..0ff44d1993 --- /dev/null +++ b/src/identity-attribute/plugin_identity_attribute_gnuid.c @@ -0,0 +1,184 @@ +/* + This file is part of GNUnet + Copyright (C) 2013, 2014, 2016 GNUnet e.V. + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + Boston, MA 02110-1301, USA. +*/ + +/** + * @file identity-provider/plugin_identity_attribute_gnuid.c + * @brief identity attribute plugin to provide the API for fundamental + * attribute types. + * + * @author Martin Schanzenbach + */ +#include "platform.h" +#include "gnunet_util_lib.h" +#include "gnunet_identity_attribute_plugin.h" +#include <inttypes.h> + + +/** + * Convert the 'value' of an attribute to a string. + * + * @param cls closure, unused + * @param type type of the attribute + * @param data value in binary encoding + * @param data_size number of bytes in @a data + * @return NULL on error, otherwise human-readable representation of the value + */ +static char * +gnuid_value_to_string (void *cls, + uint32_t type, + const void *data, + size_t data_size) +{ + + switch (type) + { + case GNUNET_IDENTITY_ATTRIBUTE_TYPE_STRING: + return GNUNET_strndup (data, data_size); + default: + return NULL; + } +} + + +/** + * Convert human-readable version of a 'value' of an attribute to the binary + * representation. + * + * @param cls closure, unused + * @param type type of the attribute + * @param s human-readable string + * @param data set to value in binary encoding (will be allocated) + * @param data_size set to number of bytes in @a data + * @return #GNUNET_OK on success + */ +static int +gnuid_string_to_value (void *cls, + uint32_t type, + const char *s, + void **data, + size_t *data_size) +{ + if (NULL == s) + return GNUNET_SYSERR; + switch (type) + { + + case GNUNET_IDENTITY_ATTRIBUTE_TYPE_STRING: + *data = GNUNET_strdup (s); + *data_size = strlen (s); + return GNUNET_OK; + default: + return GNUNET_SYSERR; + } +} + + +/** + * Mapping of attribute type numbers to human-readable + * attribute type names. + */ +static struct { + const char *name; + uint32_t number; +} gnuid_name_map[] = { + { "STRING", GNUNET_IDENTITY_ATTRIBUTE_TYPE_STRING }, + { NULL, UINT32_MAX } +}; + + +/** + * Convert a type name to the corresponding number. + * + * @param cls closure, unused + * @param gnuid_typename name to convert + * @return corresponding number, UINT32_MAX on error + */ +static uint32_t +gnuid_typename_to_number (void *cls, + const char *gnuid_typename) +{ + unsigned int i; + + i=0; + while ( (NULL != gnuid_name_map[i].name) && + (0 != strcasecmp (gnuid_typename, + gnuid_name_map[i].name)) ) + i++; + return gnuid_name_map[i].number; +} + + +/** + * Convert a type number (i.e. 1) to the corresponding type string + * + * @param cls closure, unused + * @param type number of a type to convert + * @return corresponding typestring, NULL on error + */ +static const char * +gnuid_number_to_typename (void *cls, + uint32_t type) +{ + unsigned int i; + + i=0; + while ( (NULL != gnuid_name_map[i].name) && + (type != gnuid_name_map[i].number) ) + i++; + return gnuid_name_map[i].name; +} + + +/** + * Entry point for the plugin. + * + * @param cls NULL + * @return the exported block API + */ +void * +libgnunet_plugin_identity_attribute_gnuid_init (void *cls) +{ + struct GNUNET_IDENTITY_ATTRIBUTE_PluginFunctions *api; + + api = GNUNET_new (struct GNUNET_IDENTITY_ATTRIBUTE_PluginFunctions); + api->value_to_string = &gnuid_value_to_string; + api->string_to_value = &gnuid_string_to_value; + api->typename_to_number = &gnuid_typename_to_number; + api->number_to_typename = &gnuid_number_to_typename; + return api; +} + + +/** + * Exit point from the plugin. + * + * @param cls the return value from #libgnunet_plugin_block_test_init() + * @return NULL + */ +void * +libgnunet_plugin_identity_attribute_gnuid_done (void *cls) +{ + struct GNUNET_IDENTITY_ATTRIBUTE_PluginFunctions *api = cls; + + GNUNET_free (api); + return NULL; +} + +/* end of plugin_identity_attribute_type_gnuid.c */ diff --git a/src/identity-provider/Makefile.am b/src/identity-provider/Makefile.am index f64f1aa687..3b072e59d8 100644 --- a/src/identity-provider/Makefile.am +++ b/src/identity-provider/Makefile.am @@ -63,8 +63,7 @@ libgnunet_plugin_identity_provider_sqlite_la_LDFLAGS = \ gnunet_service_identity_provider_SOURCES = \ - gnunet-service-identity-provider.c \ - identity_attribute.h + gnunet-service-identity-provider.c gnunet_service_identity_provider_LDADD = \ $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \ $(top_builddir)/src/util/libgnunetutil.la \ @@ -72,14 +71,14 @@ gnunet_service_identity_provider_LDADD = \ $(top_builddir)/src/identity/libgnunetidentity.la \ $(top_builddir)/src/statistics/libgnunetstatistics.la \ $(top_builddir)/src/credential/libgnunetcredential.la \ + $(top_builddir)/src/identity-attribute/libgnunetidentityattribute.la \ libgnunetidentityprovider.la \ $(top_builddir)/src/gns/libgnunetgns.la \ $(GN_LIBINTL) libgnunetidentityprovider_la_SOURCES = \ identity_provider_api.c \ - identity_provider.h \ - identity_attribute.c + identity_provider.h libgnunetidentityprovider_la_LIBADD = \ $(top_builddir)/src/util/libgnunetutil.la \ $(GN_LIBINTL) $(XLIB) @@ -88,13 +87,13 @@ libgnunetidentityprovider_la_LDFLAGS = \ -version-info 0:0:0 libgnunet_plugin_rest_identity_provider_la_SOURCES = \ - plugin_rest_identity_provider.c \ - jwt.c + plugin_rest_identity_provider.c libgnunet_plugin_rest_identity_provider_la_LIBADD = \ $(top_builddir)/src/identity/libgnunetidentity.la \ libgnunetidentityprovider.la \ $(top_builddir)/src/rest/libgnunetrest.la \ $(top_builddir)/src/jsonapi/libgnunetjsonapi.la \ + $(top_builddir)/src/identity-attribute/libgnunetidentityattribute.la \ $(top_builddir)/src/namestore/libgnunetnamestore.la \ $(top_builddir)/src/util/libgnunetutil.la $(XLIBS) \ $(LTLIBINTL) -ljansson -lmicrohttpd @@ -108,6 +107,7 @@ gnunet_idp_LDADD = \ $(top_builddir)/src/namestore/libgnunetnamestore.la \ libgnunetidentityprovider.la \ $(top_builddir)/src/identity/libgnunetidentity.la \ + $(top_builddir)/src/identity-attribute/libgnunetidentityattribute.la \ $(GN_LIBINTL) check_SCRIPTS = \ diff --git a/src/identity-provider/gnunet-idp.c b/src/identity-provider/gnunet-idp.c index 88136c1243..62f07842b9 100644 --- a/src/identity-provider/gnunet-idp.c +++ b/src/identity-provider/gnunet-idp.c @@ -67,6 +67,11 @@ static char* issue_attrs; static char* consume_ticket; /** + * Attribute type + */ +static char* type_str; + +/** * Ticket to revoke */ static char* revoke_ticket; @@ -119,7 +124,7 @@ static struct GNUNET_IDENTITY_PROVIDER_Ticket ticket; /** * Attribute list */ -static struct GNUNET_IDENTITY_PROVIDER_AttributeList *attr_list; +static struct GNUNET_IDENTITY_ATTRIBUTE_ClaimList *attr_list; static void do_cleanup(void *cls) @@ -166,8 +171,9 @@ store_attr_cont (void *cls, static void process_attrs (void *cls, const struct GNUNET_CRYPTO_EcdsaPublicKey *identity, - const struct GNUNET_IDENTITY_PROVIDER_Attribute *attr) + const struct GNUNET_IDENTITY_ATTRIBUTE_Claim *attr) { + char *value_str; if (NULL == identity) { GNUNET_SCHEDULER_add_now (&do_cleanup, NULL); @@ -178,8 +184,11 @@ process_attrs (void *cls, ret = 1; return; } + value_str = GNUNET_IDENTITY_ATTRIBUTE_value_to_string (attr->type, + attr->data, + attr->data_size); GNUNET_log (GNUNET_ERROR_TYPE_MESSAGE, - "%s: %s\n", attr->name, (char*)attr->data); + "%s: %s\n", attr->name, value_str); } @@ -207,7 +216,10 @@ process_rvk (void *cls, int success, const char* msg) static void iter_finished (void *cls) { - struct GNUNET_IDENTITY_PROVIDER_Attribute *attr; + struct GNUNET_IDENTITY_ATTRIBUTE_Claim *claim; + char *data; + size_t data_size; + int type; attr_iterator = NULL; if (list) @@ -244,13 +256,22 @@ iter_finished (void *cls) NULL); return; } - attr = GNUNET_IDENTITY_PROVIDER_attribute_new (attr_name, - GNUNET_IDENTITY_PROVIDER_AT_STRING, - attr_value, - strlen (attr_value) + 1); + if (NULL == type_str) + type = GNUNET_IDENTITY_ATTRIBUTE_TYPE_STRING; + else + type = GNUNET_IDENTITY_ATTRIBUTE_typename_to_number (type_str); + + GNUNET_assert (GNUNET_SYSERR != GNUNET_IDENTITY_ATTRIBUTE_string_to_value (type, + attr_value, + (void**)&data, + &data_size)); + claim = GNUNET_IDENTITY_ATTRIBUTE_claim_new (attr_name, + type, + data, + data_size); idp_op = GNUNET_IDENTITY_PROVIDER_attribute_store (idp_handle, pkey, - attr, + claim, &store_attr_cont, NULL); @@ -260,9 +281,9 @@ iter_finished (void *cls) static void iter_cb (void *cls, const struct GNUNET_CRYPTO_EcdsaPublicKey *identity, - const struct GNUNET_IDENTITY_PROVIDER_Attribute *attr) + const struct GNUNET_IDENTITY_ATTRIBUTE_Claim *attr) { - struct GNUNET_IDENTITY_PROVIDER_AttributeListEntry *le; + struct GNUNET_IDENTITY_ATTRIBUTE_ClaimListEntry *le; char *attrs_tmp; char *attr_str; @@ -275,11 +296,11 @@ iter_cb (void *cls, attr_str = strtok (NULL, ","); continue; } - le = GNUNET_new (struct GNUNET_IDENTITY_PROVIDER_AttributeListEntry); - le->attribute = GNUNET_IDENTITY_PROVIDER_attribute_new (attr->name, - attr->attribute_type, - attr->data, - attr->data_size); + le = GNUNET_new (struct GNUNET_IDENTITY_ATTRIBUTE_ClaimListEntry); + le->claim = GNUNET_IDENTITY_ATTRIBUTE_claim_new (attr->name, + attr->type, + attr->data, + attr->data_size); GNUNET_CONTAINER_DLL_insert (attr_list->list_head, attr_list->list_tail, le); @@ -321,7 +342,7 @@ ego_cb (void *cls, sizeof (struct GNUNET_IDENTITY_PROVIDER_Ticket)); - attr_list = GNUNET_new (struct GNUNET_IDENTITY_PROVIDER_AttributeList); + attr_list = GNUNET_new (struct GNUNET_IDENTITY_ATTRIBUTE_ClaimList); attr_iterator = GNUNET_IDENTITY_PROVIDER_get_attributes_start (idp_handle, pkey, @@ -404,6 +425,11 @@ main(int argc, char *const argv[]) NULL, gettext_noop ("Revoke a ticket"), &revoke_ticket), + GNUNET_GETOPT_option_string ('t', + "type", + NULL, + gettext_noop ("Type of attribute"), + &type_str), GNUNET_GETOPT_OPTION_END }; GNUNET_PROGRAM_run (argc, argv, "ct", diff --git a/src/identity-provider/gnunet-service-identity-provider.c b/src/identity-provider/gnunet-service-identity-provider.c index 2e67dc1eba..a5c178aa59 100644 --- a/src/identity-provider/gnunet-service-identity-provider.c +++ b/src/identity-provider/gnunet-service-identity-provider.c @@ -34,9 +34,9 @@ #include "gnunet_statistics_service.h" #include "gnunet_gns_service.h" #include "gnunet_identity_provider_plugin.h" +#include "gnunet_identity_attribute_lib.h" #include "gnunet_signatures.h" #include "identity_provider.h" -#include "identity_attribute.h" /** * First pass state @@ -365,7 +365,7 @@ struct AttributeStoreHandle /** * The attribute to store */ - struct GNUNET_IDENTITY_PROVIDER_Attribute *attribute; + struct GNUNET_IDENTITY_ATTRIBUTE_Claim *claim; /** * request id @@ -428,7 +428,7 @@ struct ConsumeTicketHandle /** * Attributes */ - struct GNUNET_IDENTITY_PROVIDER_AttributeList *attrs; + struct GNUNET_IDENTITY_ATTRIBUTE_ClaimList *attrs; /** * Lookup time @@ -490,12 +490,12 @@ struct TicketRevocationHandle /** * Attributes to reissue */ - struct GNUNET_IDENTITY_PROVIDER_AttributeList *attrs; + struct GNUNET_IDENTITY_ATTRIBUTE_ClaimList *attrs; /** * Attributes to revoke */ - struct GNUNET_IDENTITY_PROVIDER_AttributeList *rvk_attrs; + struct GNUNET_IDENTITY_ATTRIBUTE_ClaimList *rvk_attrs; /** * Issuer Key @@ -549,7 +549,7 @@ struct TicketIssueHandle /** * Attributes to issue */ - struct GNUNET_IDENTITY_PROVIDER_AttributeList *attrs; + struct GNUNET_IDENTITY_ATTRIBUTE_ClaimList *attrs; /** * Issuer Key @@ -809,7 +809,7 @@ static void cleanup_ticket_issue_handle (struct TicketIssueHandle *handle) { if (NULL != handle->attrs) - attribute_list_destroy (handle->attrs); + GNUNET_IDENTITY_ATTRIBUTE_list_destroy (handle->attrs); if (NULL != handle->ns_qe) GNUNET_NAMESTORE_cancel (handle->ns_qe); GNUNET_free (handle); @@ -820,7 +820,7 @@ static void send_ticket_result (struct IdpClient *client, uint32_t r_id, const struct GNUNET_IDENTITY_PROVIDER_Ticket *ticket, - const struct GNUNET_IDENTITY_PROVIDER_AttributeList *attrs) + const struct GNUNET_IDENTITY_ATTRIBUTE_ClaimList *attrs) { struct TicketResultMessage *irm; struct GNUNET_MQ_Envelope *env; @@ -873,13 +873,13 @@ store_ticket_issue_cont (void *cls, int serialize_abe_keyinfo2 (const struct GNUNET_IDENTITY_PROVIDER_Ticket *ticket, - const struct GNUNET_IDENTITY_PROVIDER_AttributeList *attrs, + const struct GNUNET_IDENTITY_ATTRIBUTE_ClaimList *attrs, const struct GNUNET_CRYPTO_AbeKey *rp_key, struct GNUNET_CRYPTO_EcdhePrivateKey **ecdh_privkey, char **result) { struct GNUNET_CRYPTO_EcdhePublicKey ecdh_pubkey; - struct GNUNET_IDENTITY_PROVIDER_AttributeListEntry *le; + struct GNUNET_IDENTITY_ATTRIBUTE_ClaimListEntry *le; char *enc_keyinfo; char *serialized_key; char *buf; @@ -896,7 +896,7 @@ serialize_abe_keyinfo2 (const struct GNUNET_IDENTITY_PROVIDER_Ticket *ticket, (void**)&serialized_key); attrs_str_len = 0; for (le = attrs->list_head; NULL != le; le = le->next) { - attrs_str_len += strlen (le->attribute->name) + 1; + attrs_str_len += strlen (le->claim->name) + 1; } buf = GNUNET_malloc (attrs_str_len + size); write_ptr = buf; @@ -904,14 +904,14 @@ serialize_abe_keyinfo2 (const struct GNUNET_IDENTITY_PROVIDER_Ticket *ticket, "Writing attributes\n"); for (le = attrs->list_head; NULL != le; le = le->next) { GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "%s\n", le->attribute->name); + "%s\n", le->claim->name); GNUNET_memcpy (write_ptr, - le->attribute->name, - strlen (le->attribute->name)); - write_ptr[strlen (le->attribute->name)] = ','; - write_ptr += strlen (le->attribute->name) + 1; + le->claim->name, + strlen (le->claim->name)); + write_ptr[strlen (le->claim->name)] = ','; + write_ptr += strlen (le->claim->name) + 1; } write_ptr--; write_ptr[0] = '\0'; //replace last , with a 0-terminator @@ -954,7 +954,7 @@ issue_ticket_after_abe_bootstrap (void *cls, struct GNUNET_CRYPTO_AbeMasterKey *abe_key) { struct TicketIssueHandle *ih = cls; - struct GNUNET_IDENTITY_PROVIDER_AttributeListEntry *le; + struct GNUNET_IDENTITY_ATTRIBUTE_ClaimListEntry *le; struct GNUNET_CRYPTO_EcdhePrivateKey *ecdhe_privkey; struct GNUNET_GNSRECORD_Data code_record[1]; struct GNUNET_CRYPTO_AbeKey *rp_key; @@ -974,8 +974,8 @@ issue_ticket_after_abe_bootstrap (void *cls, i = 0; for (le = ih->attrs->list_head; NULL != le; le = le->next) { GNUNET_asprintf (&policy, "%s_%lu", - le->attribute->name, - le->attribute->attribute_version); + le->claim->name, + le->claim->version); GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Adding attribute to key: %s\n", policy); @@ -1061,7 +1061,7 @@ handle_issue_ticket_message (void *cls, ih = GNUNET_new (struct TicketIssueHandle); attrs_len = ntohs (im->attr_len); - ih->attrs = attribute_list_deserialize ((char*)&im[1], attrs_len); + ih->attrs = GNUNET_IDENTITY_ATTRIBUTE_list_deserialize ((char*)&im[1], attrs_len); ih->r_id = ntohl (im->id); ih->client = idp; ih->identity = im->identity; @@ -1087,9 +1087,9 @@ static void cleanup_revoke_ticket_handle (struct TicketRevocationHandle *handle) { if (NULL != handle->attrs) - attribute_list_destroy (handle->attrs); + GNUNET_IDENTITY_ATTRIBUTE_list_destroy (handle->attrs); if (NULL != handle->rvk_attrs) - attribute_list_destroy (handle->rvk_attrs); + GNUNET_IDENTITY_ATTRIBUTE_list_destroy (handle->rvk_attrs); if (NULL != handle->abe_key) GNUNET_CRYPTO_cpabe_delete_master_key (handle->abe_key); if (NULL != handle->ns_qe) @@ -1132,7 +1132,7 @@ send_revocation_finished (struct TicketRevocationHandle *rh, static void ticket_reissue_proc (void *cls, const struct GNUNET_IDENTITY_PROVIDER_Ticket *ticket, - const struct GNUNET_IDENTITY_PROVIDER_AttributeList *attrs); + const struct GNUNET_IDENTITY_ATTRIBUTE_ClaimList *attrs); static void revocation_reissue_tickets (struct TicketRevocationHandle *rh); @@ -1176,11 +1176,11 @@ reissue_ticket_cont (void *cls, static void ticket_reissue_proc (void *cls, const struct GNUNET_IDENTITY_PROVIDER_Ticket *ticket, - const struct GNUNET_IDENTITY_PROVIDER_AttributeList *attrs) + const struct GNUNET_IDENTITY_ATTRIBUTE_ClaimList *attrs) { struct TicketRevocationHandle *rh = cls; - struct GNUNET_IDENTITY_PROVIDER_AttributeListEntry *le; - struct GNUNET_IDENTITY_PROVIDER_AttributeListEntry *le_rollover; + struct GNUNET_IDENTITY_ATTRIBUTE_ClaimListEntry *le; + struct GNUNET_IDENTITY_ATTRIBUTE_ClaimListEntry *le_rollover; struct GNUNET_CRYPTO_EcdhePrivateKey *ecdhe_privkey; struct GNUNET_GNSRECORD_Data code_record[1]; struct GNUNET_CRYPTO_AbeKey *rp_key; @@ -1223,11 +1223,11 @@ ticket_reissue_proc (void *cls, NULL != le_rollover; le_rollover = le_rollover->next) { - if (0 == strcmp (le_rollover->attribute->name, - le->attribute->name)) + if (0 == strcmp (le_rollover->claim->name, + le->claim->name)) { reissue_ticket = GNUNET_YES; - le->attribute->attribute_version = le_rollover->attribute->attribute_version; + le->claim->version = le_rollover->claim->version; } } } @@ -1255,8 +1255,8 @@ ticket_reissue_proc (void *cls, i = 0; for (le = attrs->list_head; NULL != le; le = le->next) { GNUNET_asprintf (&policy, "%s_%lu", - le->attribute->name, - le->attribute->attribute_version); + le->claim->name, + le->claim->version); GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Recreating key with %s\n", policy); attr_arr[i] = policy; @@ -1349,14 +1349,14 @@ reenc_next_attribute (struct TicketRevocationHandle *rh) revocation_reissue_tickets (rh); return; } - buf_size = attribute_serialize_get_size (rh->attrs->list_head->attribute); + buf_size = GNUNET_IDENTITY_ATTRIBUTE_serialize_get_size (rh->attrs->list_head->claim); buf = GNUNET_malloc (buf_size); - attribute_serialize (rh->attrs->list_head->attribute, + GNUNET_IDENTITY_ATTRIBUTE_serialize (rh->attrs->list_head->claim, buf); - rh->attrs->list_head->attribute->attribute_version++; + rh->attrs->list_head->claim->version++; GNUNET_asprintf (&policy, "%s_%lu", - rh->attrs->list_head->attribute->name, - rh->attrs->list_head->attribute->attribute_version); + rh->attrs->list_head->claim->name, + rh->attrs->list_head->claim->version); GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Encrypting with policy %s\n", policy); /** @@ -1371,7 +1371,7 @@ reenc_next_attribute (struct TicketRevocationHandle *rh) GNUNET_free (policy); rd[0].data_size = enc_size + sizeof (uint32_t); rd_buf = GNUNET_malloc (rd[0].data_size); - attr_ver = htonl (rh->attrs->list_head->attribute->attribute_version); + attr_ver = htonl (rh->attrs->list_head->claim->version); GNUNET_memcpy (rd_buf, &attr_ver, sizeof (uint32_t)); @@ -1384,7 +1384,7 @@ reenc_next_attribute (struct TicketRevocationHandle *rh) rd[0].expiration_time = GNUNET_TIME_UNIT_HOURS.rel_value_us; //TODO sane? rh->ns_qe = GNUNET_NAMESTORE_records_store (ns_handle, &rh->identity, - rh->attrs->list_head->attribute->name, + rh->attrs->list_head->claim->name, 1, rd, &attr_reenc_cont, @@ -1403,7 +1403,7 @@ attr_reenc_cont (void *cls, const char *emsg) { struct TicketRevocationHandle *rh = cls; - struct GNUNET_IDENTITY_PROVIDER_AttributeListEntry *le; + struct GNUNET_IDENTITY_ATTRIBUTE_ClaimListEntry *le; if (GNUNET_SYSERR == success) { @@ -1437,11 +1437,11 @@ attr_reenc_cont (void *cls, static void process_attributes_to_update (void *cls, const struct GNUNET_IDENTITY_PROVIDER_Ticket *ticket, - const struct GNUNET_IDENTITY_PROVIDER_AttributeList *attrs) + const struct GNUNET_IDENTITY_ATTRIBUTE_ClaimList *attrs) { struct TicketRevocationHandle *rh = cls; - rh->attrs = attribute_list_dup (attrs); + rh->attrs = GNUNET_IDENTITY_ATTRIBUTE_list_dup (attrs); GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Revocation Phase I: Collecting attributes\n"); /* Reencrypt all attributes with new key */ @@ -1514,7 +1514,7 @@ handle_revoke_ticket_message (void *cls, rh = GNUNET_new (struct TicketRevocationHandle); ticket = (struct GNUNET_IDENTITY_PROVIDER_Ticket*)&rm[1]; - rh->rvk_attrs = GNUNET_new (struct GNUNET_IDENTITY_PROVIDER_AttributeList); + rh->rvk_attrs = GNUNET_new (struct GNUNET_IDENTITY_ATTRIBUTE_ClaimList); rh->ticket = *ticket; rh->r_id = ntohl (rm->id); rh->client = idp; @@ -1537,7 +1537,7 @@ cleanup_consume_ticket_handle (struct ConsumeTicketHandle *handle) GNUNET_CRYPTO_cpabe_delete_key (handle->key, GNUNET_YES); if (NULL != handle->attrs) - attribute_list_destroy (handle->attrs); + GNUNET_IDENTITY_ATTRIBUTE_list_destroy (handle->attrs); GNUNET_free (handle); } @@ -1575,7 +1575,8 @@ process_parallel_lookup2 (void *cls, uint32_t rd_count, struct ConsumeTicketHandle *handle = parallel_lookup->handle; struct ConsumeTicketResultMessage *crm; struct GNUNET_MQ_Envelope *env; - struct GNUNET_IDENTITY_PROVIDER_AttributeListEntry *attr_le; + struct GNUNET_IDENTITY_ATTRIBUTE_ClaimListEntry *attr_le; + struct GNUNET_TIME_Absolute decrypt_duration; char *data; char *data_tmp; ssize_t attr_len; @@ -1601,16 +1602,26 @@ process_parallel_lookup2 (void *cls, uint32_t rd_count, GNUNET_break(0);//TODO if (rd->record_type == GNUNET_GNSRECORD_TYPE_ID_ATTR) { + decrypt_duration = GNUNET_TIME_absolute_get (); attr_len = GNUNET_CRYPTO_cpabe_decrypt (rd->data + sizeof (uint32_t), rd->data_size - sizeof (uint32_t), handle->key, (void**)&data); if (GNUNET_SYSERR != attr_len) { - attr_le = GNUNET_new (struct GNUNET_IDENTITY_PROVIDER_AttributeListEntry); - attr_le->attribute = attribute_deserialize (data, + GNUNET_STATISTICS_update (stats_handle, + "abe_decrypt_time_total", + GNUNET_TIME_absolute_get_duration (decrypt_duration).rel_value_us, + GNUNET_YES); + GNUNET_STATISTICS_update (stats_handle, + "abe_decrypt_count", + 1, + GNUNET_YES); + + attr_le = GNUNET_new (struct GNUNET_IDENTITY_ATTRIBUTE_ClaimListEntry); + attr_le->claim = GNUNET_IDENTITY_ATTRIBUTE_deserialize (data, attr_len); - attr_le->attribute->attribute_version = ntohl(*(uint32_t*)rd->data); + attr_le->claim->version = ntohl(*(uint32_t*)rd->data); GNUNET_CONTAINER_DLL_insert (handle->attrs->list_head, handle->attrs->list_tail, attr_le); @@ -1632,7 +1643,7 @@ process_parallel_lookup2 (void *cls, uint32_t rd_count, } GNUNET_SCHEDULER_cancel (handle->kill_task); - attrs_len = attribute_list_serialize_get_size (handle->attrs); + attrs_len = GNUNET_IDENTITY_ATTRIBUTE_list_serialize_get_size (handle->attrs); env = GNUNET_MQ_msg_extra (crm, attrs_len, GNUNET_MESSAGE_TYPE_IDENTITY_PROVIDER_CONSUME_TICKET_RESULT); @@ -1640,7 +1651,7 @@ process_parallel_lookup2 (void *cls, uint32_t rd_count, crm->attrs_len = htons (attrs_len); crm->identity = handle->ticket.identity; data_tmp = (char *) &crm[1]; - attribute_list_serialize (handle->attrs, + GNUNET_IDENTITY_ATTRIBUTE_list_serialize (handle->attrs, data_tmp); GNUNET_MQ_send (handle->client->mq, env); cleanup_consume_ticket_handle (handle); @@ -1791,7 +1802,7 @@ handle_consume_ticket_message (void *cls, ch->r_id = ntohl (cm->id); ch->client = idp; ch->identity = cm->identity; - ch->attrs = GNUNET_new (struct GNUNET_IDENTITY_PROVIDER_AttributeList); + ch->attrs = GNUNET_new (struct GNUNET_IDENTITY_ATTRIBUTE_ClaimList); GNUNET_CRYPTO_ecdsa_key_get_public (&ch->identity, &ch->identity_pub); ch->ticket = *((struct GNUNET_IDENTITY_PROVIDER_Ticket*)&cm[1]); @@ -1819,8 +1830,8 @@ handle_consume_ticket_message (void *cls, static void cleanup_as_handle (struct AttributeStoreHandle *handle) { - if (NULL != handle->attribute) - GNUNET_free (handle->attribute); + if (NULL != handle->claim) + GNUNET_free (handle->claim); if (NULL != handle->abe_key) GNUNET_CRYPTO_cpabe_delete_master_key (handle->abe_key); GNUNET_free (handle); @@ -1871,16 +1882,16 @@ attr_store_task (void *cls) GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Storing attribute\n"); - buf_size = attribute_serialize_get_size (as_handle->attribute); + buf_size = GNUNET_IDENTITY_ATTRIBUTE_serialize_get_size (as_handle->claim); buf = GNUNET_malloc (buf_size); - attribute_serialize (as_handle->attribute, + GNUNET_IDENTITY_ATTRIBUTE_serialize (as_handle->claim, buf); GNUNET_asprintf (&policy, "%s_%lu", - as_handle->attribute->name, - as_handle->attribute->attribute_version); + as_handle->claim->name, + as_handle->claim->version); GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Encrypting with policy %s\n", policy); /** @@ -1895,7 +1906,7 @@ attr_store_task (void *cls) GNUNET_free (policy); rd[0].data_size = enc_size + sizeof (uint32_t); rd_buf = GNUNET_malloc (rd[0].data_size); - attr_ver = htonl (as_handle->attribute->attribute_version); + attr_ver = htonl (as_handle->claim->version); GNUNET_memcpy (rd_buf, &attr_ver, sizeof (uint32_t)); @@ -1908,7 +1919,7 @@ attr_store_task (void *cls) rd[0].expiration_time = GNUNET_TIME_UNIT_HOURS.rel_value_us; //TODO sane? as_handle->ns_qe = GNUNET_NAMESTORE_records_store (ns_handle, &as_handle->identity, - as_handle->attribute->name, + as_handle->claim->name, 1, rd, &attr_store_cont, @@ -1973,7 +1984,7 @@ handle_attribute_store_message (void *cls, data_len = ntohs (sam->attr_len); as_handle = GNUNET_new (struct AttributeStoreHandle); - as_handle->attribute = attribute_deserialize ((char*)&sam[1], + as_handle->claim = GNUNET_IDENTITY_ATTRIBUTE_deserialize ((char*)&sam[1], data_len); as_handle->r_id = ntohl (sam->id); @@ -2269,7 +2280,7 @@ cleanup_ticket_iter_handle (struct TicketIteration *ti) static void ticket_iterate_proc (void *cls, const struct GNUNET_IDENTITY_PROVIDER_Ticket *ticket, - const struct GNUNET_IDENTITY_PROVIDER_AttributeList *attrs) + const struct GNUNET_IDENTITY_ATTRIBUTE_ClaimList *attrs) { struct TicketIterationProcResult *proc = cls; diff --git a/src/identity-provider/identity_attribute.c b/src/identity-provider/identity_attribute.c deleted file mode 100644 index c7e833326c..0000000000 --- a/src/identity-provider/identity_attribute.c +++ /dev/null @@ -1,245 +0,0 @@ -/* - This file is part of GNUnet - Copyright (C) 2010-2015 GNUnet e.V. - - GNUnet is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published - by the Free Software Foundation; either version 3, or (at your - option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - General Public License for more details. - - You should have received a copy of the GNU General Public License - along with GNUnet; see the file COPYING. If not, write to the - Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, - Boston, MA 02110-1301, USA. - */ - -/** - * @file identity-provider/identity_attribute.c - * @brief helper library to manage identity attributes - * @author Martin Schanzenbach - */ -#include "platform.h" -#include "gnunet_util_lib.h" -#include "identity_attribute.h" - -/** - * Create a new attribute. - * - * @param name the attribute name - * @param type the attribute type - * @param data the attribute value - * @param data_size the attribute value size - * @return the new attribute - */ -struct GNUNET_IDENTITY_PROVIDER_Attribute * -attribute_new (const char* attr_name, - uint32_t attr_type, - const void* data, - size_t data_size) -{ - struct GNUNET_IDENTITY_PROVIDER_Attribute *attr; - char *write_ptr; - - attr = GNUNET_malloc (sizeof (struct GNUNET_IDENTITY_PROVIDER_Attribute) + - strlen (attr_name) + 1 + - data_size); - attr->attribute_type = attr_type; - attr->data_size = data_size; - write_ptr = (char*)&attr[1]; - GNUNET_memcpy (write_ptr, - attr_name, - strlen (attr_name) + 1); - attr->name = write_ptr; - write_ptr += strlen (attr->name) + 1; - GNUNET_memcpy (write_ptr, - data, - data_size); - attr->data = write_ptr; - return attr; -} - -size_t -attribute_list_serialize_get_size (const struct GNUNET_IDENTITY_PROVIDER_AttributeList *attrs) -{ - struct GNUNET_IDENTITY_PROVIDER_AttributeListEntry *le; - size_t len = 0; - for (le = attrs->list_head; NULL != le; le = le->next) - len += attribute_serialize_get_size (le->attribute); - return len; -} - -size_t -attribute_list_serialize (const struct GNUNET_IDENTITY_PROVIDER_AttributeList *attrs, - char *result) -{ - struct GNUNET_IDENTITY_PROVIDER_AttributeListEntry *le; - size_t len; - size_t total_len; - char* write_ptr; - - write_ptr = result; - total_len = 0; - for (le = attrs->list_head; NULL != le; le = le->next) - { - len = attribute_serialize (le->attribute, - write_ptr); - total_len += len; - write_ptr += len; - } - return total_len; -} - -struct GNUNET_IDENTITY_PROVIDER_AttributeList * -attribute_list_deserialize (const char* data, - size_t data_size) -{ - struct GNUNET_IDENTITY_PROVIDER_AttributeList *attrs; - struct GNUNET_IDENTITY_PROVIDER_AttributeListEntry *le; - size_t attr_len; - const char* read_ptr; - - if (data_size < sizeof (struct Attribute)) - return NULL; - - attrs = GNUNET_new (struct GNUNET_IDENTITY_PROVIDER_AttributeList); - read_ptr = data; - while (((data + data_size) - read_ptr) >= sizeof (struct Attribute)) - { - - le = GNUNET_new (struct GNUNET_IDENTITY_PROVIDER_AttributeListEntry); - le->attribute = attribute_deserialize (read_ptr, - data_size - (read_ptr - data)); - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Deserialized attribute %s\n", le->attribute->name); - GNUNET_CONTAINER_DLL_insert (attrs->list_head, - attrs->list_tail, - le); - attr_len = attribute_serialize_get_size (le->attribute); - read_ptr += attr_len; - } - return attrs; -} - -struct GNUNET_IDENTITY_PROVIDER_AttributeList* -attribute_list_dup (const struct GNUNET_IDENTITY_PROVIDER_AttributeList *attrs) -{ - struct GNUNET_IDENTITY_PROVIDER_AttributeListEntry *le; - struct GNUNET_IDENTITY_PROVIDER_AttributeListEntry *result_le; - struct GNUNET_IDENTITY_PROVIDER_AttributeList *result; - size_t len; - - result = GNUNET_new (struct GNUNET_IDENTITY_PROVIDER_AttributeList); - for (le = attrs->list_head; NULL != le; le = le->next) - { - result_le = GNUNET_new (struct GNUNET_IDENTITY_PROVIDER_AttributeListEntry); - len = sizeof (struct GNUNET_IDENTITY_PROVIDER_Attribute) + le->attribute->data_size; - result_le->attribute = GNUNET_malloc (len); - GNUNET_memcpy (result_le->attribute, - le->attribute, - len); - result_le->attribute->name = (const char*)&result_le->attribute[1]; - GNUNET_CONTAINER_DLL_insert (result->list_head, - result->list_tail, - result_le); - } - return result; -} - - -void -attribute_list_destroy (struct GNUNET_IDENTITY_PROVIDER_AttributeList *attrs) -{ - struct GNUNET_IDENTITY_PROVIDER_AttributeListEntry *le; - struct GNUNET_IDENTITY_PROVIDER_AttributeListEntry *tmp_le; - - for (le = attrs->list_head; NULL != le;) - { - GNUNET_free (le->attribute); - tmp_le = le; - le = le->next; - GNUNET_free (tmp_le); - } - GNUNET_free (attrs); - -} - -size_t -attribute_serialize_get_size (const struct GNUNET_IDENTITY_PROVIDER_Attribute *attr) -{ - return sizeof (struct Attribute) - + strlen (attr->name) - + attr->data_size; //TODO get data_size from plugin -} - -size_t -attribute_serialize (const struct GNUNET_IDENTITY_PROVIDER_Attribute *attr, - char *result) -{ - size_t data_len_ser; - size_t name_len; - struct Attribute *attr_ser; - char* write_ptr; - - attr_ser = (struct Attribute*)result; - attr_ser->attribute_type = htons (attr->attribute_type); - attr_ser->attribute_version = htonl (attr->attribute_version); - name_len = strlen (attr->name); - attr_ser->name_len = htons (name_len); - write_ptr = (char*)&attr_ser[1]; - GNUNET_memcpy (write_ptr, attr->name, name_len); - write_ptr += name_len; - //TODO plugin-ize - //data_len_ser = plugin->serialize_attribute_value (attr, - // &attr_ser[1]); - data_len_ser = attr->data_size; - GNUNET_memcpy (write_ptr, attr->data, attr->data_size); - attr_ser->data_size = htons (data_len_ser); - - return sizeof (struct Attribute) + strlen (attr->name) + attr->data_size; -} - -struct GNUNET_IDENTITY_PROVIDER_Attribute * -attribute_deserialize (const char* data, - size_t data_size) -{ - struct GNUNET_IDENTITY_PROVIDER_Attribute *attr; - struct Attribute *attr_ser; - size_t data_len; - size_t name_len; - char* write_ptr; - - if (data_size < sizeof (struct Attribute)) - return NULL; - - attr_ser = (struct Attribute*)data; - //TODO use plugin. - data_len = ntohs (attr_ser->data_size); - name_len = ntohs (attr_ser->name_len); - attr = GNUNET_malloc (sizeof (struct GNUNET_IDENTITY_PROVIDER_Attribute) - + data_len + name_len + 1); - attr->attribute_type = ntohs (attr_ser->attribute_type); - attr->attribute_version = ntohl (attr_ser->attribute_version); - attr->data_size = ntohs (attr_ser->data_size); - - write_ptr = (char*)&attr[1]; - GNUNET_memcpy (write_ptr, - &attr_ser[1], - name_len); - write_ptr[name_len] = '\0'; - attr->name = write_ptr; - - write_ptr += name_len + 1; - GNUNET_memcpy (write_ptr, - (char*)&attr_ser[1] + name_len, - attr->data_size); - attr->data = write_ptr; - return attr; - -} - -/* end of identity_attribute.c */ diff --git a/src/identity-provider/identity_attribute.h b/src/identity-provider/identity_attribute.h deleted file mode 100644 index 3e8fadccf7..0000000000 --- a/src/identity-provider/identity_attribute.h +++ /dev/null @@ -1,149 +0,0 @@ -/* - This file is part of GNUnet. - Copyright (C) 2012-2015 GNUnet e.V. - - GNUnet is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published - by the Free Software Foundation; either version 3, or (at your - option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - General Public License for more details. - - You should have received a copy of the GNU General Public License - along with GNUnet; see the file COPYING. If not, write to the - Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, - Boston, MA 02110-1301, USA. - */ -/** - * @author Martin Schanzenbach - * @file identity-provider/identity_attribute.h - * @brief GNUnet Identity Provider library - * - */ -#ifndef IDENTITY_ATTRIBUTE_H -#define IDENTITY_ATTRIBUTE_H - -#include "gnunet_identity_provider_service.h" - -struct Attribute -{ - /** - * Attribute type - */ - uint32_t attribute_type; - - /** - * Attribute version - */ - uint32_t attribute_version; - - /** - * Name length - */ - uint32_t name_len; - - /** - * Data size - */ - uint32_t data_size; - - //followed by data_size Attribute value data -}; - -/** - * Get required size for serialization buffer - * - * @param attrs the attribute list to serialize - * - * @return the required buffer size - */ -size_t -attribute_list_serialize_get_size (const struct GNUNET_IDENTITY_PROVIDER_AttributeList *attrs); - -void -attribute_list_destroy (struct GNUNET_IDENTITY_PROVIDER_AttributeList *attrs); - - -/** - * Serialize an attribute list - * - * @param attrs the attribute list to serialize - * @param result the serialized attribute - * - * @return length of serialized data - */ -size_t -attribute_list_serialize (const struct GNUNET_IDENTITY_PROVIDER_AttributeList *attrs, - char *result); - -/** - * Deserialize an attribute list - * - * @param data the serialized attribute list - * @param data_size the length of the serialized data - * - * @return a GNUNET_IDENTITY_PROVIDER_AttributeList, must be free'd by caller - */ -struct GNUNET_IDENTITY_PROVIDER_AttributeList * -attribute_list_deserialize (const char* data, - size_t data_size); - - -/** - * Get required size for serialization buffer - * - * @param attr the attribute to serialize - * - * @return the required buffer size - */ -size_t -attribute_serialize_get_size (const struct GNUNET_IDENTITY_PROVIDER_Attribute *attr); - - - -/** - * Serialize an attribute - * - * @param attr the attribute to serialize - * @param result the serialized attribute - * - * @return length of serialized data - */ -size_t -attribute_serialize (const struct GNUNET_IDENTITY_PROVIDER_Attribute *attr, - char *result); - -/** - * Deserialize an attribute - * - * @param data the serialized attribute - * @param data_size the length of the serialized data - * - * @return a GNUNET_IDENTITY_PROVIDER_Attribute, must be free'd by caller - */ -struct GNUNET_IDENTITY_PROVIDER_Attribute * -attribute_deserialize (const char* data, - size_t data_size); - -/** - * Create a new attribute. - * - * @param name the attribute name - * @param type the attribute type - * @param data the attribute value - * @param data_size the attribute value size - * @return the new attribute - */ -struct GNUNET_IDENTITY_PROVIDER_Attribute * -attribute_new (const char* attr_name, - uint32_t attr_type, - const void* data, - size_t data_size); - -struct GNUNET_IDENTITY_PROVIDER_AttributeList* -attribute_list_dup (const struct GNUNET_IDENTITY_PROVIDER_AttributeList *attrs); - -#endif diff --git a/src/identity-provider/identity_provider_api.c b/src/identity-provider/identity_provider_api.c index 6d28709dfe..6fc8d228a5 100644 --- a/src/identity-provider/identity_provider_api.c +++ b/src/identity-provider/identity_provider_api.c @@ -29,13 +29,12 @@ #include "gnunet_protocols.h" #include "gnunet_mq_lib.h" #include "gnunet_identity_provider_service.h" +#include "gnunet_identity_attribute_lib.h" #include "identity_provider.h" -#include "identity_attribute.h" #define LOG(kind,...) GNUNET_log_from (kind, "identity-api",__VA_ARGS__) - /** * Handle for an operation with the service. */ @@ -495,9 +494,9 @@ handle_consume_ticket_result (void *cls, return; { - struct GNUNET_IDENTITY_PROVIDER_AttributeList *attrs; - struct GNUNET_IDENTITY_PROVIDER_AttributeListEntry *le; - attrs = attribute_list_deserialize ((char*)&msg[1], + struct GNUNET_IDENTITY_ATTRIBUTE_ClaimList *attrs; + struct GNUNET_IDENTITY_ATTRIBUTE_ClaimListEntry *le; + attrs = GNUNET_IDENTITY_ATTRIBUTE_list_deserialize ((char*)&msg[1], attrs_len); if (NULL != op->ar_cb) { @@ -512,8 +511,8 @@ handle_consume_ticket_result (void *cls, for (le = attrs->list_head; NULL != le; le = le->next) op->ar_cb (op->cls, &msg->identity, - le->attribute); - attribute_list_destroy (attrs); + le->claim); + GNUNET_IDENTITY_ATTRIBUTE_list_destroy (attrs); } } op->ar_cb (op->cls, @@ -619,9 +618,9 @@ handle_attribute_result (void *cls, } { - struct GNUNET_IDENTITY_PROVIDER_Attribute *attr; - attr = attribute_deserialize ((char*)&msg[1], - attr_len); + struct GNUNET_IDENTITY_ATTRIBUTE_Claim *attr; + attr = GNUNET_IDENTITY_ATTRIBUTE_deserialize ((char*)&msg[1], + attr_len); if (NULL != it) { if (NULL != it->proc) @@ -905,7 +904,7 @@ GNUNET_IDENTITY_PROVIDER_disconnect (struct GNUNET_IDENTITY_PROVIDER_Handle *h) struct GNUNET_IDENTITY_PROVIDER_Operation * GNUNET_IDENTITY_PROVIDER_attribute_store (struct GNUNET_IDENTITY_PROVIDER_Handle *h, const struct GNUNET_CRYPTO_EcdsaPrivateKey *pkey, - const struct GNUNET_IDENTITY_PROVIDER_Attribute *attr, + const struct GNUNET_IDENTITY_ATTRIBUTE_Claim *attr, GNUNET_IDENTITY_PROVIDER_ContinuationWithStatus cont, void *cont_cls) { @@ -921,14 +920,14 @@ GNUNET_IDENTITY_PROVIDER_attribute_store (struct GNUNET_IDENTITY_PROVIDER_Handle GNUNET_CONTAINER_DLL_insert_tail (h->op_head, h->op_tail, op); - attr_len = attribute_serialize_get_size (attr); + attr_len = GNUNET_IDENTITY_ATTRIBUTE_serialize_get_size (attr); op->env = GNUNET_MQ_msg_extra (sam, attr_len, GNUNET_MESSAGE_TYPE_IDENTITY_PROVIDER_ATTRIBUTE_STORE); sam->identity = *pkey; sam->id = htonl (op->r_id); - attribute_serialize (attr, + GNUNET_IDENTITY_ATTRIBUTE_serialize (attr, (char*)&sam[1]); sam->attr_len = htons (attr_len); @@ -941,24 +940,6 @@ GNUNET_IDENTITY_PROVIDER_attribute_store (struct GNUNET_IDENTITY_PROVIDER_Handle /** - * Create a new attribute. - * - * @param name the attribute name - * @param type the attribute type - * @param data the attribute value - * @param data_size the attribute value size - * @return the new attribute - */ -struct GNUNET_IDENTITY_PROVIDER_Attribute * -GNUNET_IDENTITY_PROVIDER_attribute_new (const char* attr_name, - uint32_t attr_type, - const void* data, - size_t data_size) -{ - return attribute_new (attr_name, attr_type, data, data_size); -} - -/** * List all attributes for a local identity. * This MUST lock the `struct GNUNET_IDENTITY_PROVIDER_Handle` * for any other calls than #GNUNET_IDENTITY_PROVIDER_get_attributes_next() and @@ -1089,7 +1070,7 @@ struct GNUNET_IDENTITY_PROVIDER_Operation * GNUNET_IDENTITY_PROVIDER_ticket_issue (struct GNUNET_IDENTITY_PROVIDER_Handle *h, const struct GNUNET_CRYPTO_EcdsaPrivateKey *iss, const struct GNUNET_CRYPTO_EcdsaPublicKey *rp, - const struct GNUNET_IDENTITY_PROVIDER_AttributeList *attrs, + const struct GNUNET_IDENTITY_ATTRIBUTE_ClaimList *attrs, GNUNET_IDENTITY_PROVIDER_TicketCallback cb, void *cb_cls) { @@ -1105,7 +1086,7 @@ GNUNET_IDENTITY_PROVIDER_ticket_issue (struct GNUNET_IDENTITY_PROVIDER_Handle *h GNUNET_CONTAINER_DLL_insert_tail (h->op_head, h->op_tail, op); - attr_len = attribute_list_serialize_get_size (attrs); + attr_len = GNUNET_IDENTITY_ATTRIBUTE_list_serialize_get_size (attrs); op->env = GNUNET_MQ_msg_extra (tim, attr_len, GNUNET_MESSAGE_TYPE_IDENTITY_PROVIDER_ISSUE_TICKET); @@ -1113,7 +1094,7 @@ GNUNET_IDENTITY_PROVIDER_ticket_issue (struct GNUNET_IDENTITY_PROVIDER_Handle *h tim->rp = *rp; tim->id = htonl (op->r_id); - attribute_list_serialize (attrs, + GNUNET_IDENTITY_ATTRIBUTE_list_serialize (attrs, (char*)&tim[1]); tim->attr_len = htons (attr_len); diff --git a/src/identity-provider/plugin_identity_provider_sqlite.c b/src/identity-provider/plugin_identity_provider_sqlite.c index c87f30e1ca..594e4788d4 100644 --- a/src/identity-provider/plugin_identity_provider_sqlite.c +++ b/src/identity-provider/plugin_identity_provider_sqlite.c @@ -27,7 +27,7 @@ #include "platform.h" #include "gnunet_identity_provider_service.h" #include "gnunet_identity_provider_plugin.h" -#include "identity_attribute.h" +#include "gnunet_identity_attribute_lib.h" #include "gnunet_sq_lib.h" #include <sqlite3.h> @@ -373,7 +373,7 @@ database_shutdown (struct Plugin *plugin) static int identity_provider_sqlite_store_ticket (void *cls, const struct GNUNET_IDENTITY_PROVIDER_Ticket *ticket, - const struct GNUNET_IDENTITY_PROVIDER_AttributeList *attrs) + const struct GNUNET_IDENTITY_ATTRIBUTE_ClaimList *attrs) { struct Plugin *plugin = cls; size_t attrs_len; @@ -402,9 +402,9 @@ identity_provider_sqlite_store_ticket (void *cls, GNUNET_SQ_reset (plugin->dbh, plugin->delete_ticket); - attrs_len = attribute_list_serialize_get_size (attrs); + attrs_len = GNUNET_IDENTITY_ATTRIBUTE_list_serialize_get_size (attrs); attrs_ser = GNUNET_malloc (attrs_len); - attribute_list_serialize (attrs, + GNUNET_IDENTITY_ATTRIBUTE_list_serialize (attrs, attrs_ser); struct GNUNET_SQ_QueryParam sparams[] = { GNUNET_SQ_query_param_auto_from_type (&ticket->identity), @@ -526,7 +526,7 @@ get_ticket_and_call_iterator (struct Plugin *plugin, void *iter_cls) { struct GNUNET_IDENTITY_PROVIDER_Ticket ticket; - struct GNUNET_IDENTITY_PROVIDER_AttributeList *attrs; + struct GNUNET_IDENTITY_ATTRIBUTE_ClaimList *attrs; int ret; int sret; size_t attrs_len; @@ -553,13 +553,13 @@ get_ticket_and_call_iterator (struct Plugin *plugin, } else { - attrs = attribute_list_deserialize (attrs_ser, + attrs = GNUNET_IDENTITY_ATTRIBUTE_list_deserialize (attrs_ser, attrs_len); if (NULL != iter) iter (iter_cls, &ticket, attrs); - attribute_list_destroy (attrs); + GNUNET_IDENTITY_ATTRIBUTE_list_destroy (attrs); ret = GNUNET_YES; } GNUNET_SQ_cleanup_result (rs); diff --git a/src/identity-provider/plugin_rest_identity_provider.c b/src/identity-provider/plugin_rest_identity_provider.c index ff28b592e3..d5e453a0e6 100644 --- a/src/identity-provider/plugin_rest_identity_provider.c +++ b/src/identity-provider/plugin_rest_identity_provider.c @@ -37,6 +37,7 @@ #include <jansson.h> #include <inttypes.h> #include "gnunet_signatures.h" +#include "gnunet_identity_attribute_lib.h" #include "gnunet_identity_provider_service.h" /** @@ -514,7 +515,7 @@ add_attribute_cont (struct GNUNET_REST_RequestHandle *con_handle, struct RequestHandle *handle = cls; struct EgoEntry *ego_entry; struct MHD_Response *resp; - struct GNUNET_IDENTITY_PROVIDER_Attribute *attribute; + struct GNUNET_IDENTITY_ATTRIBUTE_Claim *attribute; struct GNUNET_JSONAPI_Document *json_obj; struct GNUNET_JSONAPI_Resource *json_res; char term_data[handle->rest_handle->data_size+1]; @@ -602,8 +603,8 @@ add_attribute_cont (struct GNUNET_REST_RequestHandle *con_handle, value_json = GNUNET_JSONAPI_resource_read_attr (json_res, "value"); value_str = json_string_value (value_json); - attribute = GNUNET_IDENTITY_PROVIDER_attribute_new (name_str, - GNUNET_IDENTITY_PROVIDER_AT_STRING, + attribute = GNUNET_IDENTITY_ATTRIBUTE_claim_new (name_str, + GNUNET_IDENTITY_ATTRIBUTE_TYPE_STRING, value_str, strlen (value_str) + 1); handle->idp = GNUNET_IDENTITY_PROVIDER_connect (cfg); @@ -625,7 +626,7 @@ add_attribute_cont (struct GNUNET_REST_RequestHandle *con_handle, static void attr_collect (void *cls, const struct GNUNET_CRYPTO_EcdsaPublicKey *identity, - const struct GNUNET_IDENTITY_PROVIDER_Attribute *attr) + const struct GNUNET_IDENTITY_ATTRIBUTE_Claim *attr) { struct GNUNET_JSONAPI_Resource *json_resource; struct RequestHandle *handle = cls; @@ -839,7 +840,7 @@ revoke_ticket_cont (struct GNUNET_REST_RequestHandle *con_handle, static void consume_cont (void *cls, const struct GNUNET_CRYPTO_EcdsaPublicKey *identity, - const struct GNUNET_IDENTITY_PROVIDER_Attribute *attr) + const struct GNUNET_IDENTITY_ATTRIBUTE_Claim *attr) { struct RequestHandle *handle = cls; struct GNUNET_JSONAPI_Resource *json_resource; diff --git a/src/include/gnunet_crypto_lib.h b/src/include/gnunet_crypto_lib.h index 2b6718557c..2fd67ae1b1 100644 --- a/src/include/gnunet_crypto_lib.h +++ b/src/include/gnunet_crypto_lib.h @@ -1115,6 +1115,16 @@ GNUNET_CRYPTO_ecdsa_public_key_to_string (const struct GNUNET_CRYPTO_EcdsaPublic /** + * Convert a private key to a string. + * + * @param priv key to convert + * @return string representing @a pub + */ +char * +GNUNET_CRYPTO_eddsa_private_key_to_string (const struct GNUNET_CRYPTO_EddsaPrivateKey *priv); + + +/** * Convert a public key to a string. * * @param pub key to convert @@ -2021,13 +2031,14 @@ GNUNET_CRYPTO_rsa_public_key_cmp (struct GNUNET_CRYPTO_RsaPublicKey *p1, * @param pkey the public key of the signer * @param[out] buf set to a buffer with the blinded message to be signed * @param[out] buf_size number of bytes stored in @a buf - * @return GNUNET_YES if successful, GNUNET_NO if RSA key is malicious + * @return #GNUNET_YES if successful, #GNUNET_NO if RSA key is malicious */ int GNUNET_CRYPTO_rsa_blind (const struct GNUNET_HashCode *hash, const struct GNUNET_CRYPTO_RsaBlindingKeySecret *bks, struct GNUNET_CRYPTO_RsaPublicKey *pkey, - char **buf, size_t *buf_size); + char **buf, + size_t *buf_size); /** @@ -2040,7 +2051,8 @@ GNUNET_CRYPTO_rsa_blind (const struct GNUNET_HashCode *hash, */ struct GNUNET_CRYPTO_RsaSignature * GNUNET_CRYPTO_rsa_sign_blinded (const struct GNUNET_CRYPTO_RsaPrivateKey *key, - const void *msg, size_t msg_len); + const void *msg, + size_t msg_len); /** @@ -2110,7 +2122,7 @@ GNUNET_CRYPTO_rsa_signature_dup (const struct GNUNET_CRYPTO_RsaSignature *sig); * @return unblinded signature on success, NULL if RSA key is bad or malicious. */ struct GNUNET_CRYPTO_RsaSignature * -GNUNET_CRYPTO_rsa_unblind (struct GNUNET_CRYPTO_RsaSignature *sig, +GNUNET_CRYPTO_rsa_unblind (const struct GNUNET_CRYPTO_RsaSignature *sig, const struct GNUNET_CRYPTO_RsaBlindingKeySecret *bks, struct GNUNET_CRYPTO_RsaPublicKey *pkey); diff --git a/src/include/gnunet_getopt_lib.h b/src/include/gnunet_getopt_lib.h index f707bb0919..e38925f147 100644 --- a/src/include/gnunet_getopt_lib.h +++ b/src/include/gnunet_getopt_lib.h @@ -230,11 +230,11 @@ GNUNET_GETOPT_option_filename (char shortName, */ struct GNUNET_GETOPT_CommandLineOption GNUNET_GETOPT_option_base32_fixed_size (char shortName, - const char *name, - const char *argumentHelp, - const char *description, - void *val, - size_t val_size); + const char *name, + const char *argumentHelp, + const char *description, + void *val, + size_t val_size); /** @@ -264,9 +264,9 @@ GNUNET_GETOPT_option_base32_fixed_size (char shortName, */ struct GNUNET_GETOPT_CommandLineOption GNUNET_GETOPT_option_flag (char shortName, - const char *name, - const char *description, - int *val); + const char *name, + const char *description, + int *val); /** @@ -280,10 +280,10 @@ GNUNET_GETOPT_option_flag (char shortName, */ struct GNUNET_GETOPT_CommandLineOption GNUNET_GETOPT_option_uint (char shortName, - const char *name, - const char *argumentHelp, - const char *description, - unsigned int *val); + const char *name, + const char *argumentHelp, + const char *description, + unsigned int *val); /** @@ -297,10 +297,10 @@ GNUNET_GETOPT_option_uint (char shortName, */ struct GNUNET_GETOPT_CommandLineOption GNUNET_GETOPT_option_ulong (char shortName, - const char *name, - const char *argumentHelp, - const char *description, - unsigned long long *val); + const char *name, + const char *argumentHelp, + const char *description, + unsigned long long *val); /** @@ -315,10 +315,10 @@ GNUNET_GETOPT_option_ulong (char shortName, */ struct GNUNET_GETOPT_CommandLineOption GNUNET_GETOPT_option_relative_time (char shortName, - const char *name, - const char *argumentHelp, - const char *description, - struct GNUNET_TIME_Relative *val); + const char *name, + const char *argumentHelp, + const char *description, + struct GNUNET_TIME_Relative *val); /** @@ -333,10 +333,10 @@ GNUNET_GETOPT_option_relative_time (char shortName, */ struct GNUNET_GETOPT_CommandLineOption GNUNET_GETOPT_option_absolute_time (char shortName, - const char *name, - const char *argumentHelp, - const char *description, - struct GNUNET_TIME_Absolute *val); + const char *name, + const char *argumentHelp, + const char *description, + struct GNUNET_TIME_Absolute *val); /** @@ -350,9 +350,9 @@ GNUNET_GETOPT_option_absolute_time (char shortName, */ struct GNUNET_GETOPT_CommandLineOption GNUNET_GETOPT_option_increment_uint (char shortName, - const char *name, - const char *description, - unsigned int *val); + const char *name, + const char *description, + unsigned int *val); /** diff --git a/src/include/gnunet_identity_attribute_lib.h b/src/include/gnunet_identity_attribute_lib.h new file mode 100644 index 0000000000..4c765515b7 --- /dev/null +++ b/src/include/gnunet_identity_attribute_lib.h @@ -0,0 +1,290 @@ +/* + This file is part of GNUnet. + Copyright (C) 2017 GNUnet e.V. + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + Boston, MA 02110-1301, USA. +*/ + +/** + * @author Martin Schanzenbach + * + * @file + * Identity attribute definitions + * + * @defgroup identity-provider Identity Provider service + * @{ + */ +#ifndef GNUNET_IDENTITY_ATTRIBUTE_LIB_H +#define GNUNET_IDENTITY_ATTRIBUTE_LIB_H + +#ifdef __cplusplus +extern "C" +{ +#if 0 /* keep Emacsens' auto-indent happy */ +} +#endif +#endif + +#include "gnunet_util_lib.h" + + +/** + * No value attribute. + */ +#define GNUNET_IDENTITY_ATTRIBUTE_TYPE_NONE 0 + +/** + * String attribute. + */ +#define GNUNET_IDENTITY_ATTRIBUTE_TYPE_STRING 1 + + + +/** + * An attribute. + */ +struct GNUNET_IDENTITY_ATTRIBUTE_Claim +{ + /** + * The name of the attribute. Note "name" must never be individually + * free'd + */ + const char* name; + + /** + * Type of Claim + */ + uint32_t type; + + /** + * Version + */ + uint32_t version; + + /** + * Number of bytes in @e data. + */ + size_t data_size; + + /** + * Binary value stored as attribute value. Note: "data" must never + * be individually 'malloc'ed, but instead always points into some + * existing data area. + */ + const void *data; + +}; + +struct GNUNET_IDENTITY_ATTRIBUTE_ClaimList +{ + /** + * List head + */ + struct GNUNET_IDENTITY_ATTRIBUTE_ClaimListEntry *list_head; + + /** + * List tail + */ + struct GNUNET_IDENTITY_ATTRIBUTE_ClaimListEntry *list_tail; +}; + +struct GNUNET_IDENTITY_ATTRIBUTE_ClaimListEntry +{ + /** + * DLL + */ + struct GNUNET_IDENTITY_ATTRIBUTE_ClaimListEntry *prev; + + /** + * DLL + */ + struct GNUNET_IDENTITY_ATTRIBUTE_ClaimListEntry *next; + + /** + * The attribute claim + */ + struct GNUNET_IDENTITY_ATTRIBUTE_Claim *claim; +}; + +/** + * Create a new attribute claim. + * + * @param name the attribute name + * @param type the attribute type + * @param data the attribute value + * @param data_size the attribute value size + * @return the new attribute + */ +struct GNUNET_IDENTITY_ATTRIBUTE_Claim * +GNUNET_IDENTITY_ATTRIBUTE_claim_new (const char* attr_name, + uint32_t type, + const void* data, + size_t data_size); + + +/** + * Get required size for serialization buffer + * + * @param attrs the attribute list to serialize + * + * @return the required buffer size + */ +size_t +GNUNET_IDENTITY_ATTRIBUTE_list_serialize_get_size (const struct GNUNET_IDENTITY_ATTRIBUTE_ClaimList *attrs); + +void +GNUNET_IDENTITY_ATTRIBUTE_list_destroy (struct GNUNET_IDENTITY_ATTRIBUTE_ClaimList *attrs); + + +/** + * Serialize an attribute list + * + * @param attrs the attribute list to serialize + * @param result the serialized attribute + * + * @return length of serialized data + */ +size_t +GNUNET_IDENTITY_ATTRIBUTE_list_serialize (const struct GNUNET_IDENTITY_ATTRIBUTE_ClaimList *attrs, + char *result); + +/** + * Deserialize an attribute list + * + * @param data the serialized attribute list + * @param data_size the length of the serialized data + * + * @return a GNUNET_IDENTITY_PROVIDER_AttributeList, must be free'd by caller + */ +struct GNUNET_IDENTITY_ATTRIBUTE_ClaimList * +GNUNET_IDENTITY_ATTRIBUTE_list_deserialize (const char* data, + size_t data_size); + + +/** + * Get required size for serialization buffer + * + * @param attr the attribute to serialize + * + * @return the required buffer size + */ +size_t +GNUNET_IDENTITY_ATTRIBUTE_serialize_get_size (const struct GNUNET_IDENTITY_ATTRIBUTE_Claim *attr); + + + +/** + * Serialize an attribute + * + * @param attr the attribute to serialize + * @param result the serialized attribute + * + * @return length of serialized data + */ +size_t +GNUNET_IDENTITY_ATTRIBUTE_serialize (const struct GNUNET_IDENTITY_ATTRIBUTE_Claim *attr, + char *result); + +/** + * Deserialize an attribute + * + * @param data the serialized attribute + * @param data_size the length of the serialized data + * + * @return a GNUNET_IDENTITY_PROVIDER_Attribute, must be free'd by caller + */ +struct GNUNET_IDENTITY_ATTRIBUTE_Claim * +GNUNET_IDENTITY_ATTRIBUTE_deserialize (const char* data, + size_t data_size); + +struct GNUNET_IDENTITY_ATTRIBUTE_ClaimList* +GNUNET_IDENTITY_ATTRIBUTE_list_dup (const struct GNUNET_IDENTITY_ATTRIBUTE_ClaimList *attrs); + +/** + * Convert a type name to the corresponding number + * + * @param typename name to convert + * @return corresponding number, UINT32_MAX on error + */ +uint32_t +GNUNET_IDENTITY_ATTRIBUTE_typename_to_number (const char *typename); + +/** + * Convert human-readable version of a 'claim' of an attribute to the binary + * representation + * + * @param type type of the claim + * @param s human-readable string + * @param data set to value in binary encoding (will be allocated) + * @param data_size set to number of bytes in @a data + * @return #GNUNET_OK on success + */ +int +GNUNET_IDENTITY_ATTRIBUTE_string_to_value (uint32_t type, + const char *s, + void **data, + size_t *data_size); + +/** + * Convert the 'claim' of an attribute to a string + * + * @param type the type of attribute + * @param data claim in binary encoding + * @param data_size number of bytes in @a data + * @return NULL on error, otherwise human-readable representation of the claim + */ +char * +GNUNET_IDENTITY_ATTRIBUTE_value_to_string (uint32_t type, + const void* data, + size_t data_size); + +/** + * Convert a type number to the corresponding type string + * + * @param type number of a type + * @return corresponding typestring, NULL on error + */ +const char* +GNUNET_IDENTITY_ATTRIBUTE_number_to_typename (uint32_t type); + + +/** + * Create a JWT from attributes + * + * @param sub_key the public of the subject + * @param attrs the attribute list + * @param priv_key the key used to sign the JWT + * @return a new base64-encoded JWT string. + */ +char* +GNUNET_IDENTITY_ATTRIBUTE_jwt_create_from_list (const struct GNUNET_CRYPTO_EcdsaPublicKey *sub_key, + const struct GNUNET_IDENTITY_ATTRIBUTE_ClaimList *attrs, + const struct GNUNET_CRYPTO_EcdsaPrivateKey *priv_key); + +#if 0 /* keep Emacsens' auto-indent happy */ +{ +#endif +#ifdef __cplusplus +} +#endif + + +/* ifndef GNUNET_IDENTITY_ATTRIBUTE_LIB_H */ +#endif + +/** @} */ /* end of group identity */ + +/* end of gnunet_identity_attribute_lib.h */ diff --git a/src/include/gnunet_identity_attribute_plugin.h b/src/include/gnunet_identity_attribute_plugin.h new file mode 100644 index 0000000000..edeed57fd4 --- /dev/null +++ b/src/include/gnunet_identity_attribute_plugin.h @@ -0,0 +1,149 @@ +/* + This file is part of GNUnet + Copyright (C) 2012, 2013 GNUnet e.V. + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + Boston, MA 02110-1301, USA. +*/ + +/** + * @author Martin Schanzenbach + * + * @file + * Plugin API for the idp database backend + * + * @defgroup identity-provider-plugin IdP service plugin API + * Plugin API for the idp database backend + * @{ + */ +#ifndef GNUNET_IDENTITY_ATTRIBUTE_PLUGIN_H +#define GNUNET_IDENTITY_ATTRIBUTE_PLUGIN_H + +#include "gnunet_util_lib.h" +#include "gnunet_identity_attribute_lib.h" + +#ifdef __cplusplus +extern "C" +{ +#if 0 /* keep Emacsens' auto-indent happy */ +} +#endif +#endif + + +/** + * Function called to convert the binary value @a data of an attribute of + * type @a type to a human-readable string. + * + * @param cls closure + * @param type type of the attribute + * @param data value in binary encoding + * @param data_size number of bytes in @a data + * @return NULL on error, otherwise human-readable representation of the value + */ +typedef char * (*GNUNET_IDENTITY_ATTRIBUTE_ValueToStringFunction) (void *cls, + uint32_t type, + const void *data, + size_t data_size); + + +/** + * Function called to convert human-readable version of the value @a s + * of an attribute of type @a type to the respective binary + * representation. + * + * @param cls closure + * @param type type of the attribute + * @param s human-readable string + * @param data set to value in binary encoding (will be allocated) + * @param data_size set to number of bytes in @a data + * @return #GNUNET_OK on success + */ +typedef int (*GNUNET_IDENTITY_ATTRIBUTE_StringToValueFunction) (void *cls, + uint32_t type, + const char *s, + void **data, + size_t *data_size); + + +/** + * Function called to convert a type name to the + * corresponding number. + * + * @param cls closure + * @param typename name to convert + * @return corresponding number, UINT32_MAX on error + */ +typedef uint32_t (*GNUNET_IDENTITY_ATTRIBUTE_TypenameToNumberFunction) (void *cls, + const char *typename); + + +/** + * Function called to convert a type number (i.e. 1) to the + * corresponding type string + * + * @param cls closure + * @param type number of a type to convert + * @return corresponding typestring, NULL on error + */ +typedef const char * (*GNUNET_IDENTITY_ATTRIBUTE_NumberToTypenameFunction) (void *cls, + uint32_t type); + + +/** + * Each plugin is required to return a pointer to a struct of this + * type as the return value from its entry point. + */ +struct GNUNET_IDENTITY_ATTRIBUTE_PluginFunctions +{ + + /** + * Closure for all of the callbacks. + */ + void *cls; + + /** + * Conversion to string. + */ + GNUNET_IDENTITY_ATTRIBUTE_ValueToStringFunction value_to_string; + + /** + * Conversion to binary. + */ + GNUNET_IDENTITY_ATTRIBUTE_StringToValueFunction string_to_value; + + /** + * Typename to number. + */ + GNUNET_IDENTITY_ATTRIBUTE_TypenameToNumberFunction typename_to_number; + + /** + * Number to typename. + */ + GNUNET_IDENTITY_ATTRIBUTE_NumberToTypenameFunction number_to_typename; + +}; + + +#if 0 /* keep Emacsens' auto-indent happy */ +{ +#endif +#ifdef __cplusplus +} +#endif + +#endif + +/** @} */ /* end of group */ diff --git a/src/include/gnunet_identity_provider_plugin.h b/src/include/gnunet_identity_provider_plugin.h index c0a258ab63..4b5098d585 100644 --- a/src/include/gnunet_identity_provider_plugin.h +++ b/src/include/gnunet_identity_provider_plugin.h @@ -51,7 +51,7 @@ extern "C" */ typedef void (*GNUNET_IDENTITY_PROVIDER_TicketIterator) (void *cls, const struct GNUNET_IDENTITY_PROVIDER_Ticket *ticket, - const struct GNUNET_IDENTITY_PROVIDER_AttributeList *attrs); + const struct GNUNET_IDENTITY_ATTRIBUTE_ClaimList *attrs); /** @@ -74,7 +74,7 @@ struct GNUNET_IDENTITY_PROVIDER_PluginFunctions */ int (*store_ticket) (void *cls, const struct GNUNET_IDENTITY_PROVIDER_Ticket *ticket, - const struct GNUNET_IDENTITY_PROVIDER_AttributeList *attrs); + const struct GNUNET_IDENTITY_ATTRIBUTE_ClaimList *attrs); /** * Delete a ticket from the database. @@ -111,7 +111,6 @@ struct GNUNET_IDENTITY_PROVIDER_PluginFunctions void *iter_cls); }; - #if 0 /* keep Emacsens' auto-indent happy */ { #endif diff --git a/src/include/gnunet_identity_provider_service.h b/src/include/gnunet_identity_provider_service.h index d17a1cc9ca..6bc05d0f4d 100644 --- a/src/include/gnunet_identity_provider_service.h +++ b/src/include/gnunet_identity_provider_service.h @@ -39,7 +39,7 @@ extern "C" #endif #include "gnunet_util_lib.h" - +#include "gnunet_identity_attribute_lib.h" /** * Version number of GNUnet Identity Provider API. @@ -82,92 +82,6 @@ struct GNUNET_IDENTITY_PROVIDER_Ticket */ struct GNUNET_IDENTITY_PROVIDER_Operation; -/** - * Flags that can be set for an attribute. - */ -enum GNUNET_IDENTITY_PROVIDER_AttributeType -{ - - /** - * No value attribute. - */ - GNUNET_IDENTITY_PROVIDER_AT_NULL = 0, - - /** - * String attribute. - */ - GNUNET_IDENTITY_PROVIDER_AT_STRING = 1, - -}; - - - -/** - * An attribute. - */ -struct GNUNET_IDENTITY_PROVIDER_Attribute -{ - - /** - * Type of Attribute. - */ - uint32_t attribute_type; - - /** - * Attribute version - */ - uint32_t attribute_version; - - /** - * Number of bytes in @e data. - */ - size_t data_size; - - /** - * The name of the attribute. Note "name" must never be individually - * free'd - */ - const char* name; - - /** - * Binary value stored as attribute value. Note: "data" must never - * be individually 'malloc'ed, but instead always points into some - * existing data area. - */ - const void *data; - -}; - -struct GNUNET_IDENTITY_PROVIDER_AttributeList -{ - /** - * List head - */ - struct GNUNET_IDENTITY_PROVIDER_AttributeListEntry *list_head; - - /** - * List tail - */ - struct GNUNET_IDENTITY_PROVIDER_AttributeListEntry *list_tail; -}; - -struct GNUNET_IDENTITY_PROVIDER_AttributeListEntry -{ - /** - * DLL - */ - struct GNUNET_IDENTITY_PROVIDER_AttributeListEntry *prev; - - /** - * DLL - */ - struct GNUNET_IDENTITY_PROVIDER_AttributeListEntry *next; - - /** - * The attribute - */ - struct GNUNET_IDENTITY_PROVIDER_Attribute *attribute; -}; /** * Connect to the identity provider service. @@ -208,27 +122,12 @@ typedef void struct GNUNET_IDENTITY_PROVIDER_Operation * GNUNET_IDENTITY_PROVIDER_attribute_store (struct GNUNET_IDENTITY_PROVIDER_Handle *h, const struct GNUNET_CRYPTO_EcdsaPrivateKey *pkey, - const struct GNUNET_IDENTITY_PROVIDER_Attribute *attr, + const struct GNUNET_IDENTITY_ATTRIBUTE_Claim *attr, GNUNET_IDENTITY_PROVIDER_ContinuationWithStatus cont, void *cont_cls); /** - * Create a new attribute. - * - * @param name the attribute name - * @param type the attribute type - * @param data the attribute value - * @param data_size the attribute value size - * @return the new attribute - */ -struct GNUNET_IDENTITY_PROVIDER_Attribute * -GNUNET_IDENTITY_PROVIDER_attribute_new (const char* attr_name, - uint32_t attr_type, - const void* data, - size_t data_size); - -/** * Process an attribute that was stored in the idp. * * @param cls closure @@ -237,7 +136,7 @@ GNUNET_IDENTITY_PROVIDER_attribute_new (const char* attr_name, typedef void (*GNUNET_IDENTITY_PROVIDER_AttributeResult) (void *cls, const struct GNUNET_CRYPTO_EcdsaPublicKey *identity, - const struct GNUNET_IDENTITY_PROVIDER_Attribute *attr); + const struct GNUNET_IDENTITY_ATTRIBUTE_Claim *attr); @@ -327,7 +226,7 @@ struct GNUNET_IDENTITY_PROVIDER_Operation * GNUNET_IDENTITY_PROVIDER_ticket_issue (struct GNUNET_IDENTITY_PROVIDER_Handle *id, const struct GNUNET_CRYPTO_EcdsaPrivateKey *iss, const struct GNUNET_CRYPTO_EcdsaPublicKey *rp, - const struct GNUNET_IDENTITY_PROVIDER_AttributeList *attrs, + const struct GNUNET_IDENTITY_ATTRIBUTE_ClaimList *attrs, GNUNET_IDENTITY_PROVIDER_TicketCallback cb, void *cb_cls); diff --git a/src/include/gnunet_json_lib.h b/src/include/gnunet_json_lib.h index f2682bea7d..c12badcd9c 100644 --- a/src/include/gnunet_json_lib.h +++ b/src/include/gnunet_json_lib.h @@ -343,6 +343,16 @@ GNUNET_JSON_from_time_abs (struct GNUNET_TIME_Absolute stamp); /** + * Convert absolute timestamp to a json string. + * + * @param stamp the time stamp + * @return a json string with the timestamp in @a stamp + */ +json_t * +GNUNET_JSON_from_time_abs_nbo (struct GNUNET_TIME_AbsoluteNBO stamp); + + +/** * Convert relative timestamp to a json string. * * @param stamp the time stamp diff --git a/src/include/gnunet_scheduler_lib.h b/src/include/gnunet_scheduler_lib.h index 875f5043a4..a855ab8aba 100644 --- a/src/include/gnunet_scheduler_lib.h +++ b/src/include/gnunet_scheduler_lib.h @@ -400,6 +400,22 @@ void GNUNET_SCHEDULER_run (GNUNET_SCHEDULER_TaskCallback task, void *task_cls); +/** + * Initialize and run scheduler. This function will return when all + * tasks have completed. When @ install_signals is GNUNET_YES, then + * this function behaves in the same was as GNUNET_SCHEDULER_run does. + * If @ install_signals is GNUNET_NO then no signal handlers are + * installed. + * + * @param install_signals whether to install signals (GNUNET_YES/NO) + * @param task task to run first (and immediately) + * @param task_cls closure of @a task + */ +void +GNUNET_SCHEDULER_run_with_optional_signals (int install_signals, + GNUNET_SCHEDULER_TaskCallback task, + void *task_cls); + /** * Request the shutdown of a scheduler. Marks all tasks diff --git a/src/integration-tests/Makefile.am b/src/integration-tests/Makefile.am index 6fff0b407b..368980064e 100644 --- a/src/integration-tests/Makefile.am +++ b/src/integration-tests/Makefile.am @@ -42,7 +42,8 @@ endif do_subst = $(SED) -e 's,[@]PYTHON[@],$(PYTHON),g' -%.py: %.py.in Makefile +SUFFIXES = .py.in .py +.py.in.py: $(do_subst) < $(srcdir)/$< > $@ chmod +x $@ diff --git a/src/json/json_generator.c b/src/json/json_generator.c index e660e10c53..98f7163bcc 100644 --- a/src/json/json_generator.c +++ b/src/json/json_generator.c @@ -73,6 +73,19 @@ GNUNET_JSON_from_time_abs (struct GNUNET_TIME_Absolute stamp) /** + * Convert absolute timestamp to a json string. + * + * @param stamp the time stamp + * @return a json string with the timestamp in @a stamp + */ +json_t * +GNUNET_JSON_from_time_abs_nbo (struct GNUNET_TIME_AbsoluteNBO stamp) +{ + return GNUNET_JSON_from_time_abs (GNUNET_TIME_absolute_ntoh (stamp)); +} + + +/** * Convert relative timestamp to a json string. * * @param stamp the time stamp diff --git a/src/revocation/gnunet-service-revocation.c b/src/revocation/gnunet-service-revocation.c index 9d077f874d..8281e9a166 100644 --- a/src/revocation/gnunet-service-revocation.c +++ b/src/revocation/gnunet-service-revocation.c @@ -509,6 +509,7 @@ transmit_task_cb (void *cls) "Starting set exchange with peer `%s'\n", GNUNET_i2s (&peer_entry->id)); peer_entry->transmit_task = NULL; + GNUNET_assert (NULL == peer_entry->so); peer_entry->so = GNUNET_SET_prepare (&peer_entry->id, &revocation_set_union_app_id, NULL, @@ -758,6 +759,7 @@ handle_revocation_union_request (void *cls, { peer_entry = new_peer_entry (other_peer); } + GNUNET_assert (NULL == peer_entry->so); peer_entry->so = GNUNET_SET_accept (request, GNUNET_SET_RESULT_ADDED, (struct GNUNET_SET_Option[]) {{ 0 }}, diff --git a/src/secretsharing/Makefile.am b/src/secretsharing/Makefile.am index 5ab8739af0..c808e82006 100644 --- a/src/secretsharing/Makefile.am +++ b/src/secretsharing/Makefile.am @@ -47,7 +47,7 @@ libgnunetsecretsharing_la_SOURCES = \ secretsharing_api.c \ secretsharing_common.c \ secretsharing.h -libgnunetsecretsharing_la_LIBADD = \ +libgnunetsecretsharing_la_LIBADD = \ $(top_builddir)/src/util/libgnunetutil.la \ $(LIBGCRYPT_LIBS) \ $(LTLIBINTL) diff --git a/src/social/gnunet-social.c b/src/social/gnunet-social.c index 0e52dccfa0..de680b11c5 100644 --- a/src/social/gnunet-social.c +++ b/src/social/gnunet-social.c @@ -281,7 +281,7 @@ exit_fail () * This also indicates the end of the connection to the service. */ static void -host_left () +host_left (void *cls) { GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "The host has left the place.\n"); diff --git a/src/social/social_api.c b/src/social/social_api.c index af1d6e57ee..d57d16cfbe 100644 --- a/src/social/social_api.c +++ b/src/social/social_api.c @@ -2693,6 +2693,8 @@ GNUNET_SOCIAL_app_disconnect (struct GNUNET_SOCIAL_App *app, GNUNET_ContinuationCallback disconnect_cb, void *disconnect_cls) { + if (NULL == app) return; + app->disconnect_cb = disconnect_cb; app->disconnect_cls = disconnect_cls; diff --git a/src/sq/sq_result_helper.c b/src/sq/sq_result_helper.c index 9579863b23..f2986a0532 100644 --- a/src/sq/sq_result_helper.c +++ b/src/sq/sq_result_helper.c @@ -620,7 +620,7 @@ extract_uint16 (void *cls, void *dst) { uint64_t v; - uint32_t *u = dst; + uint16_t *u = dst; GNUNET_assert (sizeof (uint16_t) == *dst_size); if (SQLITE_INTEGER != diff --git a/src/statistics/Makefile.am b/src/statistics/Makefile.am index b2e256960e..16a1ea2d0f 100644 --- a/src/statistics/Makefile.am +++ b/src/statistics/Makefile.am @@ -90,7 +90,8 @@ endif do_subst = $(SED) -e 's,[@]PYTHON[@],$(PYTHON),g' -%.py: %.py.in Makefile +SUFFIXES = .py.in .py +.py.in.py: $(do_subst) < $(srcdir)/$< > $@ chmod +x $@ diff --git a/src/testbed/testbed_api_topology.c b/src/testbed/testbed_api_topology.c index 7bc36d1b47..7d0ccd269a 100644 --- a/src/testbed/testbed_api_topology.c +++ b/src/testbed/testbed_api_topology.c @@ -1051,7 +1051,7 @@ gen_topo_from_file (struct TopologyContext *tc, state = PEER_INDEX; while (offset < fs) { - if (0 != isspace (data[offset])) + if (0 != isspace ((unsigned char) data[offset])) { offset++; continue; diff --git a/src/topology/friends.c b/src/topology/friends.c index a960fad174..65f2700bb3 100644 --- a/src/topology/friends.c +++ b/src/topology/friends.c @@ -95,7 +95,7 @@ GNUNET_FRIENDS_parse (const struct GNUNET_CONFIGURATION_Handle *cfg, pos = 0; while (pos < fsize) { - while ((pos < fsize) && (! isspace ((int) data[pos]))) + while ((pos < fsize) && (! isspace ((unsigned char) data[pos]))) pos++; if (GNUNET_OK != GNUNET_CRYPTO_eddsa_public_key_from_string (&data[start], diff --git a/src/transport/gnunet-service-transport.c b/src/transport/gnunet-service-transport.c index ec4d821649..6b354df986 100644 --- a/src/transport/gnunet-service-transport.c +++ b/src/transport/gnunet-service-transport.c @@ -541,6 +541,13 @@ client_disconnect_cb (void *cls, GNUNET_CONTAINER_multipeermap_iterate (active_stccs, &mark_match_down, tc); + for (struct AddressToStringContext *cur = a2s_head; + NULL != cur; + cur = cur->next) + { + if (cur->tc == tc) + cur->tc = NULL; + } GNUNET_CONTAINER_DLL_remove (clients_head, clients_tail, tc); @@ -864,6 +871,8 @@ transmit_address_to_client (void *cls, GNUNET_assert ( (GNUNET_OK == res) || (GNUNET_SYSERR == res) ); + if (NULL == actx->tc) + return; if (NULL == buf) { env = GNUNET_MQ_msg (atsm, @@ -878,6 +887,7 @@ transmit_address_to_client (void *cls, GNUNET_CONTAINER_DLL_remove (a2s_head, a2s_tail, actx); + GNUNET_free (actx); return; } if (GNUNET_SYSERR == res) diff --git a/src/transport/gnunet-service-transport_ats.c b/src/transport/gnunet-service-transport_ats.c index a20c998b34..c780f9a78f 100644 --- a/src/transport/gnunet-service-transport_ats.c +++ b/src/transport/gnunet-service-transport_ats.c @@ -344,8 +344,7 @@ GST_ats_block_address (const struct GNUNET_HELLO_Address *address, } if (NULL == ai->ar) { - /* already blocked, how did it get used!? */ - GNUNET_break (0); + /* already blocked but this might be a blacklist check callback */ return; } ai->back_off = GNUNET_TIME_STD_BACKOFF (ai->back_off); diff --git a/src/transport/gnunet-service-transport_validation.c b/src/transport/gnunet-service-transport_validation.c index 4a6d427bec..27c3c70419 100644 --- a/src/transport/gnunet-service-transport_validation.c +++ b/src/transport/gnunet-service-transport_validation.c @@ -784,15 +784,24 @@ revalidate_address (void *cls) GNUNET_STATISTICS_update (GST_stats, gettext_noop ("# address revalidations started"), 1, GNUNET_NO); + if (NULL != ve->bc) + { + GST_blacklist_test_cancel (ve->bc); + ve->bc = NULL; + } bc = GST_blacklist_test_allowed (&ve->address->peer, - ve->address->transport_name, + ve->address->transport_name, &transmit_ping_if_allowed, - ve, - NULL, - NULL); + ve, + NULL, + NULL); if (NULL != bc) - ve->bc = bc; /* only set 'bc' if 'transmit_ping_if_allowed' was not already - * called... */ + { + /* If transmit_ping_if_allowed was already called it may have freed ve, + * so only set ve->bc if it has not been called. + */ + ve->bc = bc; + } } diff --git a/src/transport/test_transport_testing_restart.c b/src/transport/test_transport_testing_restart.c index 595177e036..06275055da 100644 --- a/src/transport/test_transport_testing_restart.c +++ b/src/transport/test_transport_testing_restart.c @@ -71,7 +71,8 @@ restart_cb (void *cls) p->no, GNUNET_i2s (&p->id)); ret = 0; - end (); + GNUNET_SCHEDULER_add_now (&end, + NULL); } diff --git a/src/transport/test_transport_testing_startstop.c b/src/transport/test_transport_testing_startstop.c index 6ac0250cc0..931e922c47 100644 --- a/src/transport/test_transport_testing_startstop.c +++ b/src/transport/test_transport_testing_startstop.c @@ -71,7 +71,8 @@ start_cb (void *cls) p->no, GNUNET_i2s (&p->id)); ret = 0; - end (); + GNUNET_SCHEDULER_add_now (&end, + NULL); } diff --git a/src/transport/transport-testing.c b/src/transport/transport-testing.c index 2aa6cdbb0e..68cda3bd7e 100644 --- a/src/transport/transport-testing.c +++ b/src/transport/transport-testing.c @@ -384,7 +384,7 @@ GNUNET_TRANSPORT_TESTING_start_peer (struct GNUNET_TRANSPORT_TESTING_Handle *tth { char *emsg = NULL; struct GNUNET_TRANSPORT_TESTING_PeerContext *p; - struct GNUNET_PeerIdentity *dummy; + struct GNUNET_PeerIdentity dummy; unsigned int i; if (GNUNET_NO == GNUNET_DISK_file_test (cfgname)) @@ -678,6 +678,11 @@ GNUNET_TRANSPORT_TESTING_stop_peer (struct GNUNET_TRANSPORT_TESTING_PeerContext GNUNET_CONFIGURATION_destroy (p->cfg); p->cfg = NULL; } + if (NULL != p->handlers) + { + GNUNET_free (p->handlers); + p->handlers = NULL; + } GNUNET_CONTAINER_DLL_remove (tth->p_head, tth->p_tail, p); diff --git a/src/util/crypto_ecc.c b/src/util/crypto_ecc.c index eaa49a9919..7845932ee7 100644 --- a/src/util/crypto_ecc.c +++ b/src/util/crypto_ecc.c @@ -354,6 +354,37 @@ GNUNET_CRYPTO_eddsa_public_key_to_string (const struct GNUNET_CRYPTO_EddsaPublic /** + * Convert a private key to a string. + * + * @param priv key to convert + * @return string representing @a pub + */ +char * +GNUNET_CRYPTO_eddsa_private_key_to_string (const struct GNUNET_CRYPTO_EddsaPrivateKey *priv) +{ + char *privkeybuf; + size_t keylen = (sizeof (struct GNUNET_CRYPTO_EddsaPrivateKey)) * 8; + char *end; + + if (keylen % 5 > 0) + keylen += 5 - keylen % 5; + keylen /= 5; + privkeybuf = GNUNET_malloc (keylen + 1); + end = GNUNET_STRINGS_data_to_string ((unsigned char *) priv, + sizeof (struct GNUNET_CRYPTO_EddsaPrivateKey), + privkeybuf, + keylen); + if (NULL == end) + { + GNUNET_free (privkeybuf); + return NULL; + } + *end = '\0'; + return privkeybuf; +} + + +/** * Convert a string representing a public key to a public key. * * @param enc encoded public key @@ -374,9 +405,10 @@ GNUNET_CRYPTO_ecdsa_public_key_from_string (const char *enc, if (enclen != keylen) return GNUNET_SYSERR; - if (GNUNET_OK != GNUNET_STRINGS_string_to_data (enc, enclen, - pub, - sizeof (struct GNUNET_CRYPTO_EcdsaPublicKey))) + if (GNUNET_OK != + GNUNET_STRINGS_string_to_data (enc, enclen, + pub, + sizeof (struct GNUNET_CRYPTO_EcdsaPublicKey))) return GNUNET_SYSERR; return GNUNET_OK; } @@ -403,9 +435,10 @@ GNUNET_CRYPTO_eddsa_public_key_from_string (const char *enc, if (enclen != keylen) return GNUNET_SYSERR; - if (GNUNET_OK != GNUNET_STRINGS_string_to_data (enc, enclen, - pub, - sizeof (struct GNUNET_CRYPTO_EddsaPublicKey))) + if (GNUNET_OK != + GNUNET_STRINGS_string_to_data (enc, enclen, + pub, + sizeof (struct GNUNET_CRYPTO_EddsaPublicKey))) return GNUNET_SYSERR; return GNUNET_OK; } diff --git a/src/util/crypto_paillier.c b/src/util/crypto_paillier.c index 3ed025a2ac..530a2957fc 100644 --- a/src/util/crypto_paillier.c +++ b/src/util/crypto_paillier.c @@ -370,9 +370,11 @@ GNUNET_CRYPTO_paillier_decrypt (const struct GNUNET_CRYPTO_PaillierPrivateKey *p /* mod = cmum1 / n (mod n) */ GNUNET_assert (0 != (mod = gcry_mpi_new (0))); gcry_mpi_div (mod, NULL, cmum1, n, 0); + gcry_mpi_release (cmum1); /* m = mod * mu mod n */ gcry_mpi_mulm (m, mod, mu, n); + gcry_mpi_release (mod); gcry_mpi_release (mu); gcry_mpi_release (n); } diff --git a/src/util/crypto_rsa.c b/src/util/crypto_rsa.c index 7a108c21b1..a985d8e596 100644 --- a/src/util/crypto_rsa.c +++ b/src/util/crypto_rsa.c @@ -1046,7 +1046,7 @@ GNUNET_CRYPTO_rsa_public_key_dup (const struct GNUNET_CRYPTO_RsaPublicKey *key) * @return unblinded signature on success, NULL if RSA key is bad or malicious. */ struct GNUNET_CRYPTO_RsaSignature * -GNUNET_CRYPTO_rsa_unblind (struct GNUNET_CRYPTO_RsaSignature *sig, +GNUNET_CRYPTO_rsa_unblind (const struct GNUNET_CRYPTO_RsaSignature *sig, const struct GNUNET_CRYPTO_RsaBlindingKeySecret *bks, struct GNUNET_CRYPTO_RsaPublicKey *pkey) { diff --git a/src/util/gnunet-ecc.c b/src/util/gnunet-ecc.c index 42ecc2101d..66a4bd3e91 100644 --- a/src/util/gnunet-ecc.c +++ b/src/util/gnunet-ecc.c @@ -49,6 +49,11 @@ static unsigned int list_keys_count; static int print_public_key; /** + * Flag for printing private key. + */ +static int print_private_key; + +/** * Flag for printing public key in hex. */ static int print_public_key_hex; @@ -377,7 +382,7 @@ run (void *cls, char *const *args, const char *cfgfile, create_keys (args[0], args[1]); return; } - if (print_public_key || print_public_key_hex) + if (print_public_key || print_public_key_hex || print_private_key) { char *str; struct GNUNET_DISK_FileHandle *keyfile; @@ -388,19 +393,26 @@ run (void *cls, char *const *args, const char *cfgfile, GNUNET_DISK_PERM_NONE); if (NULL == keyfile) return; - while (sizeof (pk) == GNUNET_DISK_file_read (keyfile, &pk, sizeof (pk))) + while (sizeof (pk) == + GNUNET_DISK_file_read (keyfile, &pk, sizeof (pk))) { GNUNET_CRYPTO_eddsa_key_get_public (&pk, &pub); if (print_public_key_hex) { print_hex ("HEX:", &pub, sizeof (pub)); } - else + else if (print_public_key) { str = GNUNET_CRYPTO_eddsa_public_key_to_string (&pub); FPRINTF (stdout, "%s\n", str); GNUNET_free (str); } + else if (print_private_key) + { + str = GNUNET_CRYPTO_eddsa_private_key_to_string (&pk); + FPRINTF (stdout, "%s\n", str); + GNUNET_free (str); + } } GNUNET_DISK_file_close (keyfile); } @@ -438,6 +450,10 @@ main (int argc, "print-public-key", gettext_noop ("print the public key in ASCII format"), &print_public_key), + GNUNET_GETOPT_option_flag ('P', + "print-private-key", + gettext_noop ("print the private key in ASCII format"), + &print_private_key), GNUNET_GETOPT_option_flag ('x', "print-hex", gettext_noop ("print the public key in HEX format"), diff --git a/src/util/resolver_api.c b/src/util/resolver_api.c index 33a340729d..11b8134d6d 100644 --- a/src/util/resolver_api.c +++ b/src/util/resolver_api.c @@ -469,6 +469,7 @@ handle_response (void *cls, uint16_t size; char *nret; + GNUNET_assert (NULL != rh); size = ntohs (msg->size); if (size == sizeof (struct GNUNET_MessageHeader)) { diff --git a/src/util/scheduler.c b/src/util/scheduler.c index e9c25d68a8..540a605573 100644 --- a/src/util/scheduler.c +++ b/src/util/scheduler.c @@ -787,6 +787,14 @@ void GNUNET_SCHEDULER_run (GNUNET_SCHEDULER_TaskCallback task, void *task_cls) { + GNUNET_SCHEDULER_run_with_optional_signals(GNUNET_YES, task, task_cls); +} + +void +GNUNET_SCHEDULER_run_with_optional_signals (int install_signals, + GNUNET_SCHEDULER_TaskCallback task, + void *task_cls) +{ struct GNUNET_NETWORK_FDSet *rs; struct GNUNET_NETWORK_FDSet *ws; struct GNUNET_TIME_Relative timeout; @@ -820,24 +828,29 @@ GNUNET_SCHEDULER_run (GNUNET_SCHEDULER_TaskCallback task, GNUNET_DISK_PIPE_END_READ); GNUNET_assert (NULL != pr); my_pid = getpid (); - LOG (GNUNET_ERROR_TYPE_DEBUG, - "Registering signal handlers\n"); - shc_int = GNUNET_SIGNAL_handler_install (SIGINT, + + if (GNUNET_YES == install_signals) + { + LOG (GNUNET_ERROR_TYPE_DEBUG, + "Registering signal handlers\n"); + shc_int = GNUNET_SIGNAL_handler_install (SIGINT, + &sighandler_shutdown); + shc_term = GNUNET_SIGNAL_handler_install (SIGTERM, &sighandler_shutdown); - shc_term = GNUNET_SIGNAL_handler_install (SIGTERM, - &sighandler_shutdown); #if (SIGTERM != GNUNET_TERM_SIG) - shc_gterm = GNUNET_SIGNAL_handler_install (GNUNET_TERM_SIG, + shc_gterm = GNUNET_SIGNAL_handler_install (GNUNET_TERM_SIG, &sighandler_shutdown); #endif #ifndef MINGW - shc_pipe = GNUNET_SIGNAL_handler_install (SIGPIPE, + shc_pipe = GNUNET_SIGNAL_handler_install (SIGPIPE, &sighandler_pipe); - shc_quit = GNUNET_SIGNAL_handler_install (SIGQUIT, + shc_quit = GNUNET_SIGNAL_handler_install (SIGQUIT, &sighandler_shutdown); - shc_hup = GNUNET_SIGNAL_handler_install (SIGHUP, + shc_hup = GNUNET_SIGNAL_handler_install (SIGHUP, &sighandler_shutdown); #endif + } + current_priority = GNUNET_SCHEDULER_PRIORITY_DEFAULT; current_lifeness = GNUNET_YES; GNUNET_SCHEDULER_add_with_reason_and_priority (task, @@ -953,16 +966,21 @@ GNUNET_SCHEDULER_run (GNUNET_SCHEDULER_TaskCallback task, busy_wait_warning = 0; } } - GNUNET_SIGNAL_handler_uninstall (shc_int); - GNUNET_SIGNAL_handler_uninstall (shc_term); + + if (GNUNET_YES == install_signals) + { + GNUNET_SIGNAL_handler_uninstall (shc_int); + GNUNET_SIGNAL_handler_uninstall (shc_term); #if (SIGTERM != GNUNET_TERM_SIG) - GNUNET_SIGNAL_handler_uninstall (shc_gterm); + GNUNET_SIGNAL_handler_uninstall (shc_gterm); #endif #ifndef MINGW - GNUNET_SIGNAL_handler_uninstall (shc_pipe); - GNUNET_SIGNAL_handler_uninstall (shc_quit); - GNUNET_SIGNAL_handler_uninstall (shc_hup); + GNUNET_SIGNAL_handler_uninstall (shc_pipe); + GNUNET_SIGNAL_handler_uninstall (shc_quit); + GNUNET_SIGNAL_handler_uninstall (shc_hup); #endif + } + GNUNET_DISK_pipe_close (shutdown_pipe_handle); shutdown_pipe_handle = NULL; GNUNET_NETWORK_fdset_destroy (rs); diff --git a/src/util/test_crypto_paillier.c b/src/util/test_crypto_paillier.c index 9950978c14..1e7e0b301a 100644 --- a/src/util/test_crypto_paillier.c +++ b/src/util/test_crypto_paillier.c @@ -37,6 +37,7 @@ test_crypto () struct GNUNET_CRYPTO_PaillierCiphertext ciphertext; struct GNUNET_CRYPTO_PaillierPublicKey public_key; struct GNUNET_CRYPTO_PaillierPrivateKey private_key; + int ret = 0; GNUNET_CRYPTO_paillier_create (&public_key, &private_key); @@ -54,7 +55,6 @@ test_crypto () &public_key, &ciphertext, plaintext_result); - if (0 != gcry_mpi_cmp (plaintext, plaintext_result)) { @@ -65,9 +65,11 @@ test_crypto () plaintext); gcry_log_debugmpi ("\n", plaintext_result); - return 1; + ret = 1; } - return 0; + gcry_mpi_release (plaintext); + gcry_mpi_release (plaintext_result); + return ret; } @@ -84,6 +86,7 @@ test_hom_simple (unsigned int a, struct GNUNET_CRYPTO_PaillierCiphertext c_result; struct GNUNET_CRYPTO_PaillierPublicKey public_key; struct GNUNET_CRYPTO_PaillierPrivateKey private_key; + int ret = 0; GNUNET_CRYPTO_paillier_create (&public_key, &private_key); @@ -119,9 +122,13 @@ test_hom_simple (unsigned int a, "GNUNET_CRYPTO_paillier failed simple math!\n"); gcry_log_debugmpi ("got ", hom_result); gcry_log_debugmpi ("wanted ", result); - return 1; + ret = 1; } - return 0; + gcry_mpi_release (m1); + gcry_mpi_release (m2); + gcry_mpi_release (result); + gcry_mpi_release (hom_result); + return ret; } @@ -168,7 +175,8 @@ test_hom () fprintf (stderr, "GNUNET_CRYPTO_paillier_encrypt 1 failed, should return 1 allowed operation, got %d!\n", ret); - return 1; + ret = 1; + goto out; } if (2 != (ret = GNUNET_CRYPTO_paillier_encrypt (&public_key, m2, @@ -178,7 +186,8 @@ test_hom () fprintf (stderr, "GNUNET_CRYPTO_paillier_encrypt 2 failed, should return 2 allowed operation, got %d!\n", ret); - return 1; + ret = 1; + goto out; } if (0 != (ret = GNUNET_CRYPTO_paillier_hom_add (&public_key, @@ -189,7 +198,8 @@ test_hom () fprintf (stderr, "GNUNET_CRYPTO_paillier_hom_add failed, expected 0 remaining operations, got %d!\n", ret); - return 1; + ret = 1; + goto out; } GNUNET_CRYPTO_paillier_decrypt (&private_key, @@ -203,9 +213,14 @@ test_hom () "GNUNET_CRYPTO_paillier miscalculated with large numbers!\n"); gcry_log_debugmpi ("got", hom_result); gcry_log_debugmpi ("wanted", result); - return 1; + ret = 1; } - return 0; +out: + gcry_mpi_release (m1); + gcry_mpi_release (m2); + gcry_mpi_release (result); + gcry_mpi_release (hom_result); + return ret; } diff --git a/src/util/test_mq.c b/src/util/test_mq.c index 442c110dbd..9e8fc844ee 100644 --- a/src/util/test_mq.c +++ b/src/util/test_mq.c @@ -51,6 +51,7 @@ test1 () GNUNET_assert (NULL != mm); GNUNET_assert (42 == ntohs (mm->header.type)); GNUNET_assert (sizeof (struct MyMessage) == ntohs (mm->header.size)); + GNUNET_MQ_discard (mqm); } |
