From 7f07f09d52aed7c449a330d8a82c1280776e49e0 Mon Sep 17 00:00:00 2001 From: ng0 Date: Sat, 21 Oct 2017 16:37:12 +0000 Subject: move docmentation to separate folder --- doc/documentation/Makefile.am | 214 + doc/documentation/README.txt | 64 + doc/documentation/chapters/developer.texi | 7926 ++++++++++++++++++++ doc/documentation/chapters/installation.texi | 3826 ++++++++++ doc/documentation/chapters/philosophy.texi | 373 + doc/documentation/chapters/user.texi | 2009 +++++ doc/documentation/chapters/vocabulary.texi | 47 + doc/documentation/docstyle.css | 76 + doc/documentation/fdl-1.3.texi | 505 ++ doc/documentation/gnunet-c-tutorial.texi | 1519 ++++ doc/documentation/gnunet.texi | 255 + doc/documentation/gpl-3.0.texi | 717 ++ doc/documentation/images/daemon_lego_block.png | Bin 0 -> 7636 bytes doc/documentation/images/daemon_lego_block.svg | 126 + doc/documentation/images/gnunet-0-10-peerinfo.png | Bin 0 -> 80127 bytes .../images/gnunet-fs-gtk-0-10-star-tab.png | Bin 0 -> 63464 bytes .../images/gnunet-gtk-0-10-download-area.png | Bin 0 -> 7634 bytes .../images/gnunet-gtk-0-10-fs-menu.png | Bin 0 -> 8614 bytes .../images/gnunet-gtk-0-10-fs-publish-editing.png | Bin 0 -> 55507 bytes .../images/gnunet-gtk-0-10-fs-publish-select.png | Bin 0 -> 43448 bytes .../gnunet-gtk-0-10-fs-publish-with-file.png | Bin 0 -> 27371 bytes .../gnunet-gtk-0-10-fs-publish-with-file_0.png | Bin 0 -> 27371 bytes .../images/gnunet-gtk-0-10-fs-publish.png | Bin 0 -> 26496 bytes .../images/gnunet-gtk-0-10-fs-published.png | Bin 0 -> 59635 bytes .../images/gnunet-gtk-0-10-fs-search.png | Bin 0 -> 72151 bytes doc/documentation/images/gnunet-gtk-0-10-fs.png | Bin 0 -> 55706 bytes .../images/gnunet-gtk-0-10-gns-a-done.png | Bin 0 -> 30880 bytes doc/documentation/images/gnunet-gtk-0-10-gns-a.png | Bin 0 -> 29895 bytes doc/documentation/images/gnunet-gtk-0-10-gns.png | Bin 0 -> 63783 bytes .../images/gnunet-gtk-0-10-identity.png | Bin 0 -> 62404 bytes .../images/gnunet-gtk-0-10-search-selected.png | Bin 0 -> 104599 bytes .../images/gnunet-gtk-0-10-traffic.png | Bin 0 -> 68515 bytes doc/documentation/images/gnunet-gtk-0-10.png | Bin 0 -> 72897 bytes .../images/gnunet-namestore-gtk-phone.png | Bin 0 -> 32631 bytes .../images/gnunet-namestore-gtk-vpn.png | Bin 0 -> 35836 bytes doc/documentation/images/gnunet-setup-exit.png | Bin 0 -> 30062 bytes .../images/gnunet-tutorial-service.png | Bin 0 -> 40142 bytes .../images/gnunet-tutorial-system.png | Bin 0 -> 46982 bytes doc/documentation/images/iceweasel-preferences.png | Bin 0 -> 57047 bytes doc/documentation/images/iceweasel-proxy.png | Bin 0 -> 38773 bytes doc/documentation/images/lego_stack.svg | 737 ++ doc/documentation/images/service_lego_block.png | Bin 0 -> 15157 bytes doc/documentation/images/service_lego_block.svg | 345 + doc/documentation/images/service_stack.png | Bin 0 -> 18862 bytes doc/documentation/images/structure.dot | 124 + doc/documentation/testbed_test.c | 99 + doc/documentation/tutorial-examples/001.c | 29 + doc/documentation/tutorial-examples/002.c | 17 + doc/documentation/tutorial-examples/003.c | 11 + doc/documentation/tutorial-examples/004.c | 5 + doc/documentation/tutorial-examples/005.c | 8 + doc/documentation/tutorial-examples/006.c | 31 + doc/documentation/tutorial-examples/007.c | 10 + doc/documentation/tutorial-examples/008.c | 22 + doc/documentation/tutorial-examples/009.c | 9 + doc/documentation/tutorial-examples/010.c | 8 + doc/documentation/tutorial-examples/011.c | 8 + doc/documentation/tutorial-examples/012.c | 4 + doc/documentation/tutorial-examples/013.1.c | 3 + doc/documentation/tutorial-examples/013.c | 12 + doc/documentation/tutorial-examples/014.c | 9 + doc/documentation/tutorial-examples/015.c | 8 + doc/documentation/tutorial-examples/016.c | 4 + doc/documentation/tutorial-examples/017.c | 4 + doc/documentation/tutorial-examples/018.c | 2 + doc/documentation/tutorial-examples/019.c | 18 + doc/documentation/tutorial-examples/020.c | 25 + doc/documentation/tutorial-examples/021.c | 13 + doc/documentation/tutorial-examples/022.c | 8 + doc/documentation/tutorial-examples/023.c | 17 + doc/documentation/tutorial-examples/024.c | 9 + doc/documentation/tutorial-examples/025.c | 15 + doc/documentation/tutorial-examples/026.c | 52 + 73 files changed, 19323 insertions(+) create mode 100644 doc/documentation/Makefile.am create mode 100644 doc/documentation/README.txt create mode 100644 doc/documentation/chapters/developer.texi create mode 100644 doc/documentation/chapters/installation.texi create mode 100644 doc/documentation/chapters/philosophy.texi create mode 100644 doc/documentation/chapters/user.texi create mode 100644 doc/documentation/chapters/vocabulary.texi create mode 100644 doc/documentation/docstyle.css create mode 100644 doc/documentation/fdl-1.3.texi create mode 100644 doc/documentation/gnunet-c-tutorial.texi create mode 100644 doc/documentation/gnunet.texi create mode 100644 doc/documentation/gpl-3.0.texi create mode 100644 doc/documentation/images/daemon_lego_block.png create mode 100644 doc/documentation/images/daemon_lego_block.svg create mode 100644 doc/documentation/images/gnunet-0-10-peerinfo.png create mode 100644 doc/documentation/images/gnunet-fs-gtk-0-10-star-tab.png create mode 100644 doc/documentation/images/gnunet-gtk-0-10-download-area.png create mode 100644 doc/documentation/images/gnunet-gtk-0-10-fs-menu.png create mode 100644 doc/documentation/images/gnunet-gtk-0-10-fs-publish-editing.png create mode 100644 doc/documentation/images/gnunet-gtk-0-10-fs-publish-select.png create mode 100644 doc/documentation/images/gnunet-gtk-0-10-fs-publish-with-file.png create mode 100644 doc/documentation/images/gnunet-gtk-0-10-fs-publish-with-file_0.png create mode 100644 doc/documentation/images/gnunet-gtk-0-10-fs-publish.png create mode 100644 doc/documentation/images/gnunet-gtk-0-10-fs-published.png create mode 100644 doc/documentation/images/gnunet-gtk-0-10-fs-search.png create mode 100644 doc/documentation/images/gnunet-gtk-0-10-fs.png create mode 100644 doc/documentation/images/gnunet-gtk-0-10-gns-a-done.png create mode 100644 doc/documentation/images/gnunet-gtk-0-10-gns-a.png create mode 100644 doc/documentation/images/gnunet-gtk-0-10-gns.png create mode 100644 doc/documentation/images/gnunet-gtk-0-10-identity.png create mode 100644 doc/documentation/images/gnunet-gtk-0-10-search-selected.png create mode 100644 doc/documentation/images/gnunet-gtk-0-10-traffic.png create mode 100644 doc/documentation/images/gnunet-gtk-0-10.png create mode 100644 doc/documentation/images/gnunet-namestore-gtk-phone.png create mode 100644 doc/documentation/images/gnunet-namestore-gtk-vpn.png create mode 100644 doc/documentation/images/gnunet-setup-exit.png create mode 100644 doc/documentation/images/gnunet-tutorial-service.png create mode 100644 doc/documentation/images/gnunet-tutorial-system.png create mode 100644 doc/documentation/images/iceweasel-preferences.png create mode 100644 doc/documentation/images/iceweasel-proxy.png create mode 100644 doc/documentation/images/lego_stack.svg create mode 100644 doc/documentation/images/service_lego_block.png create mode 100644 doc/documentation/images/service_lego_block.svg create mode 100644 doc/documentation/images/service_stack.png create mode 100644 doc/documentation/images/structure.dot create mode 100644 doc/documentation/testbed_test.c create mode 100644 doc/documentation/tutorial-examples/001.c create mode 100644 doc/documentation/tutorial-examples/002.c create mode 100644 doc/documentation/tutorial-examples/003.c create mode 100644 doc/documentation/tutorial-examples/004.c create mode 100644 doc/documentation/tutorial-examples/005.c create mode 100644 doc/documentation/tutorial-examples/006.c create mode 100644 doc/documentation/tutorial-examples/007.c create mode 100644 doc/documentation/tutorial-examples/008.c create mode 100644 doc/documentation/tutorial-examples/009.c create mode 100644 doc/documentation/tutorial-examples/010.c create mode 100644 doc/documentation/tutorial-examples/011.c create mode 100644 doc/documentation/tutorial-examples/012.c create mode 100644 doc/documentation/tutorial-examples/013.1.c create mode 100644 doc/documentation/tutorial-examples/013.c create mode 100644 doc/documentation/tutorial-examples/014.c create mode 100644 doc/documentation/tutorial-examples/015.c create mode 100644 doc/documentation/tutorial-examples/016.c create mode 100644 doc/documentation/tutorial-examples/017.c create mode 100644 doc/documentation/tutorial-examples/018.c create mode 100644 doc/documentation/tutorial-examples/019.c create mode 100644 doc/documentation/tutorial-examples/020.c create mode 100644 doc/documentation/tutorial-examples/021.c create mode 100644 doc/documentation/tutorial-examples/022.c create mode 100644 doc/documentation/tutorial-examples/023.c create mode 100644 doc/documentation/tutorial-examples/024.c create mode 100644 doc/documentation/tutorial-examples/025.c create mode 100644 doc/documentation/tutorial-examples/026.c (limited to 'doc/documentation') diff --git a/doc/documentation/Makefile.am b/doc/documentation/Makefile.am new file mode 100644 index 0000000000..21852a9c32 --- /dev/null +++ b/doc/documentation/Makefile.am @@ -0,0 +1,214 @@ +# This Makefile.am is in the public domain +docdir = $(datadir)/doc/gnunet/ + +infoimagedir = $(infodir)/images + +#DOT_FILES = images/$(wildcard *.dot) + +#DOT_VECTOR_GRAPHICS = \ +# $(DOT_FILES:%.dot=%.eps) \ +# $(DOT_FILES:%.dot=%.pdf) + +AM_MAKEINFOHTMLFLAGS = --no-split --css-ref=docstyle.css + +dist_infoimage_DATA = \ + images/gnunet-gtk-0-10-gns-a-done.png \ + images/gnunet-gtk-0-10-gns-a.png \ + images/daemon_lego_block.png \ + images/gnunet-gtk-0-10-gns.png \ + images/gnunet-0-10-peerinfo.png \ + images/gnunet-gtk-0-10-identity.png \ + images/gnunet-fs-gtk-0-10-star-tab.png \ + images/gnunet-gtk-0-10.png \ + images/gnunet-gtk-0-10-download-area.png \ + images/gnunet-gtk-0-10-search-selected.png \ + images/gnunet-gtk-0-10-fs-menu.png \ + images/gnunet-gtk-0-10-traffic.png \ + images/gnunet-gtk-0-10-fs.png \ + images/gnunet-namestore-gtk-phone.png \ + images/gnunet-gtk-0-10-fs-publish-editing.png \ + images/gnunet-namestore-gtk-vpn.png \ + images/gnunet-gtk-0-10-fs-published.png \ + images/gnunet-setup-exit.png \ + images/gnunet-gtk-0-10-fs-publish.png \ + images/iceweasel-preferences.png \ + images/gnunet-gtk-0-10-fs-publish-select.png \ + images/iceweasel-proxy.png \ + images/gnunet-gtk-0-10-fs-publish-with-file_0.png \ + images/service_lego_block.png \ + images/gnunet-gtk-0-10-fs-publish-with-file.png \ + images/service_stack.png \ + images/gnunet-gtk-0-10-fs-search.png \ + images/gnunet-tutorial-service.png \ + images/gnunet-tutorial-system.png \ + images/daemon_lego_block.svg \ + images/lego_stack.svg \ + images/service_lego_block.svg \ + images/structure.dot + +# images/$(wildcard *.png) \ +# images/$(wildcard *.svg) +# $(DOT_FILES:%.dot=%.png) + +#DOT_OPTIONS = \ +# -Gratio=.9 -Gnodesep=.005 -Granksep=.00005 \ +# -Nfontsite=9 -Nheight=.1 -Nwidth=.1 + +# .dot.png: +# $(AM_V_DOT)$(DOT) -Tpng $(DOT_OPTIONS) < "$<" > "$(srcdir)/$@.tmp"; \ +# mv "$(srcdir)/$@.tmp" "$(srcdir)/$@" + +# .dot.pdf: +# $(AM_V_DOT)$(DOT) -Tpdf $(DOT_OPTIONS) < "$<" > "$(srcdir)/$@.tmp"; \ +# mv "$(srcdir)/$@.tmp" "$(srcdir)/$@" + +# .dot.eps: +# $(AM_V_DOT)$(DOT) -Teps $(DOT_OPTIONS) < "$<" > "$(srcdir)/$@.tmp"; \ +# mv "$(srcdir)/$@.tmp" "$(srcdir)/$@" + +# .png.eps: +# $(AM_V_GEN)convert "$<" "$@-tmp.eps"; \ +# mv "$@-tmp.eps" "$@" + +# pdf-local: $(DOT_FILES=%.dot=$(top_srcdir)/%.pdf) +# info-local: $(DOT_FILES=%.dot=$(top_srcdir)/%.png) +# ps-local: $(DOT_FILES=%.dot=$(top_srcdir)/%.eps) \ +# $(top_srcdir)/%D%/images/coreutils-size-map.eps +# dvi-local: ps-local + +gnunet_tutorial_examples = \ + 001.c \ + 002.c \ + 003.c \ + 004.c \ + 005.c \ + 006.c \ + 007.c \ + 008.c \ + 009.c \ + 010.c \ + 011.c \ + 012.c \ + 013.c \ + 013.1.c \ + 014.c \ + 015.c \ + 016.c \ + 017.c \ + 018.c \ + 019.c \ + 020.c \ + 021.c \ + 022.c \ + 023.c \ + 024.c \ + 025.c \ + 026.c + +info_TEXINFOS = \ + gnunet.texi \ + gnunet-c-tutorial.texi + +gnunet_TEXINFOS = \ + chapters/developer.texi \ + chapters/installation.texi \ + chapters/philosophy.texi \ + chapters/user.texi \ + chapters/vocabulary.texi \ + fdl-1.3.texi \ + gpl-3.0.texi + +EXTRA_DIST = \ + $(gnunet_TEXINFOS) \ + $(gnunet_tutorial_examples) \ + docstyle.css + + +# $(DOT_FILES) \ +# $(DOT_VECTOR_GRAPHICS) + +DISTCLEANFILES = \ + gnunet.cps \ + gnunet-c-tutorial.cps \ + chapters/developer.cps \ + chapters/installation.cps \ + chapter/philosophy.cps \ + chapters/user.cps \ + fdl-1.3.cps \ + gpl-3.0.cps + +# if HAVE_EXTENDED_DOCUMENTATION_BUILDING +daemon_lego_block.png: images/daemon_lego_block.svg + convert images/daemon_lego_block.svg images/daemon_lego_block.png && + pngcrush images/daemon_lego_block.png images/daemon_lego_block.png + +service_lego_block.png: images/service_lego_block.svg + convert images/service_lego_block.svg images/service_lego_block.png && + pngcrush images/service_lego_block.png images/serivce_lego_block.png + +lego_stack.png: images/lego_stack.svg + convert images/lego_stack.svg images/lego_stack.png && + pngcrush images/lego_stack.png images/lego_stack.png + +# FIXME: The usage of 'date' strings causes a warning. +# version.texi: +# echo "@set UPDATED $(date +'%d %B %Y')" > $@ +# echo "@set UPDATED-MONTH $(date +'%B %Y')" >> $@ +# echo "@set EDITION $(PACKAGE_VERSION)" >> $@ +# echo "@set VERSION $(PACKAGE_VERSION)" >> $@ + +# Workaround for makeinfo error. Whcih in turn introduces more +# date-related 'warnings'. Well. +version2.texi: + echo "@set UPDATED $(date +'%d %B %Y')" > $@ + echo "@set UPDATED-MONTH $(date +'%B %Y')" >> $@ + echo "@set EDITION $(PACKAGE_VERSION)" >> $@ + echo "@set VERSION $(PACKAGE_VERSION)" >> $@ + +# FIXME: rm *.html and *.pdf +#doc-clean: +# @rm *.aux *.log *.toc *.cp *.cps + +doc-all-install: + @mkdir -p $(DESTDIR)/$(docdir) + @mkdir -p $(DESTDIR)/$(infoimagedir) + @mkdir -p $(DESTDIR)/$(infodir) + @install -m 0755 gnunet.pdf $(DESTDIR)/$(docdir) + @install -m 0755 gnunet-c-tutorial.pdf $(DESTDIR)/$(docdir) + @install -m 0755 gnunet-c-tutorial.info $(DESTDIR)/$(infodir) + @install -m 0755 gnunet.info $(DESTDIR)/$(infodir) + @install gnunet.html $(DESTDIR)/$(docdir) + @install gnunet-c-tutorial.html $(DESTDIR)/$(docdir) + +# @cp -r images $(DESTDIR)/$(infoimagedir) + +# TODO: Add more to clean. +clean: + @rm gnunet.pdf + @rm gnunet.html + @rm gnunet.info + @rm gnunet.info-1 + @rm gnunet.info-2 + @rm gnunet.info-3 + @rm gnunet-c-tutorial.pdf + @rm gnunet-c-tutorial.info + @rm gnunet-c-tutorial.html + @rm -r gnunet.t2p + @rm -r gnunet-c-tutorial.t2p + +# CLEANFILES = \ +# gnunet.log \ +# gnunet-c-tutorial.log \ +# $(wildcard *.aux) \ +# $(wildcard *.toc) \ +# $(wildcard *.cp) \ +# $(wildcard *.cps) + +#.PHONY: version.texi +# if HAVE_EXTENDED_DOCUMENTATION_BUILDING_PDF + +# if HAVE_EXTENDED_DOCUMENTATION_BUILDING_HTML + +# endif +# endif +# endif diff --git a/doc/documentation/README.txt b/doc/documentation/README.txt new file mode 100644 index 0000000000..d4c19bba61 --- /dev/null +++ b/doc/documentation/README.txt @@ -0,0 +1,64 @@ +To be moved to an appropriate section of "how to write documentation" or +"how to contribute to the documentation": + +1. When writing documentation, please use gender-neutral wording when + referring to people, such as singular “they”, “their”, “them”, and + so forth. -> https://en.wikipedia.org/wiki/Singular_they + +2. Keep line length below 74 characters. + +3. Do not use tab characters (see chapter 2.1 texinfo manual) +* What's left to do + +- Which Texlive modules are needed? Decrease the size. +- Update the content of gnunet documentation. + +* How to use (hack) on this + +** with guix + +Adjust accordingly, ie read the Guix Documentation: +setenv GUIX_PACKAGE_PATH "gnunet/contrib/packages/guix/packages" +guix environment gnunet-doc +and +guix build -f contrib/packages/guix/gnunet-doc.scm + +** without guix + +You need to have Texinfo and Texlive in your path. +sh bootstrap +./configure --enable-documentation +cd doc +make (format you want) + +for example: make html, make info, make pdf + +* structure (relations) + +** gnunet.texi + -> chapters/developer.texi + -> chapters/installation.texi + -> chapters/philosophy.texi + -> chapters/user.texi + -> chapters/vocabulary.texi + -> images/* + -> gpl-3.0.texi + -> fdl-1.3.texi + +** gnunet-c-tutorial.texi + -> figs/Service.pdf + -> figs/System.pdf + -> tutorial-examples/*.c + -> gpl-3.0.texi + -> fdl-1.3.texi + +- gnunet-c-tutorial-v1.pdf: original LaTeX "gnunet-c-tutorial.pdf". +- man folder: the man pages. +- doxygen folder +- outdated-and-old-installation-instructions.txt: self described within the file. + + +Use `gendocs', add to the manual/ directory of the web site. + + $ cd doc + $ gendocs.sh gnunet "GNUnet 0.10.X Reference Manual" diff --git a/doc/documentation/chapters/developer.texi b/doc/documentation/chapters/developer.texi new file mode 100644 index 0000000000..e690e5f5bd --- /dev/null +++ b/doc/documentation/chapters/developer.texi @@ -0,0 +1,7926 @@ +@c *********************************************************************** +@node GNUnet Developer Handbook +@chapter GNUnet Developer Handbook + +This book is intended to be an introduction for programmers that want to +extend the GNUnet framework. GNUnet is more than a simple peer-to-peer +application. For developers, GNUnet is: + +@itemize @bullet +@item Free software under the GNU General Public License, with a community +that believes in the GNU philosophy +@item +A set of standards, including coding conventions and architectural rules +@item +A set of layered protocols, both specifying the communication between +peers as well as the communication between components of a single peer. +@item +A set of libraries with well-defined APIs suitable for writing extensions +@end itemize + +In particular, the architecture specifies that a peer consists of many +processes communicating via protocols. Processes can be written in almost +any language. C and Java APIs exist for accessing existing services and +for writing extensions. It is possible to write extensions in other +languages by implementing the necessary IPC protocols. + +GNUnet can be extended and improved along many possible dimensions, and +anyone interested in free software and freedom-enhancing networking is +welcome to join the effort. This developer handbook attempts to provide +an initial introduction to some of the key design choices and central +components of the system. This manual is far from complete, and we +welcome informed contributions, be it in the form of new chapters or +insightful comments. + +However, the website is experiencing a constant onslaught of sophisticated +link-spam entered manually by exploited workers solving puzzles and +customizing text. To limit this commercial defacement, we are strictly +moderating comments and have disallowed "normal" users from posting new +content. However, this is really only intended to keep the spam at bay. If +you are a real user or aspiring developer, please drop us a note +(IRC, e-mail, contact form) with your user profile ID number included. +We will then relax these restrictions on your account. We're sorry for +this inconvenience; however, few people would want to read this site +if 99% of it was advertisements for bogus websites. + + + +@c *********************************************************************** + + + + + + + + +@menu +* Developer Introduction:: +* Code overview:: +* System Architecture:: +* Subsystem stability:: +* Naming conventions and coding style guide:: +* Build-system:: +* Developing extensions for GNUnet using the gnunet-ext template:: +* Writing testcases:: +* GNUnet's TESTING library:: +* Performance regression analysis with Gauger:: +* GNUnet's TESTBED Subsystem:: +* libgnunetutil:: +* The Automatic Restart Manager (ARM):: +* GNUnet's TRANSPORT Subsystem:: +* NAT library:: +* Distance-Vector plugin:: +* SMTP plugin:: +* Bluetooth plugin:: +* WLAN plugin:: +* The ATS Subsystem:: +* GNUnet's CORE Subsystem:: +* GNUnet's CADET subsystem:: +* GNUnet's NSE subsystem:: +* GNUnet's HOSTLIST subsystem:: +* GNUnet's IDENTITY subsystem:: +* GNUnet's NAMESTORE Subsystem:: +* GNUnet's PEERINFO subsystem:: +* GNUnet's PEERSTORE subsystem:: +* GNUnet's SET Subsystem:: +* GNUnet's STATISTICS subsystem:: +* GNUnet's Distributed Hash Table (DHT):: +* The GNU Name System (GNS):: +* The GNS Namecache:: +* The REVOCATION Subsystem:: +* GNUnet's File-sharing (FS) Subsystem:: +* GNUnet's REGEX Subsystem:: +@end menu + +@node Developer Introduction +@section Developer Introduction + +This developer handbook is intended as first introduction to GNUnet for +new developers that want to extend the GNUnet framework. After the +introduction, each of the GNUnet subsystems (directories in the +@file{src/} tree) is (supposed to be) covered in its own chapter. In +addition to this documentation, GNUnet developers should be aware of the +services available on the GNUnet server to them. + +New developers can have a look a the GNUnet tutorials for C and java +available in the @file{src/} directory of the repository or under the +following links: + +@c ** FIXME: Link to files in source, not online. +@c ** FIXME: Where is the Java tutorial? +@itemize @bullet +@item @uref{https://gnunet.org/git/gnunet.git/plain/doc/gnunet-c-tutoria +l.pdf, GNUnet C tutorial} +@item GNUnet Java tutorial +@end itemize + +In addition to this book, the GNUnet server contains various resources for +GNUnet developers. They are all conveniently reachable via the "Developer" +entry in the navigation menu. Some additional tools (such as static +analysis reports) require a special developer access to perform certain +operations. If you feel you need access, you should contact +@uref{http://grothoff.org/christian/, Christian Grothoff}, +GNUnet's maintainer. + +The public subsystems on the GNUnet server that help developers are: + +@itemize @bullet +@item The Version control system keeps our code and enables distributed +development. Only developers with write access can commit code, everyone +else is encouraged to submit patches to the +@uref{https://lists.gnu.org/mailman/listinfo/gnunet-developers, +GNUnet-developers mailinglist}. +@item The GNUnet bugtracking system is used to track feature requests, +open bug reports and their resolutions. Anyone can report bugs, only +developers can claim to have fixed them. +@item A buildbot is used to check GNUnet builds automatically on a range +of platforms. Builds are triggered automatically after 30 minutes of no +changes to Git. +@item The current quality of our automated test suite is assessed using +Code coverage analysis. This analysis is run daily; however the webpage +is only updated if all automated tests pass at that time. Testcases that +improve our code coverage are always welcome. +@item We try to automatically find bugs using a static analysis scan. +This scan is run daily; however the webpage is only updated if all +automated tests pass at the time. Note that not everything that is +flagged by the analysis is a bug, sometimes even good code can be marked +as possibly problematic. Nevertheless, developers are encouraged to at +least be aware of all issues in their code that are listed. +@item We use Gauger for automatic performance regression visualization. +Details on how to use Gauger are here. +@item We use @uref{http://junit.org/, junit} to automatically test +gnunet-java. Automatically generated, current reports on the test suite +are here. +@item We use Cobertura to generate test coverage reports for gnunet-java. +Current reports on test coverage are here. +@end itemize + + + +@c *********************************************************************** +@menu +* Project overview:: +@end menu + +@node Project overview +@subsection Project overview + +The GNUnet project consists at this point of several sub-projects. This +section is supposed to give an initial overview about the various +sub-projects. Note that this description also lists projects that are far +from complete, including even those that have literally not a single line +of code in them yet. + +GNUnet sub-projects in order of likely relevance are currently: + +@table @asis + +@item gnunet Core of the P2P framework, including file-sharing, VPN and +chat applications; this is what the developer handbook covers mostly +@item gnunet-gtk Gtk+-based user interfaces, including gnunet-fs-gtk +(file-sharing), gnunet-statistics-gtk (statistics over time), +gnunet-peerinfo-gtk (information about current connections and known +peers), gnunet-chat-gtk (chat GUI) and gnunet-setup (setup tool for +"everything") +@item gnunet-fuse Mounting directories shared via GNUnet's file-sharing +on Linux +@item gnunet-update Installation and update tool +@item gnunet-ext Template for starting 'external' GNUnet projects +@item gnunet-java Java APIs for writing GNUnet services and applications +@c ** FIXME: Point to new website repository once we have it: +@c ** @item svn/gnunet-www/ Code and media helping drive the GNUnet +website +@item eclectic Code to run +GNUnet nodes on testbeds for research, development, testing and evaluation +@c ** FIXME: Solve the status and location of gnunet-qt +@item gnunet-qt qt-based GNUnet GUI (dead?) +@item gnunet-cocoa cocoa-based GNUnet GUI (dead?) + +@end table + +We are also working on various supporting libraries and tools: +@c ** FIXME: What about gauger, and what about libmwmodem? + +@table @asis +@item libextractor GNU libextractor (meta data extraction) +@item libmicrohttpd GNU libmicrohttpd (embedded HTTP(S) server library) +@item gauger Tool for performance regression analysis +@item monkey Tool for automated debugging of distributed systems +@item libmwmodem Library for accessing satellite connection quality +reports +@end table + +Finally, there are various external projects (see links for a list of +those that have a public website) which build on top of the GNUnet +framework. + +@c *********************************************************************** +@node Code overview +@section Code overview + +This section gives a brief overview of the GNUnet source code. +Specifically, we sketch the function of each of the subdirectories in +the @file{gnunet/src/} directory. The order given is roughly bottom-up +(in terms of the layers of the system). + +@table @asis +@item util/ --- libgnunetutil Library with general utility functions, all +GNUnet binaries link against this library. Anything from memory +allocation and data structures to cryptography and inter-process +communication. The goal is to provide an OS-independent interface and +more 'secure' or convenient implementations of commonly used primitives. +The API is spread over more than a dozen headers, developers should study +those closely to avoid duplicating existing functions. +@item hello/ --- libgnunethello HELLO messages are used to +describe under which addresses a peer can be reached (for example, +protocol, IP, port). This library manages parsing and generating of HELLO +messages. +@item block/ --- libgnunetblock The DHT and other components of GNUnet +store information in units called 'blocks'. Each block has a type and the +type defines a particular format and how that binary format is to be +linked to a hash code (the key for the DHT and for databases). The block +library is a wapper around block plugins which provide the necessary +functions for each block type. +@item statistics/ The statistics service enables associating +values (of type uint64_t) with a componenet name and a string. The main +uses is debugging (counting events), performance tracking and user +entertainment (what did my peer do today?). +@item arm/ The automatic-restart-manager (ARM) service +is the GNUnet master service. Its role is to start gnunet-services, to +re-start them when they crashed and finally to shut down the system when +requested. +@item peerinfo/ The peerinfo service keeps track of which peers are known +to the local peer and also tracks the validated addresses for each peer +(in the form of a HELLO message) for each of those peers. The peer is not +necessarily connected to all peers known to the peerinfo service. +Peerinfo provides persistent storage for peer identities --- peers are +not forgotten just because of a system restart. +@item datacache/ --- libgnunetdatacache The datacache +library provides (temporary) block storage for the DHT. Existing plugins +can store blocks in Sqlite, Postgres or MySQL databases. All data stored +in the cache is lost when the peer is stopped or restarted (datacache +uses temporary tables). +@item datastore/ The datastore service stores file-sharing blocks in +databases for extended periods of time. In contrast to the datacache, data +is not lost when peers restart. However, quota restrictions may still +cause old, expired or low-priority data to be eventually discarded. +Existing plugins can store blocks in Sqlite, Postgres or MySQL databases. +@item template/ Template for writing a new service. Does nothing. +@item ats/ The automatic transport +selection (ATS) service is responsible for deciding which address (i.e. +which transport plugin) should be used for communication with other peers, +and at what bandwidth. +@item nat/ --- libgnunetnat Library that provides basic +functions for NAT traversal. The library supports NAT traversal with +manual hole-punching by the user, UPnP and ICMP-based autonomous NAT +traversal. The library also includes an API for testing if the current +configuration works and the @code{gnunet-nat-server} which provides an +external service to test the local configuration. +@item fragmentation/ --- libgnunetfragmentation Some +transports (UDP and WLAN, mostly) have restrictions on the maximum +transfer unit (MTU) for packets. The fragmentation library can be used to +break larger packets into chunks of at most 1k and transmit the resulting +fragments reliabily (with acknowledgement, retransmission, timeouts, +etc.). +@item transport/ The transport service is responsible for managing the +basic P2P communication. It uses plugins to support P2P communication +over TCP, UDP, HTTP, HTTPS and other protocols.The transport service +validates peer addresses, enforces bandwidth restrictions, limits the +total number of connections and enforces connectivity restrictions (i.e. +friends-only). +@item peerinfo-tool/ +This directory contains the gnunet-peerinfo binary which can be used to +inspect the peers and HELLOs known to the peerinfo service. +@item core/ The core +service is responsible for establishing encrypted, authenticated +connections with other peers, encrypting and decrypting messages and +forwarding messages to higher-level services that are interested in them. +@item testing/ --- +libgnunettesting The testing library allows starting (and stopping) peers +for writing testcases.@ +It also supports automatic generation of configurations for peers +ensuring that the ports and paths are disjoint. libgnunettesting is also +the foundation for the testbed service +@item testbed/ The testbed service is +used for creating small or large scale deployments of GNUnet peers for +evaluation of protocols. It facilitates peer depolyments on multiple +hosts (for example, in a cluster) and establishing varous network +topologies (both underlay and overlay). +@item nse/ The network size estimation (NSE) service +implements a protocol for (securely) estimating the current size of the +P2P network. +@item dht/ The distributed hash table (DHT) service provides a +distributed implementation of a hash table to store blocks under hash +keys in the P2P network. +@item hostlist/ The hostlist service allows learning about +other peers in the network by downloading HELLO messages from an HTTP +server, can be configured to run such an HTTP server and also implements +a P2P protocol to advertise and automatically learn about other peers +that offer a public hostlist server. +@item topology/ The topology service is responsible for +maintaining the mesh topology. It tries to maintain connections to friends +(depending on the configuration) and also tries to ensure that the peer +has a decent number of active connections at all times. If necessary, new +connections are added. All peers should run the topology service, +otherwise they may end up not being connected to any other peer (unless +some other service ensures that core establishes the required +connections). The topology service also tells the transport service which +connections are permitted (for friend-to-friend networking) +@item fs/ The file-sharing (FS) service implements GNUnet's +file-sharing application. Both anonymous file-sharing (using gap) and +non-anonymous file-sharing (using dht) are supported. +@item cadet/ The CADET +service provides a general-purpose routing abstraction to create +end-to-end encrypted tunnels in mesh networks. We wrote a paper +documenting key aspects of the design. +@item tun/ --- libgnunettun Library for building IPv4, IPv6 +packets and creating checksums for UDP, TCP and ICMP packets. The header +defines C structs for common Internet packet formats and in particular +structs for interacting with TUN (virtual network) interfaces. +@item mysql/ --- +libgnunetmysql Library for creating and executing prepared MySQL +statements and to manage the connection to the MySQL database. +Essentially a lightweight wrapper for the interaction between GNUnet +components and libmysqlclient. +@item dns/ Service that allows intercepting and modifying DNS requests of +the local machine. Currently used for IPv4-IPv6 protocol translation +(DNS-ALG) as implemented by "pt/" and for the GNUnet naming system. The +service can also be configured to offer an exit service for DNS traffic. +@item vpn/ The virtual +public network (VPN) service provides a virtual tunnel interface (VTUN) +for IP routing over GNUnet. Needs some other peers to run an "exit" +service to work. +Can be activated using the "gnunet-vpn" tool or integrated with DNS using +the "pt" daemon. +@item exit/ Daemon to allow traffic from the VPN to exit this +peer to the Internet or to specific IP-based services of the local peer. +Currently, an exit service can only be restricted to IPv4 or IPv6, not to +specific ports and or IP address ranges. If this is not acceptable, +additional firewall rules must be added manually. exit currently only +works for normal UDP, TCP and ICMP traffic; DNS queries need to leave the +system via a DNS service. +@item pt/ protocol translation daemon. This daemon enables 4-to-6, +6-to-4, 4-over-6 or 6-over-4 transitions for the local system. It +essentially uses "DNS" to intercept DNS replies and then maps results to +those offered by the VPN, which then sends them using mesh to some daemon +offering an appropriate exit service. +@item identity/ Management of egos (alter egos) of a user; identities are +essentially named ECC private keys and used for zones in the GNU name +system and for namespaces in file-sharing, but might find other uses later +@item revocation/ Key revocation service, can be used to revoke the +private key of an identity if it has been compromised +@item namecache/ Cache +for resolution results for the GNU name system; data is encrypted and can +be shared among users, loss of the data should ideally only result in a +performance degradation (persistence not required) +@item namestore/ Database +for the GNU name system with per-user private information, persistence +required +@item gns/ GNU name system, a GNU approach to DNS and PKI. +@item dv/ A plugin +for distance-vector (DV)-based routing. DV consists of a service and a +transport plugin to provide peers with the illusion of a direct P2P +connection for connections that use multiple (typically up to 3) hops in +the actual underlay network. +@item regex/ Service for the (distributed) evaluation of +regular expressions. +@item scalarproduct/ The scalar product service offers an +API to perform a secure multiparty computation which calculates a scalar +product between two peers without exposing the private input vectors of +the peers to each other. +@item consensus/ The consensus service will allow a set +of peers to agree on a set of values via a distributed set union +computation. +@item rest/ The rest API allows access to GNUnet services using RESTful +interaction. The services provide plugins that can exposed by the rest +server. +@item experimentation/ The experimentation daemon coordinates distributed +experimentation to evaluate transport and ats properties +@end table + +@c *********************************************************************** +@node System Architecture +@section System Architecture + +GNUnet developers like legos. The blocks are indestructible, can be +stacked together to construct complex buildings and it is generally easy +to swap one block for a different one that has the same shape. GNUnet's +architecture is based on legos: + +@c images here + +This chapter documents the GNUnet lego system, also known as GNUnet's +system architecture. + +The most common GNUnet component is a service. Services offer an API (or +several, depending on what you count as "an API") which is implemented as +a library. The library communicates with the main process of the service +using a service-specific network protocol. The main process of the service +typically doesn't fully provide everything that is needed --- it has holes +to be filled by APIs to other services. + +A special kind of component in GNUnet are user interfaces and daemons. +Like services, they have holes to be filled by APIs of other services. +Unlike services, daemons do not implement their own network protocol and +they have no API: + +The GNUnet system provides a range of services, daemons and user +interfaces, which are then combined into a layered GNUnet instance (also +known as a peer). + +Note that while it is generally possible to swap one service for another +compatible service, there is often only one implementation. However, +during development we often have a "new" version of a service in parallel +with an "old" version. While the "new" version is not working, developers +working on other parts of the service can continue their development by +simply using the "old" service. Alternative design ideas can also be +easily investigated by swapping out individual components. This is +typically achieved by simply changing the name of the "BINARY" in the +respective configuration section. + +Key properties of GNUnet services are that they must be separate +processes and that they must protect themselves by applying tight error +checking against the network protocol they implement (thereby achieving a +certain degree of robustness). + +On the other hand, the APIs are implemented to tolerate failures of the +service, isolating their host process from errors by the service. If the +service process crashes, other services and daemons around it should not +also fail, but instead wait for the service process to be restarted by +ARM. + + +@c *********************************************************************** +@node Subsystem stability +@section Subsystem stability + +This page documents the current stability of the various GNUnet +subsystems. Stability here describes the expected degree of compatibility +with future versions of GNUnet. For each subsystem we distinguish between +compatibility on the P2P network level (communication protocol between +peers), the IPC level (communication between the service and the service +library) and the API level (stability of the API). P2P compatibility is +relevant in terms of which applications are likely going to be able to +communicate with future versions of the network. IPC communication is +relevant for the implementation of language bindings that re-implement the +IPC messages. Finally, API compatibility is relevant to developers that +hope to be able to avoid changes to applications build on top of the APIs +of the framework. + +The following table summarizes our current view of the stability of the +respective protocols or APIs: + +@multitable @columnfractions .20 .20 .20 .20 +@headitem Subsystem @tab P2P @tab IPC @tab C API +@item util @tab n/a @tab n/a @tab stable +@item arm @tab n/a @tab stable @tab stable +@item ats @tab n/a @tab unstable @tab testing +@item block @tab n/a @tab n/a @tab stable +@item cadet @tab testing @tab testing @tab testing +@item consensus @tab experimental @tab experimental @tab experimental +@item core @tab stable @tab stable @tab stable +@item datacache @tab n/a @tab n/a @tab stable +@item datastore @tab n/a @tab stable @tab stable +@item dht @tab stable @tab stable @tab stable +@item dns @tab stable @tab stable @tab stable +@item dv @tab testing @tab testing @tab n/a +@item exit @tab testing @tab n/a @tab n/a +@item fragmentation @tab stable @tab n/a @tab stable +@item fs @tab stable @tab stable @tab stable +@item gns @tab stable @tab stable @tab stable +@item hello @tab n/a @tab n/a @tab testing +@item hostlist @tab stable @tab stable @tab n/a +@item identity @tab stable @tab stable @tab n/a +@item multicast @tab experimental @tab experimental @tab experimental +@item mysql @tab stable @tab n/a @tab stable +@item namestore @tab n/a @tab stable @tab stable +@item nat @tab n/a @tab n/a @tab stable +@item nse @tab stable @tab stable @tab stable +@item peerinfo @tab n/a @tab stable @tab stable +@item psyc @tab experimental @tab experimental @tab experimental +@item pt @tab n/a @tab n/a @tab n/a +@item regex @tab stable @tab stable @tab stable +@item revocation @tab stable @tab stable @tab stable +@item social @tab experimental @tab experimental @tab experimental +@item statistics @tab n/a @tab stable @tab stable +@item testbed @tab n/a @tab testing @tab testing +@item testing @tab n/a @tab n/a @tab testing +@item topology @tab n/a @tab n/a @tab n/a +@item transport @tab stable @tab stable @tab stable +@item tun @tab n/a @tab n/a @tab stable +@item vpn @tab testing @tab n/a @tab n/a +@end multitable + +Here is a rough explanation of the values: + +@table @samp +@item stable +No incompatible changes are planned at this time; for IPC/APIs, if +there are incompatible changes, they will be minor and might only require +minimal changes to existing code; for P2P, changes will be avoided if at +all possible for the 0.10.x-series + +@item testing +No incompatible changes are +planned at this time, but the code is still known to be in flux; so while +we have no concrete plans, our expectation is that there will still be +minor modifications; for P2P, changes will likely be extensions that +should not break existing code + +@item unstable +Changes are planned and will happen; however, they +will not be totally radical and the result should still resemble what is +there now; nevertheless, anticipated changes will break protocol/API +compatibility + +@item experimental +Changes are planned and the result may look nothing like +what the API/protocol looks like today + +@item unknown +Someone should think about where this subsystem headed + +@item n/a +This subsystem does not have an API/IPC-protocol/P2P-protocol +@end table + +@c *********************************************************************** +@node Naming conventions and coding style guide +@section Naming conventions and coding style guide + +Here you can find some rules to help you write code for GNUnet. + + + +@c *********************************************************************** +@menu +* Naming conventions:: +* Coding style:: +@end menu + +@node Naming conventions +@subsection Naming conventions + + +@c *********************************************************************** +@menu +* include files:: +* binaries:: +* logging:: +* configuration:: +* exported symbols:: +* private (library-internal) symbols (including structs and macros):: +* testcases:: +* performance tests:: +* src/ directories:: +@end menu + +@node include files +@subsubsection include files + +@itemize @bullet +@item _lib: library without need for a process +@item _service: library that needs a service process +@item _plugin: plugin definition +@item _protocol: structs used in network protocol +@item exceptions: +@itemize @bullet +@item gnunet_config.h --- generated +@item platform.h --- first included +@item plibc.h --- external library +@item gnunet_common.h --- fundamental routines +@item gnunet_directories.h --- generated +@item gettext.h --- external library +@end itemize +@end itemize + +@c *********************************************************************** +@node binaries +@subsubsection binaries + +@itemize @bullet +@item gnunet-service-xxx: service process (has listen socket) +@item gnunet-daemon-xxx: daemon process (no listen socket) +@item gnunet-helper-xxx[-yyy]: SUID helper for module xxx +@item gnunet-yyy: command-line tool for end-users +@item libgnunet_plugin_xxx_yyy.so: plugin for API xxx +@item libgnunetxxx.so: library for API xxx +@end itemize + +@c *********************************************************************** +@node logging +@subsubsection logging + +@itemize @bullet +@item services and daemons use their directory name in GNUNET_log_setup +(i.e. 'core') and log using plain 'GNUNET_log'. +@item command-line tools use their full name in GNUNET_log_setup (i.e. +'gnunet-publish') and log using plain 'GNUNET_log'. +@item service access libraries log using 'GNUNET_log_from' and use +'DIRNAME-api' for the component (i.e. 'core-api') +@item pure libraries (without associated service) use 'GNUNET_log_from' +with the component set to their library name (without lib or '.so'), +which should also be their directory name (i.e. 'nat') +@item plugins should use 'GNUNET_log_from' with the directory name and the +plugin name combined to produce the component name (i.e. 'transport-tcp'). +@item logging should be unified per-file by defining a LOG macro with the +appropriate arguments, along these lines:@ #define LOG(kind,...) +GNUNET_log_from (kind, "example-api",__VA_ARGS__) +@end itemize + +@c *********************************************************************** +@node configuration +@subsubsection configuration + +@itemize @bullet +@item paths (that are substituted in all filenames) are in PATHS (have as +few as possible) +@item all options for a particular module (src/MODULE) are under [MODULE] +@item options for a plugin of a module are under [MODULE-PLUGINNAME] +@end itemize + +@c *********************************************************************** +@node exported symbols +@subsubsection exported symbols + +@itemize @bullet +@item must start with "GNUNET_modulename_" and be defined in +"modulename.c" +@item exceptions: those defined in gnunet_common.h +@end itemize + +@c *********************************************************************** +@node private (library-internal) symbols (including structs and macros) +@subsubsection private (library-internal) symbols (including structs and macros) + +@itemize @bullet +@item must NOT start with any prefix +@item must not be exported in a way that linkers could use them or@ other +libraries might see them via headers; they must be either@ +declared/defined in C source files or in headers that are in@ the +respective directory under src/modulename/ and NEVER be@ declared +in src/include/. +@end itemize + +@node testcases +@subsubsection testcases + +@itemize @bullet +@item must be called "test_module-under-test_case-description.c" +@item "case-description" maybe omitted if there is only one test +@end itemize + +@c *********************************************************************** +@node performance tests +@subsubsection performance tests + +@itemize @bullet +@item must be called "perf_module-under-test_case-description.c" +@item "case-description" maybe omitted if there is only one performance +test +@item Must only be run if HAVE_BENCHMARKS is satisfied +@end itemize + +@c *********************************************************************** +@node src/ directories +@subsubsection src/ directories + +@itemize @bullet +@item gnunet-NAME: end-user applications (i.e., gnunet-search, gnunet-arm) +@item gnunet-service-NAME: service processes with accessor library (i.e., +gnunet-service-arm) +@item libgnunetNAME: accessor library (_service.h-header) or standalone +library (_lib.h-header) +@item gnunet-daemon-NAME: daemon process without accessor library (i.e., +gnunet-daemon-hostlist) and no GNUnet management port +@item libgnunet_plugin_DIR_NAME: loadable plugins (i.e., +libgnunet_plugin_transport_tcp) +@end itemize + +@c *********************************************************************** +@node Coding style +@subsection Coding style + +@itemize @bullet +@item GNU guidelines generally apply +@item Indentation is done with spaces, two per level, no tabs +@item C99 struct initialization is fine +@item declare only one variable per line, so@ + +@example +int i; int j; +@end example + +instead of + +@example +int i,j; +@end example + +This helps keep diffs small and forces developers to think precisely about +the type of every variable. Note that @code{char *} is different from +@code{const char*} and @code{int} is different from @code{unsigned int} +or @code{uint32_t}. Each variable type should be chosen with care. + +@item While @code{goto} should generally be avoided, having a @code{goto} +to the end of a function to a block of clean up statements (free, close, +etc.) can be acceptable. + +@item Conditions should be written with constants on the left (to avoid +accidental assignment) and with the 'true' target being either the +'error' case or the significantly simpler continuation. For example: + +@example +if (0 != stat ("filename," &sbuf)) @{ error(); @} else @{ + /* handle normal case here */ +@} +@end example + +instead of + +@example +if (stat ("filename," &sbuf) == 0) @{ + /* handle normal case here */ +@} else @{ error(); @} +@end example + +If possible, the error clause should be terminated with a 'return' (or +'goto' to some cleanup routine) and in this case, the 'else' clause +should be omitted: + +@example +if (0 != stat ("filename," &sbuf)) @{ error(); return; @} +/* handle normal case here */ +@end example + +This serves to avoid deep nesting. The 'constants on the left' rule +applies to all constants (including. @code{GNUNET_SCHEDULER_NO_TASK}), +NULL, and enums). With the two above rules (constants on left, errors in +'true' branch), there is only one way to write most branches correctly. + +@item Combined assignments and tests are allowed if they do not hinder +code clarity. For example, one can write: + +@example +if (NULL == (value = lookup_function())) @{ error(); return; @} +@end example + + +@item Use @code{break} and @code{continue} wherever possible to avoid +deep(er) nesting. Thus, we would write: + +@example +next = head; while (NULL != (pos = next)) @{ next = pos->next; if (! +should_free (pos)) continue; GNUNET_CONTAINER_DLL_remove (head, tail, pos); +GNUNET_free (pos); @} +@end example + + +instead of +@example +next = head; while (NULL != (pos = next)) @{ next = +pos->next; if (should_free (pos)) @{ + /* unnecessary nesting! */ + GNUNET_CONTAINER_DLL_remove (head, tail, pos); GNUNET_free (pos); @} @} +@end example + + +@item We primarily use @code{for} and @code{while} loops. A @code{while} +loop is used if the method for advancing in the loop is not a +straightforward increment operation. In particular, we use: + +@example +next = head; +while (NULL != (pos = next)) +@{ + next = pos->next; + if (! should_free (pos)) + continue; + GNUNET_CONTAINER_DLL_remove (head, tail, pos); + GNUNET_free (pos); +@} +@end example + + +to free entries in a list (as the iteration changes the structure of the +list due to the free; the equivalent @code{for} loop does no longer +follow the simple @code{for} paradigm of @code{for(INIT;TEST;INC)}). +However, for loops that do follow the simple @code{for} paradigm we do +use @code{for}, even if it involves linked lists: + +@example +/* simple iteration over a linked list */ +for (pos = head; NULL != pos; pos = pos->next) +@{ + use (pos); +@} +@end example + + +@item The first argument to all higher-order functions in GNUnet must be +declared to be of type @code{void *} and is reserved for a closure. We do +not use inner functions, as trampolines would conflict with setups that +use non-executable stacks.@ The first statement in a higher-order +function, which unusually should be part of the variable declarations, +should assign the @code{cls} argument to the precise expected type. +For example: + +@example +int callback (void *cls, char *args) @{ + struct Foo *foo = cls; int other_variables; + + /* rest of function */ +@} +@end example + + +@item It is good practice to write complex @code{if} expressions instead +of using deeply nested @code{if} statements. However, except for addition +and multiplication, all operators should use parens. This is fine: + +@example +if ( (1 == foo) || ((0 == bar) && (x != y)) ) + return x; +@end example + + +However, this is not: +@example +if (1 == foo) + return x; +if (0 == bar && x != y) + return x; +@end example + + +Note that splitting the @code{if} statement above is debateable as the +@code{return x} is a very trivial statement. However, once the logic after +the branch becomes more complicated (and is still identical), the "or" +formulation should be used for sure. + +@item There should be two empty lines between the end of the function and +the comments describing the following function. There should be a single +empty line after the initial variable declarations of a function. If a +function has no local variables, there should be no initial empty line. If +a long function consists of several complex steps, those steps might be +separated by an empty line (possibly followed by a comment describing the +following step). The code should not contain empty lines in arbitrary +places; if in doubt, it is likely better to NOT have an empty line (this +way, more code will fit on the screen). +@end itemize + +@c *********************************************************************** +@node Build-system +@section Build-system + +If you have code that is likely not to compile or build rules you might +want to not trigger for most developers, use "if HAVE_EXPERIMENTAL" in +your Makefile.am. Then it is OK to (temporarily) add non-compiling (or +known-to-not-port) code. + +If you want to compile all testcases but NOT run them, run configure with +the @code{--enable-test-suppression} option. + +If you want to run all testcases, including those that take a while, run +configure with the @code{--enable-expensive-testcases} option. + +If you want to compile and run benchmarks, run configure with the +@code{--enable-benchmarks} option. + +If you want to obtain code coverage results, run configure with the +@code{--enable-coverage} option and run the coverage.sh script in +@file{contrib/}. + +@c *********************************************************************** +@node Developing extensions for GNUnet using the gnunet-ext template +@section Developing extensions for GNUnet using the gnunet-ext template + + +For developers who want to write extensions for GNUnet we provide the +gnunet-ext template to provide an easy to use skeleton. + +gnunet-ext contains the build environment and template files for the +development of GNUnet services, command line tools, APIs and tests. + +First of all you have to obtain gnunet-ext from git: + +@code{git clone https://gnunet.org/git/gnunet-ext.git} + +The next step is to bootstrap and configure it. For configure you have to +provide the path containing GNUnet with +@code{--with-gnunet=/path/to/gnunet} and the prefix where you want the +install the extension using @code{--prefix=/path/to/install}: + +@example +./bootstrap +./configure --prefix=/path/to/install --with-gnunet=/path/to/gnunet +@end example + +When your GNUnet installation is not included in the default linker search +path, you have to add @code{/path/to/gnunet} to the file +@file{/etc/ld.so.conf} and run @code{ldconfig} or your add it to the +environmental variable @code{LD_LIBRARY_PATH} by using + +@code{export LD_LIBRARY_PATH=/path/to/gnunet/lib} + +@c *********************************************************************** +@node Writing testcases +@section Writing testcases + +Ideally, any non-trivial GNUnet code should be covered by automated +testcases. Testcases should reside in the same place as the code that is +being tested. The name of source files implementing tests should begin +with "test_" followed by the name of the file that contains the code that +is being tested. + +Testcases in GNUnet should be integrated with the autotools build system. +This way, developers and anyone building binary packages will be able to +run all testcases simply by running @code{make check}. The final +testcases shipped with the distribution should output at most some brief +progress information and not display debug messages by default. The +success or failure of a testcase must be indicated by returning zero +(success) or non-zero (failure) from the main method of the testcase. The +integration with the autotools is relatively straightforward and only +requires modifications to the @code{Makefile.am} in the directory +containing the testcase. For a testcase testing the code in @code{foo.c} +the @code{Makefile.am} would contain the following lines: + +@example +check_PROGRAMS = test_foo TESTS = $(check_PROGRAMS) test_foo_SOURCES = +test_foo.c test_foo_LDADD = $(top_builddir)/src/util/libgnunetutil.la +@end example + +Naturally, other libraries used by the testcase may be specified in the +@code{LDADD} directive as necessary. + +Often testcases depend on additional input files, such as a configuration +file. These support files have to be listed using the EXTRA_DIST +directive in order to ensure that they are included in the distribution. +Example: + +@example +EXTRA_DIST = test_foo_data.conf +@end example + +Executing @code{make check} will run all testcases in the current +directory and all subdirectories. Testcases can be compiled individually +by running @code{make test_foo} and then invoked directly using +@code{./test_foo}. Note that due to the use of plugins in GNUnet, it is +typically necessary to run @code{make install} before running any +testcases. Thus the canonical command @code{make check install} has to be +changed to @code{make install check} for GNUnet. + +@c *********************************************************************** +@node GNUnet's TESTING library +@section GNUnet's TESTING library + +The TESTING library is used for writing testcases which involve starting a +single or multiple peers. While peers can also be started by testcases +using the ARM subsystem, using TESTING library provides an elegant way to +do this. The configurations of the peers are auto-generated from a given +template to have non-conflicting port numbers ensuring that peers' +services do not run into bind errors. This is achieved by testing ports' +availability by binding a listening socket to them before allocating them +to services in the generated configurations. + +An another advantage while using TESTING is that it shortens the testcase +startup time as the hostkeys for peers are copied from a pre-computed set +of hostkeys instead of generating them at peer startup which may take a +considerable amount of time when starting multiple peers or on an embedded +processor. + +TESTING also allows for certain services to be shared among peers. This +feature is invaluable when testing with multiple peers as it helps to +reduce the number of services run per each peer and hence the total +number of processes run per testcase. + +TESTING library only handles creating, starting and stopping peers. +Features useful for testcases such as connecting peers in a topology are +not available in TESTING but are available in the TESTBED subsystem. +Furthermore, TESTING only creates peers on the localhost, however by +using TESTBED testcases can benefit from creating peers across multiple +hosts. + +@menu +* API:: +* Finer control over peer stop:: +* Helper functions:: +* Testing with multiple processes:: +@end menu + +@c *********************************************************************** +@node API +@subsection API + +TESTING abstracts a group of peers as a TESTING system. All peers in a +system have common hostname and no two services of these peers have a +same port or a UNIX domain socket path. + +TESTING system can be created with the function +@code{GNUNET_TESTING_system_create()} which returns a handle to the +system. This function takes a directory path which is used for generating +the configurations of peers, an IP address from which connections to the +peers' services should be allowed, the hostname to be used in peers' +configuration, and an array of shared service specifications of type +@code{struct GNUNET_TESTING_SharedService}. + +The shared service specification must specify the name of the service to +share, the configuration pertaining to that shared service and the +maximum number of peers that are allowed to share a single instance of +the shared service. + +TESTING system created with @code{GNUNET_TESTING_system_create()} chooses +ports from the default range 12000 - 56000 while auto-generating +configurations for peers. This range can be customised with the function +@code{GNUNET_TESTING_system_create_with_portrange()}. This function is +similar to @code{GNUNET_TESTING_system_create()} except that it take 2 +additional parameters --- the start and end of the port range to use. + +A TESTING system is destroyed with the funciton +@code{GNUNET_TESTING_system_destory()}. This function takes the handle of +the system and a flag to remove the files created in the directory used +to generate configurations. + +A peer is created with the function +@code{GNUNET_TESTING_peer_configure()}. This functions takes the system +handle, a configuration template from which the configuration for the peer +is auto-generated and the index from where the hostkey for the peer has to +be copied from. When successfull, this function returs a handle to the +peer which can be used to start and stop it and to obtain the identity of +the peer. If unsuccessful, a NULL pointer is returned with an error +message. This function handles the generated configuration to have +non-conflicting ports and paths. + +Peers can be started and stopped by calling the functions +@code{GNUNET_TESTING_peer_start()} and @code{GNUNET_TESTING_peer_stop()} +respectively. A peer can be destroyed by calling the function +@code{GNUNET_TESTING_peer_destroy}. When a peer is destroyed, the ports +and paths in allocated in its configuration are reclaimed for usage in new +peers. + +@c *********************************************************************** +@node Finer control over peer stop +@subsection Finer control over peer stop + +Using @code{GNUNET_TESTING_peer_stop()} is normally fine for testcases. +However, calling this function for each peer is inefficient when trying to +shutdown multiple peers as this function sends the termination signal to +the given peer process and waits for it to terminate. It would be faster +in this case to send the termination signals to the peers first and then +wait on them. This is accomplished by the functions +@code{GNUNET_TESTING_peer_kill()} which sends a termination signal to the +peer, and the function @code{GNUNET_TESTING_peer_wait()} which waits on +the peer. + +Further finer control can be achieved by choosing to stop a peer +asynchronously with the function @code{GNUNET_TESTING_peer_stop_async()}. +This function takes a callback parameter and a closure for it in addition +to the handle to the peer to stop. The callback function is called with +the given closure when the peer is stopped. Using this function +eliminates blocking while waiting for the peer to terminate. + +An asynchronous peer stop can be cancelled by calling the function +@code{GNUNET_TESTING_peer_stop_async_cancel()}. Note that calling this +function does not prevent the peer from terminating if the termination +signal has already been sent to it. It does, however, cancels the +callback to be called when the peer is stopped. + +@c *********************************************************************** +@node Helper functions +@subsection Helper functions + +Most of the testcases can benefit from an abstraction which configures a +peer and starts it. This is provided by the function +@code{GNUNET_TESTING_peer_run()}. This function takes the testing +directory pathname, a configuration template, a callback and its closure. +This function creates a peer in the given testing directory by using the +configuration template, starts the peer and calls the given callback with +the given closure. + +The function @code{GNUNET_TESTING_peer_run()} starts the ARM service of +the peer which starts the rest of the configured services. A similar +function @code{GNUNET_TESTING_service_run} can be used to just start a +single service of a peer. In this case, the peer's ARM service is not +started; instead, only the given service is run. + +@c *********************************************************************** +@node Testing with multiple processes +@subsection Testing with multiple processes + +When testing GNUnet, the splitting of the code into a services and clients +often complicates testing. The solution to this is to have the testcase +fork @code{gnunet-service-arm}, ask it to start the required server and +daemon processes and then execute appropriate client actions (to test the +client APIs or the core module or both). If necessary, multiple ARM +services can be forked using different ports (!) to simulate a network. +However, most of the time only one ARM process is needed. Note that on +exit, the testcase should shutdown ARM with a @code{TERM} signal (to give +it the chance to cleanly stop its child processes). + +The following code illustrates spawning and killing an ARM process from a +testcase: + +@example +static void run (void *cls, char *const *args, const char +*cfgfile, const struct GNUNET_CONFIGURATION_Handle *cfg) @{ struct +GNUNET_OS_Process *arm_pid; arm_pid = GNUNET_OS_start_process (NULL, NULL, +"gnunet-service-arm", "gnunet-service-arm", "-c", cfgname, NULL); + /* do real test work here */ + if (0 != GNUNET_OS_process_kill (arm_pid, SIGTERM)) GNUNET_log_strerror + (GNUNET_ERROR_TYPE_WARNING, "kill"); GNUNET_assert (GNUNET_OK == + GNUNET_OS_process_wait (arm_pid)); GNUNET_OS_process_close (arm_pid); @} + +GNUNET_PROGRAM_run (argc, argv, "NAME-OF-TEST", "nohelp", options, &run, cls); +@end example + + +An alternative way that works well to test plugins is to implement a +mock-version of the environment that the plugin expects and then to +simply load the plugin directly. + +@c *********************************************************************** +@node Performance regression analysis with Gauger +@section Performance regression analysis with Gauger + +To help avoid performance regressions, GNUnet uses Gauger. Gauger is a +simple logging tool that allows remote hosts to send performance data to +a central server, where this data can be analyzed and visualized. Gauger +shows graphs of the repository revisions and the performace data recorded +for each revision, so sudden performance peaks or drops can be identified +and linked to a specific revision number. + +In the case of GNUnet, the buildbots log the performance data obtained +during the tests after each build. The data can be accesed on GNUnet's +Gauger page. + +The menu on the left allows to select either the results of just one +build bot (under "Hosts") or review the data from all hosts for a given +test result (under "Metrics"). In case of very different absolute value +of the results, for instance arm vs. amd64 machines, the option +"Normalize" on a metric view can help to get an idea about the +performance evolution across all hosts. + +Using Gauger in GNUnet and having the performance of a module tracked over +time is very easy. First of course, the testcase must generate some +consistent metric, which makes sense to have logged. Highly volatile or +random dependant metrics probably are not ideal candidates for meaningful +regression detection. + +To start logging any value, just include @code{gauger.h} in your testcase +code. Then, use the macro @code{GAUGER()} to make the buildbots log +whatever value is of interest for you to @code{gnunet.org}'s Gauger +server. No setup is necessary as most buildbots have already everything +in place and new metrics are created on demand. To delete a metric, you +need to contact a member of the GNUnet development team (a file will need +to be removed manually from the respective directory). + +The code in the test should look like this: + +@example +[other includes] +#include + +int main (int argc, char *argv[]) @{ + + [run test, generate data] GAUGER("YOUR_MODULE", "METRIC_NAME", (float)value, + "UNIT"); @} +@end example + + +Where: + +@table @asis + +@item @strong{YOUR_MODULE} is a category in the gauger page and should be +the name of the module or subsystem like "Core" or "DHT" +@item @strong{METRIC} is +the name of the metric being collected and should be concise and +descriptive, like "PUT operations in sqlite-datastore". +@item @strong{value} is the value +of the metric that is logged for this run. +@item @strong{UNIT} is the unit in +which the value is measured, for instance "kb/s" or "kb of RAM/node". +@end table + +If you wish to use Gauger for your own project, you can grab a copy of the +latest stable release or check out Gauger's Subversion repository. + +@c *********************************************************************** +@node GNUnet's TESTBED Subsystem +@section GNUnet's TESTBED Subsystem + +The TESTBED subsystem facilitates testing and measuring of multi-peer +deployments on a single host or over multiple hosts. + +The architecture of the testbed module is divided into the following: +@itemize @bullet + +@item Testbed API: An API which is used by the testing driver programs. It +provides with functions for creating, destroying, starting, stopping +peers, etc. + +@item Testbed service (controller): A service which is started through the +Testbed API. This service handles operations to create, destroy, start, +stop peers, connect them, modify their configurations. + +@item Testbed helper: When a controller has to be started on a host, the +testbed API starts the testbed helper on that host which in turn starts +the controller. The testbed helper receives a configuration for the +controller through its stdin and changes it to ensure the controller +doesn't run into any port conflict on that host. +@end itemize + + +The testbed service (controller) is different from the other GNUnet +services in that it is not started by ARM and is not supposed to be run +as a daemon. It is started by the testbed API through a testbed helper. +In a typical scenario involving multiple hosts, a controller is started +on each host. Controllers take up the actual task of creating peers, +starting and stopping them on the hosts they run. + +While running deployments on a single localhost the testbed API starts the +testbed helper directly as a child process. When running deployments on +remote hosts the testbed API starts Testbed Helpers on each remote host +through remote shell. By default testbed API uses SSH as a remote shell. +This can be changed by setting the environmental variable +GNUNET_TESTBED_RSH_CMD to the required remote shell program. This +variable can also contain parameters which are to be passed to the remote +shell program. For e.g: + +@example +export GNUNET_TESTBED_RSH_CMD="ssh -o BatchMode=yes \ +-o NoHostAuthenticationForLocalhost=yes %h"@ +@end example + +Substitutions are allowed int the above command string also allows for +substitions. through placemarks which begin with a `%'. At present the +following substitutions are supported + +@itemize @bullet +@item +%h: hostname +@item +%u: username +@item +%p: port +@end itemize + +Note that the substitution placemark is replaced only when the +corresponding field is available and only once. Specifying @code{%u@@%h} +doesn't work either. If you want to user username substitutions for SSH +use the argument @code{-l} before the username substitution. +Ex: @code{ssh -l %u -p %p %h} + +The testbed API and the helper communicate through the helpers stdin and +stdout. As the helper is started through a remote shell on remote hosts +any output messages from the remote shell interfere with the communication +and results in a failure while starting the helper. For this reason, it is +suggested to use flags to make the remote shells produce no output +messages and to have password-less logins. The default remote shell, SSH, +the default options are: + +@example +-o BatchMode=yes -o NoHostBasedAuthenticationForLocalhost=yes" +@end example + +Password-less logins should be ensured by using SSH keys. + +Since the testbed API executes the remote shell as a non-interactive +shell, certain scripts like .bashrc, .profiler may not be executed. If +this is the case testbed API can be forced to execute an interactive +shell by setting up the environmental variable +`GNUNET_TESTBED_RSH_CMD_SUFFIX' to a shell program. +An example could be: + +@example +export GNUNET_TESTBED_RSH_CMD_SUFFIX="sh -lc" +@end example + +The testbed API will then execute the remote shell program as: + +@example +$GNUNET_TESTBED_RSH_CMD -p $port $dest $GNUNET_TESTBED_RSH_CMD_SUFFIX \ +gnunet-helper-testbed +@end example + +On some systems, problems may arise while starting testbed helpers if +GNUnet is installed into a custom location since the helper may not be +found in the standard path. This can be addressed by setting the variable +`HELPER_BINARY_PATH' to the path of the testbed helper. Testbed API will +then use this path to start helper binaries both locally and remotely. + +Testbed API can accessed by including "gnunet_testbed_service.h" file and +linking with -lgnunettestbed. + + + +@c *********************************************************************** +@menu +* Supported Topologies:: +* Hosts file format:: +* Topology file format:: +* Testbed Barriers:: +* Automatic large-scale deployment of GNUnet in the PlanetLab testbed:: +* TESTBED Caveats:: +@end menu + +@node Supported Topologies +@subsection Supported Topologies + +While testing multi-peer deployments, it is often needed that the peers +are connected in some topology. This requirement is addressed by the +function @code{GNUNET_TESTBED_overlay_connect()} which connects any given +two peers in the testbed. + +The API also provides a helper function +@code{GNUNET_TESTBED_overlay_configure_topology()} to connect a given set +of peers in any of the following supported topologies: + +@itemize @bullet + +@item @code{GNUNET_TESTBED_TOPOLOGY_CLIQUE}: All peers are connected with +each other + +@item @code{GNUNET_TESTBED_TOPOLOGY_LINE}: Peers are connected to form a +line + +@item @code{GNUNET_TESTBED_TOPOLOGY_RING}: Peers are connected to form a +ring topology + +@item @code{GNUNET_TESTBED_TOPOLOGY_2D_TORUS}: Peers are connected to +form a 2 dimensional torus topology. The number of peers may not be a +perfect square, in that case the resulting torus may not have the uniform +poloidal and toroidal lengths + +@item @code{GNUNET_TESTBED_TOPOLOGY_ERDOS_RENYI}: Topology is generated +to form a random graph. The number of links to be present should be given + +@item @code{GNUNET_TESTBED_TOPOLOGY_SMALL_WORLD}: Peers are connected to +form a 2D Torus with some random links among them. The number of random +links are to be given + +@item @code{GNUNET_TESTBED_TOPOLOGY_SMALL_WORLD_RING}: Peers are +connected to form a ring with some random links among them. The number of +random links are to be given + +@item @code{GNUNET_TESTBED_TOPOLOGY_SCALE_FREE}: Connects peers in a +topology where peer connectivity follows power law - new peers are +connected with high probabililty to well connected peers. +@footnote{See Emergence of Scaling in Random Networks. Science 286, +509-512, 1999.} + +@item @code{GNUNET_TESTBED_TOPOLOGY_FROM_FILE}: The topology information +is loaded from a file. The path to the file has to be given. See Topology +file format for the format of this file. + +@item @code{GNUNET_TESTBED_TOPOLOGY_NONE}: No topology +@end itemize + + +The above supported topologies can be specified respectively by setting +the variable @code{OVERLAY_TOPOLOGY} to the following values in the +configuration passed to Testbed API functions +@code{GNUNET_TESTBED_test_run()} and +@code{GNUNET_TESTBED_run()}: +@itemize @bullet +@item @code{CLIQUE} +@item @code{RING} +@item @code{LINE} +@item @code{2D_TORUS} +@item @code{RANDOM} +@item @code{SMALL_WORLD} +@item @code{SMALL_WORLD_RING} +@item @code{SCALE_FREE} +@item @code{FROM_FILE} +@item @code{NONE} +@end itemize + + +Topologies @code{RANDOM}, @code{SMALL_WORLD} and @code{SMALL_WORLD_RING} +require the option @code{OVERLAY_RANDOM_LINKS} to be set to the number of +random links to be generated in the configuration. The option will be +ignored for the rest of the topologies. + +Topology @code{SCALE_FREE} requires the options +@code{SCALE_FREE_TOPOLOGY_CAP} to be set to the maximum number of peers +which can connect to a peer and @code{SCALE_FREE_TOPOLOGY_M} to be set to +how many peers a peer should be atleast connected to. + +Similarly, the topology @code{FROM_FILE} requires the option +@code{OVERLAY_TOPOLOGY_FILE} to contain the path of the file containing +the topology information. This option is ignored for the rest of the +topologies. See Topology file format for the format of this file. + +@c *********************************************************************** +@node Hosts file format +@subsection Hosts file format + +The testbed API offers the function GNUNET_TESTBED_hosts_load_from_file() +to load from a given file details about the hosts which testbed can use +for deploying peers. This function is useful to keep the data about hosts +separate instead of hard coding them in code. + +Another helper function from testbed API, GNUNET_TESTBED_run() also takes +a hosts file name as its parameter. It uses the above function to +populate the hosts data structures and start controllers to deploy peers. + +These functions require the hosts file to be of the following format: +@itemize @bullet +@item Each line is interpreted to have details about a host +@item Host details should include the username to use for logging into the +host, the hostname of the host and the port number to use for the remote +shell program. All thee values should be given. +@item These details should be given in the following format: +@code{@@:} +@end itemize + +Note that having canonical hostnames may cause problems while resolving +the IP addresses (See this bug). Hence it is advised to provide the hosts' +IP numerical addresses as hostnames whenever possible. + +@c *********************************************************************** +@node Topology file format +@subsection Topology file format + +A topology file describes how peers are to be connected. It should adhere +to the following format for testbed to parse it correctly. + +Each line should begin with the target peer id. This should be followed by +a colon(`:') and origin peer ids seperated by `|'. All spaces except for +newline characters are ignored. The API will then try to connect each +origin peer to the target peer. + +For example, the following file will result in 5 overlay connections: +[2->1], [3->1],[4->3], [0->3], [2->0]@ @code{@ 1:2|3@ 3:4| 0@ 0: 2@ } + +@c *********************************************************************** +@node Testbed Barriers +@subsection Testbed Barriers + +The testbed subsystem's barriers API facilitates coordination among the +peers run by the testbed and the experiment driver. The concept is +similar to the barrier synchronisation mechanism found in parallel +programming or multi-threading paradigms - a peer waits at a barrier upon +reaching it until the barrier is reached by a predefined number of peers. +This predefined number of peers required to cross a barrier is also called +quorum. We say a peer has reached a barrier if the peer is waiting for the +barrier to be crossed. Similarly a barrier is said to be reached if the +required quorum of peers reach the barrier. A barrier which is reached is +deemed as crossed after all the peers waiting on it are notified. + +The barriers API provides the following functions: +@itemize @bullet +@item @strong{@code{GNUNET_TESTBED_barrier_init()}:} function to +initialse a barrier in the experiment +@item @strong{@code{GNUNET_TESTBED_barrier_cancel()}:} function to cancel +a barrier which has been initialised before +@item @strong{@code{GNUNET_TESTBED_barrier_wait()}:} function to signal +barrier service that the caller has reached a barrier and is waiting for +it to be crossed +@item @strong{@code{GNUNET_TESTBED_barrier_wait_cancel()}:} function to +stop waiting for a barrier to be crossed +@end itemize + + +Among the above functions, the first two, namely +@code{GNUNET_TESTBED_barrier_init()} and +@code{GNUNET_TESTBED_barrier_cancel()} are used by experiment drivers. All +barriers should be initialised by the experiment driver by calling +@code{GNUNET_TESTBED_barrier_init()}. This function takes a name to +identify the barrier, the quorum required for the barrier to be crossed +and a notification callback for notifying the experiment driver when the +barrier is crossed. @code{GNUNET_TESTBED_barrier_cancel()} cancels an +initialised barrier and frees the resources allocated for it. This +function can be called upon a initialised barrier before it is crossed. + +The remaining two functions @code{GNUNET_TESTBED_barrier_wait()} and +@code{GNUNET_TESTBED_barrier_wait_cancel()} are used in the peer's +processes. @code{GNUNET_TESTBED_barrier_wait()} connects to the local +barrier service running on the same host the peer is running on and +registers that the caller has reached the barrier and is waiting for the +barrier to be crossed. Note that this function can only be used by peers +which are started by testbed as this function tries to access the local +barrier service which is part of the testbed controller service. Calling +@code{GNUNET_TESTBED_barrier_wait()} on an uninitialised barrier results +in failure. @code{GNUNET_TESTBED_barrier_wait_cancel()} cancels the +notification registered by @code{GNUNET_TESTBED_barrier_wait()}. + + +@c *********************************************************************** +@menu +* Implementation:: +@end menu + +@node Implementation +@subsubsection Implementation + +Since barriers involve coordination between experiment driver and peers, +the barrier service in the testbed controller is split into two +components. The first component responds to the message generated by the +barrier API used by the experiment driver (functions +@code{GNUNET_TESTBED_barrier_init()} and +@code{GNUNET_TESTBED_barrier_cancel()}) and the second component to the +messages generated by barrier API used by peers (functions +@code{GNUNET_TESTBED_barrier_wait()} and +@code{GNUNET_TESTBED_barrier_wait_cancel()}). + +Calling @code{GNUNET_TESTBED_barrier_init()} sends a +@code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_INIT} message to the master +controller. The master controller then registers a barrier and calls +@code{GNUNET_TESTBED_barrier_init()} for each its subcontrollers. In this +way barrier initialisation is propagated to the controller hierarchy. +While propagating initialisation, any errors at a subcontroller such as +timeout during further propagation are reported up the hierarchy back to +the experiment driver. + +Similar to @code{GNUNET_TESTBED_barrier_init()}, +@code{GNUNET_TESTBED_barrier_cancel()} propagates +@code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_CANCEL} message which causes +controllers to remove an initialised barrier. + +The second component is implemented as a separate service in the binary +`gnunet-service-testbed' which already has the testbed controller service. +Although this deviates from the gnunet process architecture of having one +service per binary, it is needed in this case as this component needs +access to barrier data created by the first component. This component +responds to @code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_WAIT} messages from +local peers when they call @code{GNUNET_TESTBED_barrier_wait()}. Upon +receiving @code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_WAIT} message, the +service checks if the requested barrier has been initialised before and +if it was not initialised, an error status is sent through +@code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_STATUS} message to the local +peer and the connection from the peer is terminated. If the barrier is +initialised before, the barrier's counter for reached peers is incremented +and a notification is registered to notify the peer when the barrier is +reached. The connection from the peer is left open. + +When enough peers required to attain the quorum send +@code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_WAIT} messages, the controller +sends a @code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_STATUS} message to its +parent informing that the barrier is crossed. If the controller has +started further subcontrollers, it delays this message until it receives +a similar notification from each of those subcontrollers. Finally, the +barriers API at the experiment driver receives the +@code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_STATUS} when the barrier is +reached at all the controllers. + +The barriers API at the experiment driver responds to the +@code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_STATUS} message by echoing it +back to the master controller and notifying the experiment controller +through the notification callback that a barrier has been crossed. The +echoed @code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_STATUS} message is +propagated by the master controller to the controller hierarchy. This +propagation triggers the notifications registered by peers at each of the +controllers in the hierarchy. Note the difference between this downward +propagation of the @code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_STATUS} +message from its upward propagation --- the upward propagation is needed +for ensuring that the barrier is reached by all the controllers and the +downward propagation is for triggering that the barrier is crossed. + +@c *********************************************************************** +@node Automatic large-scale deployment of GNUnet in the PlanetLab testbed +@subsection Automatic large-scale deployment of GNUnet in the PlanetLab testbed + +PlanetLab is as a testbed for computer networking and distributed systems +research. It was established in 2002 and as of June 2010 was composed of +1090 nodes at 507 sites worldwide. + +To automate the GNUnet we created a set of automation tools to simplify +the large-scale deployment. We provide you a set of scripts you can use +to deploy GNUnet on a set of nodes and manage your installation. + +Please also check @uref{https://gnunet.org/installation-fedora8-svn} and +@uref{https://gnunet.org/installation-fedora12-svn} to find detailled +instructions how to install GNUnet on a PlanetLab node. + + +@c *********************************************************************** +@menu +* PlanetLab Automation for Fedora8 nodes:: +* Install buildslave on PlanetLab nodes running fedora core 8:: +* Setup a new PlanetLab testbed using GPLMT:: +* Why do i get an ssh error when using the regex profiler?:: +@end menu + +@node PlanetLab Automation for Fedora8 nodes +@subsubsection PlanetLab Automation for Fedora8 nodes + +@c *********************************************************************** +@node Install buildslave on PlanetLab nodes running fedora core 8 +@subsubsection Install buildslave on PlanetLab nodes running fedora core 8 +@c ** Actually this is a subsubsubsection, but must be fixed differently +@c ** as subsubsection is the lowest. + +Since most of the PlanetLab nodes are running the very old fedora core 8 +image, installing the buildslave software is quite some pain. For our +PlanetLab testbed we figured out how to install the buildslave software +best. + +@c This is a vvery terrible way to suggest installing software. +@c FIXME: Is there an official, safer way instead of blind-piping a +@c script? +@c FIXME: Use newer pypi URLs below. +Install Distribute for python:@ @code{@ curl +http://python-distribute.org/distribute_setup.py | sudo python@ } + +Install Distribute for zope.interface <= 3.8.0 (4.0 and 4.0.1 will not +work): + +@example +wget https://pypi.python.org/packages/source/z/zope.interface/zope.interface-3.8.0.tar.gz +tar zvfz zope.interface-3.8.0.tar.gz@ cd zope.interface-3.8.0 +sudo python setup.py install +@end example + +Install the buildslave software (0.8.6 was the latest version): + +@example +wget http://buildbot.googlecode.com/files/buildbot-slave-0.8.6p1.tar.gz +tar xvfz buildbot-slave-0.8.6p1.tar.gz@ cd buildslave-0.8.6p1 +sudo python setup.py install +@end example + +The setup will download the matching twisted package and install it. +It will also try to install the latest version of zope.interface which +will fail to install. Buildslave will work anyway since version 3.8.0 +was installed before! + +@c *********************************************************************** +@node Setup a new PlanetLab testbed using GPLMT +@subsubsection Setup a new PlanetLab testbed using GPLMT + +@itemize @bullet +@item Get a new slice and assign nodes +Ask your PlanetLab PI to give you a new slice and assign the nodes you +need +@item Install a buildmaster +You can stick to the buildbot documentation:@ +@uref{http://buildbot.net/buildbot/docs/current/manual/installation.html} +@item Install the buildslave software on all nodes +To install the buildslave on all nodes assigned to your slice you can use +the tasklist @code{install_buildslave_fc8.xml} provided with GPLMT: + +@example +./gplmt.py -c contrib/tumple_gnunet.conf -t \ +contrib/tasklists/install_buildslave_fc8.xml -a -p +@end example + +@item Create the buildmaster configuration and the slave setup commands + +The master and the and the slaves have need to have credentials and the +master has to have all nodes configured. This can be done with the +@code{create_buildbot_configuration.py} script in the @code{scripts} +directory + +This scripts takes a list of nodes retrieved directly from PlanetLab or +read from a file and a configuration template and creates: + +@itemize @bullet +@item a tasklist which can be executed with gplmt to setup the slaves +@item a master.cfg file containing a PlanetLab nodes +@end itemize + +A configuration template is included in the , most important is +that the script replaces the following tags in the template: + +%GPLMT_BUILDER_DEFINITION :@ GPLMT_BUILDER_SUMMARY@ GPLMT_SLAVES@ +%GPLMT_SCHEDULER_BUILDERS + +Create configuration for all nodes assigned to a slice:@ @code{@ +./create_buildbot_configuration.py -u -p -s -m -t