diff options
Diffstat (limited to 'doc/documentation')
81 files changed, 21668 insertions, 0 deletions
diff --git a/doc/documentation/Makefile.am b/doc/documentation/Makefile.am new file mode 100644 index 0000000000..571999731d --- /dev/null +++ b/doc/documentation/Makefile.am @@ -0,0 +1,233 @@ +# This Makefile.am is in the public domain +docdir = $(datadir)/doc/gnunet/ + +infoimagedir = $(infodir)/images + +#DOT_FILES = images/$(wildcard *.dot) + +#DOT_VECTOR_GRAPHICS = \ +# $(DOT_FILES:%.dot=%.eps) \ +# $(DOT_FILES:%.dot=%.pdf) + +AM_MAKEINFOHTMLFLAGS = --no-split --css-ref=docstyle.css + +dist_infoimage_DATA = \ + images/gnunet-gtk-0-10-gns-a-done.png \ + images/gnunet-gtk-0-10-gns-a.png \ + images/daemon_lego_block.png \ + images/gnunet-gtk-0-10-gns.png \ + images/gnunet-0-10-peerinfo.png \ + images/gnunet-gtk-0-10-identity.png \ + images/gnunet-fs-gtk-0-10-star-tab.png \ + images/gnunet-gtk-0-10.png \ + images/gnunet-gtk-0-10-download-area.png \ + images/gnunet-gtk-0-10-search-selected.png \ + images/gnunet-gtk-0-10-fs-menu.png \ + images/gnunet-gtk-0-10-traffic.png \ + images/gnunet-gtk-0-10-fs.png \ + images/gnunet-namestore-gtk-phone.png \ + images/gnunet-gtk-0-10-fs-publish-editing.png \ + images/gnunet-namestore-gtk-vpn.png \ + images/gnunet-gtk-0-10-fs-published.png \ + images/gnunet-setup-exit.png \ + images/gnunet-gtk-0-10-fs-publish.png \ + images/iceweasel-preferences.png \ + images/gnunet-gtk-0-10-fs-publish-select.png \ + images/iceweasel-proxy.png \ + images/gnunet-gtk-0-10-fs-publish-with-file_0.png \ + images/service_lego_block.png \ + images/gnunet-gtk-0-10-fs-publish-with-file.png \ + images/service_stack.png \ + images/gnunet-gtk-0-10-fs-search.png \ + images/gnunet-tutorial-service.png \ + images/gnunet-tutorial-system.png \ + images/daemon_lego_block.svg \ + images/lego_stack.svg \ + images/service_lego_block.svg \ + images/structure.dot + +# images/$(wildcard *.png) \ +# images/$(wildcard *.svg) +# $(DOT_FILES:%.dot=%.png) + +#DOT_OPTIONS = \ +# -Gratio=.9 -Gnodesep=.005 -Granksep=.00005 \ +# -Nfontsite=9 -Nheight=.1 -Nwidth=.1 + +# .dot.png: +# $(AM_V_DOT)$(DOT) -Tpng $(DOT_OPTIONS) < "$<" > "$(srcdir)/$@.tmp"; \ +# mv "$(srcdir)/$@.tmp" "$(srcdir)/$@" + +# .dot.pdf: +# $(AM_V_DOT)$(DOT) -Tpdf $(DOT_OPTIONS) < "$<" > "$(srcdir)/$@.tmp"; \ +# mv "$(srcdir)/$@.tmp" "$(srcdir)/$@" + +# .dot.eps: +# $(AM_V_DOT)$(DOT) -Teps $(DOT_OPTIONS) < "$<" > "$(srcdir)/$@.tmp"; \ +# mv "$(srcdir)/$@.tmp" "$(srcdir)/$@" + +# .png.eps: +# $(AM_V_GEN)convert "$<" "$@-tmp.eps"; \ +# mv "$@-tmp.eps" "$@" + +# pdf-local: $(DOT_FILES=%.dot=$(top_srcdir)/%.pdf) +# info-local: $(DOT_FILES=%.dot=$(top_srcdir)/%.png) +# ps-local: $(DOT_FILES=%.dot=$(top_srcdir)/%.eps) \ +# $(top_srcdir)/%D%/images/coreutils-size-map.eps +# dvi-local: ps-local + +gnunet_tutorial_examples = \ + 001.c \ + 002.c \ + 003.c \ + 004.c \ + 005.c \ + 006.c \ + 007.c \ + 008.c \ + 009.c \ + 010.c \ + 011.c \ + 012.c \ + 013.c \ + 013.1.c \ + 014.c \ + 015.c \ + 016.c \ + 017.c \ + 018.c \ + 019.c \ + 020.c \ + 021.c \ + 022.c \ + 023.c \ + 024.c \ + 025.c \ + 026.c + +info_TEXINFOS = \ + gnunet.texi \ + gnunet-c-tutorial.texi + +gnunet_TEXINFOS = \ + chapters/developer.texi \ + chapters/installation.texi \ + chapters/philosophy.texi \ + chapters/user.texi \ + chapters/vocabulary.texi \ + chapters/configuration.texi \ + chapters/contributing.texi \ + fdl-1.3.texi \ + gpl-3.0.texi + +EXTRA_DIST = \ + $(gnunet_TEXINFOS) \ + $(gnunet_tutorial_examples) \ + htmlxref.cnf \ + run-gendocs.sh \ + docstyle.css + + +# $(DOT_FILES) \ +# $(DOT_VECTOR_GRAPHICS) + +DISTCLEANFILES = \ + gnunet.cps \ + gnunet-c-tutorial.cps \ + chapters/developer.cps \ + chapters/installation.cps \ + chapter/philosophy.cps \ + chapters/user.cps \ + chapters/configuration.cps \ + chapters/vocabulary.cps \ + fdl-1.3.cps \ + gpl-3.0.cps + +# if HAVE_EXTENDED_DOCUMENTATION_BUILDING +daemon_lego_block.png: images/daemon_lego_block.svg + convert images/daemon_lego_block.svg images/daemon_lego_block.png && + pngcrush images/daemon_lego_block.png images/daemon_lego_block.png + +service_lego_block.png: images/service_lego_block.svg + convert images/service_lego_block.svg images/service_lego_block.png && + pngcrush images/service_lego_block.png images/serivce_lego_block.png + +lego_stack.png: images/lego_stack.svg + convert images/lego_stack.svg images/lego_stack.png && + pngcrush images/lego_stack.png images/lego_stack.png + +# FIXME: The usage of 'date' strings causes a warning. +# version.texi: +# echo "@set UPDATED $(date +'%d %B %Y')" > $@ +# echo "@set UPDATED-MONTH $(date +'%B %Y')" >> $@ +# echo "@set EDITION $(PACKAGE_VERSION)" >> $@ +# echo "@set VERSION $(PACKAGE_VERSION)" >> $@ + +# Workaround for makeinfo error. Whcih in turn introduces more +# date-related 'warnings'. Well. +version2.texi: + echo "@set UPDATED $(date +'%d %B %Y')" > $@ + echo "@set UPDATED-MONTH $(date +'%B %Y')" >> $@ + echo "@set EDITION $(PACKAGE_VERSION)" >> $@ + echo "@set VERSION $(PACKAGE_VERSION)" >> $@ + +# FIXME: rm *.html and *.pdf +#doc-clean: +# @rm *.aux *.log *.toc *.cp *.cps + +doc-all-install: + @mkdir -p $(DESTDIR)/$(docdir) + @mkdir -p $(DESTDIR)/$(infoimagedir) + @mkdir -p $(DESTDIR)/$(infodir) + @install -m 0755 gnunet.pdf $(DESTDIR)/$(docdir) + @install -m 0755 gnunet-c-tutorial.pdf $(DESTDIR)/$(docdir) + @install -m 0755 gnunet-c-tutorial.info $(DESTDIR)/$(infodir) + @install -m 0755 gnunet.info $(DESTDIR)/$(infodir) + @install gnunet.html $(DESTDIR)/$(docdir) + @install gnunet-c-tutorial.html $(DESTDIR)/$(docdir) + +doc-gendoc-install: + @mkdir -p $(DESTDIR)/$(docdir) + @cp -r manual $(DESTDIR)/$(docdir) + +# @cp -r images $(DESTDIR)/$(infoimagedir) + +dev-build: version.texi version2.texi + @makeinfo --pdf gnunet.texi + @makeinfo --pdf gnunet-c-tutorial.texi + @makeinfo --html gnunet.texi + @makeinfo --html gnunet-c-tutorial.texi + @makeinfo --no-split gnunet.texi + @makeinfo --no-split gnunet-c-tutorial.texi + +# TODO: Add more to clean. +clean: + @rm gnunet.pdf + @rm gnunet.html + @rm gnunet.info + @rm gnunet.info-1 + @rm gnunet.info-2 + @rm gnunet.info-3 + @rm gnunet-c-tutorial.pdf + @rm gnunet-c-tutorial.info + @rm gnunet-c-tutorial.html + @rm -r gnunet.t2p + @rm -r gnunet-c-tutorial.t2p + @rm -r manual + +# CLEANFILES = \ +# gnunet.log \ +# gnunet-c-tutorial.log \ +# $(wildcard *.aux) \ +# $(wildcard *.toc) \ +# $(wildcard *.cp) \ +# $(wildcard *.cps) + +#.PHONY: version.texi +# if HAVE_EXTENDED_DOCUMENTATION_BUILDING_PDF + +# if HAVE_EXTENDED_DOCUMENTATION_BUILDING_HTML + +# endif +# endif +# endif diff --git a/doc/documentation/README.txt b/doc/documentation/README.txt new file mode 100644 index 0000000000..b2a7815580 --- /dev/null +++ b/doc/documentation/README.txt @@ -0,0 +1,99 @@ +To be moved to an appropriate section of "how to write documentation" or +"how to contribute to the documentation": + +1. When writing documentation, please use gender-neutral wording when + referring to people, such as singular “they”, “their”, “them”, and + so forth. -> https://en.wikipedia.org/wiki/Singular_they + +2. Keep line length below 74 characters. + - Expection by texi2pdf output so far: URLs will break + (inserted whitespace) when they contain linebreaks + within the @url{} / @uref{}. + +3. Do not use tab characters (see chapter 2.1 texinfo manual) + +4. Use neutral language and third person perspective in the text + +4.1 So, when you refer to a user in general or addressing the user, + refer to (1). +4.1.1 Unsolved exceptions for canonical reasons: + When refering to Alice, use "she". + When refering to Bob, use "he". + These are long established examples and they + should either be replaced (avoid Alice and Bob + examples when you can) or followed. + +5. Use 2 spaces between sentences, so instead of: + + We do this and the other thing. This is done by foo. + + Write: + + We do this and the other thing. This is done by foo. + +6. Use @footnote{} instead of putting an @*ref{} to the footnote on a + collected footnote-page. + In a 200+ pages handbook it's better to have footnotes accessible + without having to skip over to the end. + +6.1 Avoid unnecessary footnotes, keep the text self-explanatory and + in a simple language where possible/necessary. + +* Completion Levels: + +** chapters/philosophy: around 100% fixed after initial export. + +* What's left to do + +- Which Texlive modules are needed? Decrease the size. +- Update the content of gnunet documentation. + +* How to use (hack) on this + +** with guix + +Adjust accordingly, ie read the Guix Documentation: +setenv GUIX_PACKAGE_PATH "gnunet/contrib/packages/guix/packages" +guix environment gnunet-doc +and +guix build -f contrib/packages/guix/gnunet-doc.scm + +** without guix + +You need to have Texinfo and Texlive in your path. +sh bootstrap +./configure --enable-documentation +cd doc +make (format you want) + +for example: make html, make info, make pdf + +* structure (relations) + +** gnunet.texi + -> chapters/developer.texi + -> chapters/installation.texi + -> chapters/philosophy.texi + -> chapters/user.texi + -> chapters/vocabulary.texi + -> images/* + -> gpl-3.0.texi + -> fdl-1.3.texi + +** gnunet-c-tutorial.texi + -> figs/Service.pdf + -> figs/System.pdf + -> tutorial-examples/*.c + -> gpl-3.0.texi + -> fdl-1.3.texi + +- gnunet-c-tutorial-v1.pdf: original LaTeX "gnunet-c-tutorial.pdf". +- man folder: the man pages. +- doxygen folder +- outdated-and-old-installation-instructions.txt: self described within the file. + + +Use `gendocs', add to the manual/ directory of the web site. + + $ cd doc + $ gendocs.sh gnunet "GNUnet 0.10.X Reference Manual" diff --git a/doc/documentation/chapters/configuration.texi b/doc/documentation/chapters/configuration.texi new file mode 100644 index 0000000000..286c72e7ab --- /dev/null +++ b/doc/documentation/chapters/configuration.texi @@ -0,0 +1,5 @@ +@node Configuration Handbook +@chapter Configuration Handbook + +This chapter has yet to be written. It is intended to be about in-depth +configuration of GNUnet. diff --git a/doc/documentation/chapters/contributing.texi b/doc/documentation/chapters/contributing.texi new file mode 100644 index 0000000000..5844fb497f --- /dev/null +++ b/doc/documentation/chapters/contributing.texi @@ -0,0 +1,102 @@ +@node GNUnet Contributors Handbook +@chapter GNUnet Contributors Handbook + +@menu +* Contributing to GNUnet:: +* Licenses of contributions:: +* Copyright Assignment:: +* Contributing to the Reference Manual:: +@end menu + +@node Contributing to GNUnet +@section Contributing to GNUnet + +@node Licenses of contributions +@section Licenses of contributions + +GNUnet is a @uref{https://www.gnu.org/, GNU} package. +All code contributions must thus be put under the +@uref{https://www.gnu.org/copyleft/gpl.html, GNU Public License (GPL)}. +All documentation should be put under FSF approved licenses +(see @uref{https://www.gnu.org/copyleft/fdl.html, fdl}). + +By submitting documentation, translations, and other content to GNUnet +you automatically grant the right to publish code under the +GNU Public License and documentation under either or both the +GNU Public License or the GNU Free Documentation License. +When contributing to the GNUnet project, GNU standards and the +@uref{https://www.gnu.org/philosophy/philosophy.html, GNU philosophy} +should be adhered to. + +@cindex copyright assignment +@node Copyright Assignment +@section Copyright Assignment +We require a formal copyright assignment for GNUnet contributors +to GNUnet e.V.; nevertheless, we do allow pseudonymous contributions. +By signing the copyright agreement and submitting your code (or +documentation) to us, you agree to share the rights to your code +with GNUnet e.V.; GNUnet e.V. receives non-exclusive ownership +rights, and in particular is allowed to dual-license the code. You +retain non-exclusive rights to your contributions, so you can also +share your contributions freely with other projects. + +GNUnet e.V. will publish all accepted contributions under the GPLv3 +or any later version. The association may decide to publish +contributions under additional licenses (dual-licensing). + +We do not intentionally remove your name from your contributions; +however, due to extensive editing it is not always trivial to +attribute contributors properly. If you find that you significantly +contributed to a file (or the project as a whole) and are not listed +in the respective authors file or section, please do let us know. + +@node Contributing to the Reference Manual +@section Contributing to the Reference Manual + +@itemize @bullet + +@item When writing documentation, please use +@uref{https://en.wikipedia.org/wiki/Singular_they, gender-neutral wording} +when referring to people, such as singular “they”, “their”, “them”, and so +forth. + +@item Keep line length below 74 characters, except for URLs. +URLs break in the PDF output when they contain linebreaks. + +@item Do not use tab characters (see chapter 2.1 texinfo manual) + +@item Use neutral language and third person perspective in the text + +@item So, when you refer to a user in general or addressing the user, +refer to (1). +@itemize @bullet +@item Unsolved exceptions for canonical reasons: When refering to Alice, +use "she". When refering to Bob, use "he". These are long established +examples and they should either be replaced (avoid Alice and Bob +examples when you can) or followed. +@end itemize + +@c FIXME: This is questionable, it feels like bike shed painging to do +@c this for several k lines. It only helps to jump between sentences in +@c editors afaik. +@c @item Use 2 spaces between sentences, so instead of: + +@c @example +@c We do this and the other thing. This is done by foo. +@c @end example + +@c Write: + +@c @example +@c We do this and the other thing. This is done by foo. +@c @end example + +@item Use @@footnote@{@} instead of putting an @@*ref@{@} to the +footnote on a collected footnote-page. +In a 200+ pages handbook it's better to have footnotes accessible +without having to skip over to the end. + +@item Avoid unnecessary footnotes, keep the text self-explanatory and +in a simple language where possible/necessary. + +@end itemize diff --git a/doc/documentation/chapters/developer.texi b/doc/documentation/chapters/developer.texi new file mode 100644 index 0000000000..70fd7c7ebe --- /dev/null +++ b/doc/documentation/chapters/developer.texi @@ -0,0 +1,8341 @@ +@c *********************************************************************** +@node GNUnet Developer Handbook +@chapter GNUnet Developer Handbook + +This book is intended to be an introduction for programmers that want to +extend the GNUnet framework. GNUnet is more than a simple peer-to-peer +application. + +For developers, GNUnet is: + +@itemize @bullet +@item developed by a community that believes in the GNU philosophy +@item Free Software (Free as in Freedom), licensed under the +GNU General Public License +@item A set of standards, including coding conventions and +architectural rules +@item A set of layered protocols, both specifying the communication +between peers as well as the communication between components +of a single peer +@item A set of libraries with well-defined APIs suitable for +writing extensions +@end itemize + +In particular, the architecture specifies that a peer consists of many +processes communicating via protocols. Processes can be written in almost +any language. +C and Java @footnote{As well as Guile} APIs exist for accessing existing +services and for writing extensions. +It is possible to write extensions in other languages by +implementing the necessary IPC protocols. + +GNUnet can be extended and improved along many possible dimensions, and +anyone interested in Free Software and Freedom-enhancing Networking is +welcome to join the effort. This Developer Handbook attempts to provide +an initial introduction to some of the key design choices and central +components of the system. +This part of the GNUNet documentation is far from complete, +and we welcome informed contributions, be it in the form of +new chapters, sections or insightful comments. + +@menu +* Developer Introduction:: +* Code overview:: +* System Architecture:: +* Subsystem stability:: +* Naming conventions and coding style guide:: +* Build-system:: +* Developing extensions for GNUnet using the gnunet-ext template:: +* Writing testcases:: +* TESTING library:: +* Performance regression analysis with Gauger:: +* TESTBED Subsystem:: +* libgnunetutil:: +* Automatic Restart Manager (ARM):: +* TRANSPORT Subsystem:: +* NAT library:: +* Distance-Vector plugin:: +* SMTP plugin:: +* Bluetooth plugin:: +* WLAN plugin:: +* ATS Subsystem:: +* CORE Subsystem:: +* CADET Subsystem:: +* NSE Subsystem:: +* HOSTLIST Subsystem:: +* IDENTITY Subsystem:: +* NAMESTORE Subsystem:: +* PEERINFO Subsystem:: +* PEERSTORE Subsystem:: +* SET Subsystem:: +* STATISTICS Subsystem:: +* Distributed Hash Table (DHT):: +* GNU Name System (GNS):: +* GNS Namecache:: +* REVOCATION Subsystem:: +* File-sharing (FS) Subsystem:: +* REGEX Subsystem:: +@end menu + +@node Developer Introduction +@section Developer Introduction + +This Developer Handbook is intended as first introduction to GNUnet for +new developers that want to extend the GNUnet framework. After the +introduction, each of the GNUnet subsystems (directories in the +@file{src/} tree) is (supposed to be) covered in its own chapter. In +addition to this documentation, GNUnet developers should be aware of the +services available on the GNUnet server to them. + +New developers can have a look a the GNUnet tutorials for C and java +available in the @file{src/} directory of the repository or under the +following links: + +@c ** FIXME: Link to files in source, not online. +@c ** FIXME: Where is the Java tutorial? +@itemize @bullet +@item @uref{https://gnunet.org/git/gnunet.git/plain/doc/gnunet-c-tutorial.pdf, GNUnet C tutorial} +@item GNUnet Java tutorial +@end itemize + +In addition to the GNUnet Reference Documentation you are reading, +the GNUnet server at @uref{https://gnunet.org} contains +various resources for GNUnet developers and those +who aspire to become regular contributors. +They are all conveniently reachable via the "Developer" +entry in the navigation menu. Some additional tools (such as static +analysis reports) require a special developer access to perform certain +operations. If you want (or require) access, you should contact +@uref{http://grothoff.org/christian/, Christian Grothoff}, +GNUnet's maintainer. + +The public subsystems on the GNUnet server that help developers are: + +@itemize @bullet + +@item The version control system (git) keeps our code and enables +distributed development. +It is pubclicly accessible at @uref{https://gnunet.org/git/}. +Only developers with write access can commit code, everyone else is +encouraged to submit patches to the +@uref{https://lists.gnu.org/mailman/listinfo/gnunet-developers, GNUnet-developers mailinglist}. + +@item The bugtracking system (Mantis). +We use it to track feature requests, open bug reports and their +resolutions. +It can be accessed at @uref{https://gnunet.org/bugs/}. +Anyone can report bugs, but only developers can claim to have fixed them. + +@item Our site installation of the +CI@footnote{Continuous Integration} system @code{Buildbot} is used +to check GNUnet builds automatically on a range of platforms. +The web interface of this CI is exposed at +@uref{https://gnunet.org/buildbot/}. +Builds are triggered automatically 30 minutes after the last commit to +our repository was made. + +@item The current quality of our automated test suite is assessed using +Code coverage analysis. This analysis is run daily; however the webpage +is only updated if all automated tests pass at that time. Testcases that +improve our code coverage are always welcome. + +@item We try to automatically find bugs using a static analysis scan. +This scan is run daily; however the webpage is only updated if all +automated tests pass at the time. Note that not everything that is +flagged by the analysis is a bug, sometimes even good code can be marked +as possibly problematic. Nevertheless, developers are encouraged to at +least be aware of all issues in their code that are listed. + +@item We use Gauger for automatic performance regression visualization. +Details on how to use Gauger are here. + +@item We use @uref{http://junit.org/, junit} to automatically test +@command{gnunet-java}. +Automatically generated, current reports on the test suite are here. + +@item We use Cobertura to generate test coverage reports for gnunet-java. +Current reports on test coverage are here. + +@end itemize + + + +@c *********************************************************************** +@menu +* Project overview:: +@end menu + +@node Project overview +@subsection Project overview + +The GNUnet project consists at this point of several sub-projects. This +section is supposed to give an initial overview about the various +sub-projects. Note that this description also lists projects that are far +from complete, including even those that have literally not a single line +of code in them yet. + +GNUnet sub-projects in order of likely relevance are currently: + +@table @asis + +@item @command{gnunet} +Core of the P2P framework, including file-sharing, VPN and +chat applications; this is what the Developer Handbook covers mostly +@item @command{gnunet-gtk} +Gtk+-based user interfaces, including: + +@itemize @bullet +@item @command{gnunet-fs-gtk} (file-sharing), +@item @command{gnunet-statistics-gtk} (statistics over time), +@item @command{gnunet-peerinfo-gtk} +(information about current connections and known peers), +@item @command{gnunet-chat-gtk} (chat GUI) and +@item @command{gnunet-setup} (setup tool for "everything") +@end itemize + +@item @command{gnunet-fuse} +Mounting directories shared via GNUnet's file-sharing +on GNU/Linux distributions +@item @command{gnunet-update} +Installation and update tool +@item @command{gnunet-ext} +Template for starting 'external' GNUnet projects +@item @command{gnunet-java} +Java APIs for writing GNUnet services and applications +@c ** FIXME: Point to new website repository once we have it: +@c ** @item svn/gnunet-www/ Code and media helping drive the GNUnet +@c website +@item @command{eclectic} +Code to run GNUnet nodes on testbeds for research, development, +testing and evaluation +@c ** FIXME: Solve the status and location of gnunet-qt +@item @command{gnunet-qt} +Qt-based GNUnet GUI (is it depreacated?) +@item @command{gnunet-cocoa} +cocoa-based GNUnet GUI (is it depreacated?) +@item @command{gnunet-guile} + +@end table + +We are also working on various supporting libraries and tools: +@c ** FIXME: What about gauger, and what about libmwmodem? + +@table @asis +@item @command{libextractor} +GNU libextractor (meta data extraction) +@item @command{libmicrohttpd} +GNU libmicrohttpd (embedded HTTP(S) server library) +@item @command{gauger} +Tool for performance regression analysis +@item @command{monkey} +Tool for automated debugging of distributed systems +@item @command{libmwmodem} +Library for accessing satellite connection quality +reports +@item @command{libgnurl} +gnURL (feature-restricted variant of cURL/libcurl) +@end table + +Finally, there are various external projects (see links for a list of +those that have a public website) which build on top of the GNUnet +framework. + +@c *********************************************************************** +@node Code overview +@section Code overview + +This section gives a brief overview of the GNUnet source code. +Specifically, we sketch the function of each of the subdirectories in +the @file{gnunet/src/} directory. The order given is roughly bottom-up +(in terms of the layers of the system). + +@table @asis +@item @file{util/} --- libgnunetutil +Library with general utility functions, all +GNUnet binaries link against this library. Anything from memory +allocation and data structures to cryptography and inter-process +communication. The goal is to provide an OS-independent interface and +more 'secure' or convenient implementations of commonly used primitives. +The API is spread over more than a dozen headers, developers should study +those closely to avoid duplicating existing functions. +@pxref{libgnunetutil}. +@item @file{hello/} --- libgnunethello +HELLO messages are used to +describe under which addresses a peer can be reached (for example, +protocol, IP, port). This library manages parsing and generating of HELLO +messages. +@item @file{block/} --- libgnunetblock +The DHT and other components of GNUnet +store information in units called 'blocks'. Each block has a type and the +type defines a particular format and how that binary format is to be +linked to a hash code (the key for the DHT and for databases). The block +library is a wapper around block plugins which provide the necessary +functions for each block type. +@item @file{statistics/} --- statistics service +The statistics service enables associating +values (of type uint64_t) with a componenet name and a string. The main +uses is debugging (counting events), performance tracking and user +entertainment (what did my peer do today?). +@item @file{arm/} --- Automatic Restart Manager (ARM) +The automatic-restart-manager (ARM) service +is the GNUnet master service. Its role is to start gnunet-services, to +re-start them when they crashed and finally to shut down the system when +requested. +@item @file{peerinfo/} --- peerinfo service +The peerinfo service keeps track of which peers are known +to the local peer and also tracks the validated addresses for each peer +(in the form of a HELLO message) for each of those peers. The peer is not +necessarily connected to all peers known to the peerinfo service. +Peerinfo provides persistent storage for peer identities --- peers are +not forgotten just because of a system restart. +@item @file{datacache/} --- libgnunetdatacache +The datacache library provides (temporary) block storage for the DHT. +Existing plugins can store blocks in Sqlite, Postgres or MySQL databases. +All data stored in the cache is lost when the peer is stopped or +restarted (datacache uses temporary tables). +@item @file{datastore/} --- datastore service +The datastore service stores file-sharing blocks in +databases for extended periods of time. In contrast to the datacache, data +is not lost when peers restart. However, quota restrictions may still +cause old, expired or low-priority data to be eventually discarded. +Existing plugins can store blocks in Sqlite, Postgres or MySQL databases. +@item @file{template/} --- service template +Template for writing a new service. Does nothing. +@item @file{ats/} --- Automatic Transport Selection +The automatic transport selection (ATS) service +is responsible for deciding which address (i.e. +which transport plugin) should be used for communication with other peers, +and at what bandwidth. +@item @file{nat/} --- libgnunetnat +Library that provides basic functions for NAT traversal. +The library supports NAT traversal with +manual hole-punching by the user, UPnP and ICMP-based autonomous NAT +traversal. The library also includes an API for testing if the current +configuration works and the @code{gnunet-nat-server} which provides an +external service to test the local configuration. +@item @file{fragmentation/} --- libgnunetfragmentation +Some transports (UDP and WLAN, mostly) have restrictions on the maximum +transfer unit (MTU) for packets. The fragmentation library can be used to +break larger packets into chunks of at most 1k and transmit the resulting +fragments reliabily (with acknowledgement, retransmission, timeouts, +etc.). +@item @file{transport/} --- transport service +The transport service is responsible for managing the +basic P2P communication. It uses plugins to support P2P communication +over TCP, UDP, HTTP, HTTPS and other protocols.The transport service +validates peer addresses, enforces bandwidth restrictions, limits the +total number of connections and enforces connectivity restrictions (i.e. +friends-only). +@item @file{peerinfo-tool/} --- gnunet-peerinfo +This directory contains the gnunet-peerinfo binary which can be used to +inspect the peers and HELLOs known to the peerinfo service. +@item @file{core/} +The core service is responsible for establishing encrypted, authenticated +connections with other peers, encrypting and decrypting messages and +forwarding messages to higher-level services that are interested in them. +@item @file{testing/} --- libgnunettesting +The testing library allows starting (and stopping) peers +for writing testcases. +It also supports automatic generation of configurations for peers +ensuring that the ports and paths are disjoint. libgnunettesting is also +the foundation for the testbed service +@item @file{testbed/} --- testbed service +The testbed service is used for creating small or large scale deployments +of GNUnet peers for evaluation of protocols. +It facilitates peer depolyments on multiple +hosts (for example, in a cluster) and establishing varous network +topologies (both underlay and overlay). +@item @file{nse/} --- Network Size Estimation +The network size estimation (NSE) service +implements a protocol for (securely) estimating the current size of the +P2P network. +@item @file{dht/} --- distributed hash table +The distributed hash table (DHT) service provides a +distributed implementation of a hash table to store blocks under hash +keys in the P2P network. +@item @file{hostlist/} --- hostlist service +The hostlist service allows learning about +other peers in the network by downloading HELLO messages from an HTTP +server, can be configured to run such an HTTP server and also implements +a P2P protocol to advertise and automatically learn about other peers +that offer a public hostlist server. +@item @file{topology/} --- topology service +The topology service is responsible for +maintaining the mesh topology. It tries to maintain connections to friends +(depending on the configuration) and also tries to ensure that the peer +has a decent number of active connections at all times. If necessary, new +connections are added. All peers should run the topology service, +otherwise they may end up not being connected to any other peer (unless +some other service ensures that core establishes the required +connections). The topology service also tells the transport service which +connections are permitted (for friend-to-friend networking) +@item @file{fs/} --- file-sharing +The file-sharing (FS) service implements GNUnet's +file-sharing application. Both anonymous file-sharing (using gap) and +non-anonymous file-sharing (using dht) are supported. +@item @file{cadet/} --- cadet service +The CADET service provides a general-purpose routing abstraction to create +end-to-end encrypted tunnels in mesh networks. We wrote a paper +documenting key aspects of the design. +@item @file{tun/} --- libgnunettun +Library for building IPv4, IPv6 packets and creating +checksums for UDP, TCP and ICMP packets. The header +defines C structs for common Internet packet formats and in particular +structs for interacting with TUN (virtual network) interfaces. +@item @file{mysql/} --- libgnunetmysql +Library for creating and executing prepared MySQL +statements and to manage the connection to the MySQL database. +Essentially a lightweight wrapper for the interaction between GNUnet +components and libmysqlclient. +@item @file{dns/} +Service that allows intercepting and modifying DNS requests of +the local machine. Currently used for IPv4-IPv6 protocol translation +(DNS-ALG) as implemented by "pt/" and for the GNUnet naming system. The +service can also be configured to offer an exit service for DNS traffic. +@item @file{vpn/} --- VPN service +The virtual public network (VPN) service provides a virtual +tunnel interface (VTUN) for IP routing over GNUnet. +Needs some other peers to run an "exit" service to work. +Can be activated using the "gnunet-vpn" tool or integrated with DNS using +the "pt" daemon. +@item @file{exit/} +Daemon to allow traffic from the VPN to exit this +peer to the Internet or to specific IP-based services of the local peer. +Currently, an exit service can only be restricted to IPv4 or IPv6, not to +specific ports and or IP address ranges. If this is not acceptable, +additional firewall rules must be added manually. exit currently only +works for normal UDP, TCP and ICMP traffic; DNS queries need to leave the +system via a DNS service. +@item @file{pt/} +protocol translation daemon. This daemon enables 4-to-6, +6-to-4, 4-over-6 or 6-over-4 transitions for the local system. It +essentially uses "DNS" to intercept DNS replies and then maps results to +those offered by the VPN, which then sends them using mesh to some daemon +offering an appropriate exit service. +@item @file{identity/} +Management of egos (alter egos) of a user; identities are +essentially named ECC private keys and used for zones in the GNU name +system and for namespaces in file-sharing, but might find other uses later +@item @file{revocation/} +Key revocation service, can be used to revoke the +private key of an identity if it has been compromised +@item @file{namecache/} +Cache for resolution results for the GNU name system; +data is encrypted and can be shared among users, +loss of the data should ideally only result in a +performance degradation (persistence not required) +@item @file{namestore/} +Database for the GNU name system with per-user private information, +persistence required +@item @file{gns/} +GNU name system, a GNU approach to DNS and PKI. +@item @file{dv/} +A plugin for distance-vector (DV)-based routing. +DV consists of a service and a transport plugin to provide peers +with the illusion of a direct P2P connection for connections +that use multiple (typically up to 3) hops in the actual underlay network. +@item @file{regex/} +Service for the (distributed) evaluation of regular expressions. +@item @file{scalarproduct/} +The scalar product service offers an API to perform a secure multiparty +computation which calculates a scalar product between two peers +without exposing the private input vectors of the peers to each other. +@item @file{consensus/} +The consensus service will allow a set of peers to agree +on a set of values via a distributed set union computation. +@item @file{rest/} +The rest API allows access to GNUnet services using RESTful interaction. +The services provide plugins that can exposed by the rest server. +@item @file{experimentation/} +The experimentation daemon coordinates distributed +experimentation to evaluate transport and ATS properties. +@end table + +@c *********************************************************************** +@node System Architecture +@section System Architecture + +GNUnet developers like LEGOs. The blocks are indestructible, can be +stacked together to construct complex buildings and it is generally easy +to swap one block for a different one that has the same shape. GNUnet's +architecture is based on LEGOs: + +@c images here + +This chapter documents the GNUnet LEGO system, also known as GNUnet's +system architecture. + +The most common GNUnet component is a service. Services offer an API (or +several, depending on what you count as "an API") which is implemented as +a library. The library communicates with the main process of the service +using a service-specific network protocol. The main process of the service +typically doesn't fully provide everything that is needed --- it has holes +to be filled by APIs to other services. + +A special kind of component in GNUnet are user interfaces and daemons. +Like services, they have holes to be filled by APIs of other services. +Unlike services, daemons do not implement their own network protocol and +they have no API: + +The GNUnet system provides a range of services, daemons and user +interfaces, which are then combined into a layered GNUnet instance (also +known as a peer). + +Note that while it is generally possible to swap one service for another +compatible service, there is often only one implementation. However, +during development we often have a "new" version of a service in parallel +with an "old" version. While the "new" version is not working, developers +working on other parts of the service can continue their development by +simply using the "old" service. Alternative design ideas can also be +easily investigated by swapping out individual components. This is +typically achieved by simply changing the name of the "BINARY" in the +respective configuration section. + +Key properties of GNUnet services are that they must be separate +processes and that they must protect themselves by applying tight error +checking against the network protocol they implement (thereby achieving a +certain degree of robustness). + +On the other hand, the APIs are implemented to tolerate failures of the +service, isolating their host process from errors by the service. If the +service process crashes, other services and daemons around it should not +also fail, but instead wait for the service process to be restarted by +ARM. + + +@c *********************************************************************** +@node Subsystem stability +@section Subsystem stability + +This section documents the current stability of the various GNUnet +subsystems. Stability here describes the expected degree of compatibility +with future versions of GNUnet. For each subsystem we distinguish between +compatibility on the P2P network level (communication protocol between +peers), the IPC level (communication between the service and the service +library) and the API level (stability of the API). P2P compatibility is +relevant in terms of which applications are likely going to be able to +communicate with future versions of the network. IPC communication is +relevant for the implementation of language bindings that re-implement the +IPC messages. Finally, API compatibility is relevant to developers that +hope to be able to avoid changes to applications build on top of the APIs +of the framework. + +The following table summarizes our current view of the stability of the +respective protocols or APIs: + +@multitable @columnfractions .20 .20 .20 .20 +@headitem Subsystem @tab P2P @tab IPC @tab C API +@item util @tab n/a @tab n/a @tab stable +@item arm @tab n/a @tab stable @tab stable +@item ats @tab n/a @tab unstable @tab testing +@item block @tab n/a @tab n/a @tab stable +@item cadet @tab testing @tab testing @tab testing +@item consensus @tab experimental @tab experimental @tab experimental +@item core @tab stable @tab stable @tab stable +@item datacache @tab n/a @tab n/a @tab stable +@item datastore @tab n/a @tab stable @tab stable +@item dht @tab stable @tab stable @tab stable +@item dns @tab stable @tab stable @tab stable +@item dv @tab testing @tab testing @tab n/a +@item exit @tab testing @tab n/a @tab n/a +@item fragmentation @tab stable @tab n/a @tab stable +@item fs @tab stable @tab stable @tab stable +@item gns @tab stable @tab stable @tab stable +@item hello @tab n/a @tab n/a @tab testing +@item hostlist @tab stable @tab stable @tab n/a +@item identity @tab stable @tab stable @tab n/a +@item multicast @tab experimental @tab experimental @tab experimental +@item mysql @tab stable @tab n/a @tab stable +@item namestore @tab n/a @tab stable @tab stable +@item nat @tab n/a @tab n/a @tab stable +@item nse @tab stable @tab stable @tab stable +@item peerinfo @tab n/a @tab stable @tab stable +@item psyc @tab experimental @tab experimental @tab experimental +@item pt @tab n/a @tab n/a @tab n/a +@item regex @tab stable @tab stable @tab stable +@item revocation @tab stable @tab stable @tab stable +@item social @tab experimental @tab experimental @tab experimental +@item statistics @tab n/a @tab stable @tab stable +@item testbed @tab n/a @tab testing @tab testing +@item testing @tab n/a @tab n/a @tab testing +@item topology @tab n/a @tab n/a @tab n/a +@item transport @tab stable @tab stable @tab stable +@item tun @tab n/a @tab n/a @tab stable +@item vpn @tab testing @tab n/a @tab n/a +@end multitable + +Here is a rough explanation of the values: + +@table @samp +@item stable +No incompatible changes are planned at this time; for IPC/APIs, if +there are incompatible changes, they will be minor and might only require +minimal changes to existing code; for P2P, changes will be avoided if at +all possible for the 0.10.x-series + +@item testing +No incompatible changes are +planned at this time, but the code is still known to be in flux; so while +we have no concrete plans, our expectation is that there will still be +minor modifications; for P2P, changes will likely be extensions that +should not break existing code + +@item unstable +Changes are planned and will happen; however, they +will not be totally radical and the result should still resemble what is +there now; nevertheless, anticipated changes will break protocol/API +compatibility + +@item experimental +Changes are planned and the result may look nothing like +what the API/protocol looks like today + +@item unknown +Someone should think about where this subsystem headed + +@item n/a +This subsystem does not have an API/IPC-protocol/P2P-protocol +@end table + +@c *********************************************************************** +@node Naming conventions and coding style guide +@section Naming conventions and coding style guide + +Here you can find some rules to help you write code for GNUnet. + +@c *********************************************************************** +@menu +* Naming conventions:: +* Coding style:: +@end menu + +@node Naming conventions +@subsection Naming conventions + + +@c *********************************************************************** +@menu +* include files:: +* binaries:: +* logging:: +* configuration:: +* exported symbols:: +* private (library-internal) symbols (including structs and macros):: +* testcases:: +* performance tests:: +* src/ directories:: +@end menu + +@node include files +@subsubsection include files + +@itemize @bullet +@item _lib: library without need for a process +@item _service: library that needs a service process +@item _plugin: plugin definition +@item _protocol: structs used in network protocol +@item exceptions: +@itemize @bullet +@item gnunet_config.h --- generated +@item platform.h --- first included +@item plibc.h --- external library +@item gnunet_common.h --- fundamental routines +@item gnunet_directories.h --- generated +@item gettext.h --- external library +@end itemize +@end itemize + +@c *********************************************************************** +@node binaries +@subsubsection binaries + +@itemize @bullet +@item gnunet-service-xxx: service process (has listen socket) +@item gnunet-daemon-xxx: daemon process (no listen socket) +@item gnunet-helper-xxx[-yyy]: SUID helper for module xxx +@item gnunet-yyy: command-line tool for end-users +@item libgnunet_plugin_xxx_yyy.so: plugin for API xxx +@item libgnunetxxx.so: library for API xxx +@end itemize + +@c *********************************************************************** +@node logging +@subsubsection logging + +@itemize @bullet +@item services and daemons use their directory name in +@code{GNUNET_log_setup} (i.e. 'core') and log using +plain 'GNUNET_log'. +@item command-line tools use their full name in +@code{GNUNET_log_setup} (i.e. 'gnunet-publish') and log using +plain 'GNUNET_log'. +@item service access libraries log using +'@code{GNUNET_log_from}' and use '@code{DIRNAME-api}' for the +component (i.e. 'core-api') +@item pure libraries (without associated service) use +'@code{GNUNET_log_from}' with the component set to their +library name (without lib or '@file{.so}'), +which should also be their directory name (i.e. '@file{nat}') +@item plugins should use '@code{GNUNET_log_from}' +with the directory name and the plugin name combined to produce +the component name (i.e. 'transport-tcp'). +@item logging should be unified per-file by defining a +@code{LOG} macro with the appropriate arguments, +along these lines: + +@example +#define LOG(kind,...) +GNUNET_log_from (kind, "example-api",__VA_ARGS__) +@end example + +@end itemize + +@c *********************************************************************** +@node configuration +@subsubsection configuration + +@itemize @bullet +@item paths (that are substituted in all filenames) are in PATHS +(have as few as possible) +@item all options for a particular module (@file{src/MODULE}) +are under @code{[MODULE]} +@item options for a plugin of a module +are under @code{[MODULE-PLUGINNAME]} +@end itemize + +@c *********************************************************************** +@node exported symbols +@subsubsection exported symbols + +@itemize @bullet +@item must start with @code{GNUNET_modulename_} and be defined in +@file{modulename.c} +@item exceptions: those defined in @file{gnunet_common.h} +@end itemize + +@c *********************************************************************** +@node private (library-internal) symbols (including structs and macros) +@subsubsection private (library-internal) symbols (including structs and macros) + +@itemize @bullet +@item must NOT start with any prefix +@item must not be exported in a way that linkers could use them or@ other +libraries might see them via headers; they must be either +declared/defined in C source files or in headers that are in the +respective directory under @file{src/modulename/} and NEVER be declared +in @file{src/include/}. +@end itemize + +@node testcases +@subsubsection testcases + +@itemize @bullet +@item must be called @file{test_module-under-test_case-description.c} +@item "case-description" maybe omitted if there is only one test +@end itemize + +@c *********************************************************************** +@node performance tests +@subsubsection performance tests + +@itemize @bullet +@item must be called @file{perf_module-under-test_case-description.c} +@item "case-description" maybe omitted if there is only one performance +test +@item Must only be run if @code{HAVE_BENCHMARKS} is satisfied +@end itemize + +@c *********************************************************************** +@node src/ directories +@subsubsection src/ directories + +@itemize @bullet +@item gnunet-NAME: end-user applications (i.e., gnunet-search, gnunet-arm) +@item gnunet-service-NAME: service processes with accessor library (i.e., +gnunet-service-arm) +@item libgnunetNAME: accessor library (_service.h-header) or standalone +library (_lib.h-header) +@item gnunet-daemon-NAME: daemon process without accessor library (i.e., +gnunet-daemon-hostlist) and no GNUnet management port +@item libgnunet_plugin_DIR_NAME: loadable plugins (i.e., +libgnunet_plugin_transport_tcp) +@end itemize + +@cindex Coding style +@node Coding style +@subsection Coding style + +@c XXX: Adjust examples to GNU Standards! +@itemize @bullet +@item We follow the GNU Coding Standards (@pxref{Top, The GNU Coding Standards,, standards, The GNU Coding Standards}); +@item Indentation is done with spaces, two per level, no tabs; +@item C99 struct initialization is fine; +@item declare only one variable per line, for example: + +@noindent +instead of + +@example +int i,j; +@end example + +@noindent +write: + +@example +int i; +int j; +@end example + +@c TODO: include actual example from a file in source + +@noindent +This helps keep diffs small and forces developers to think precisely about +the type of every variable. +Note that @code{char *} is different from @code{const char*} and +@code{int} is different from @code{unsigned int} or @code{uint32_t}. +Each variable type should be chosen with care. + +@item While @code{goto} should generally be avoided, having a +@code{goto} to the end of a function to a block of clean up +statements (free, close, etc.) can be acceptable. + +@item Conditions should be written with constants on the left (to avoid +accidental assignment) and with the 'true' target being either the +'error' case or the significantly simpler continuation. For example: + +@example +if (0 != stat ("filename," &sbuf)) @{ + error(); + @} + else @{ + /* handle normal case here */ + @} +@end example + +@noindent +instead of + +@example +if (stat ("filename," &sbuf) == 0) @{ + /* handle normal case here */ + @} else @{ + error(); + @} +@end example + +@noindent +If possible, the error clause should be terminated with a 'return' (or +'goto' to some cleanup routine) and in this case, the 'else' clause +should be omitted: + +@example +if (0 != stat ("filename," &sbuf)) @{ + error(); + return; + @} +/* handle normal case here */ +@end example + +This serves to avoid deep nesting. The 'constants on the left' rule +applies to all constants (including. @code{GNUNET_SCHEDULER_NO_TASK}), +NULL, and enums). With the two above rules (constants on left, errors in +'true' branch), there is only one way to write most branches correctly. + +@item Combined assignments and tests are allowed if they do not hinder +code clarity. For example, one can write: + +@example +if (NULL == (value = lookup_function())) @{ + error(); + return; + @} +@end example + +@item Use @code{break} and @code{continue} wherever possible to avoid +deep(er) nesting. Thus, we would write: + +@example +next = head; +while (NULL != (pos = next)) @{ + next = pos->next; + if (! should_free (pos)) + continue; + GNUNET_CONTAINER_DLL_remove (head, tail, pos); + GNUNET_free (pos); + @} +@end example + +instead of + +@example +next = head; while (NULL != (pos = next)) @{ + next = pos->next; + if (should_free (pos)) @{ + /* unnecessary nesting! */ + GNUNET_CONTAINER_DLL_remove (head, tail, pos); + GNUNET_free (pos); + @} + @} +@end example + +@item We primarily use @code{for} and @code{while} loops. +A @code{while} loop is used if the method for advancing in the loop is +not a straightforward increment operation. In particular, we use: + +@example +next = head; +while (NULL != (pos = next)) +@{ + next = pos->next; + if (! should_free (pos)) + continue; + GNUNET_CONTAINER_DLL_remove (head, tail, pos); + GNUNET_free (pos); +@} +@end example + +to free entries in a list (as the iteration changes the structure of the +list due to the free; the equivalent @code{for} loop does no longer +follow the simple @code{for} paradigm of @code{for(INIT;TEST;INC)}). +However, for loops that do follow the simple @code{for} paradigm we do +use @code{for}, even if it involves linked lists: + +@example +/* simple iteration over a linked list */ +for (pos = head; + NULL != pos; + pos = pos->next) +@{ + use (pos); +@} +@end example + + +@item The first argument to all higher-order functions in GNUnet must be +declared to be of type @code{void *} and is reserved for a closure. We do +not use inner functions, as trampolines would conflict with setups that +use non-executable stacks. +The first statement in a higher-order function, which unusually should +be part of the variable declarations, should assign the +@code{cls} argument to the precise expected type. For example: + +@example +int callback (void *cls, char *args) @{ + struct Foo *foo = cls; + int other_variables; + + /* rest of function */ +@} +@end example + + +@item It is good practice to write complex @code{if} expressions instead +of using deeply nested @code{if} statements. However, except for addition +and multiplication, all operators should use parens. This is fine: + +@example +if ( (1 == foo) || ((0 == bar) && (x != y)) ) + return x; +@end example + + +However, this is not: + +@example +if (1 == foo) + return x; +if (0 == bar && x != y) + return x; +@end example + +@noindent +Note that splitting the @code{if} statement above is debateable as the +@code{return x} is a very trivial statement. However, once the logic after +the branch becomes more complicated (and is still identical), the "or" +formulation should be used for sure. + +@item There should be two empty lines between the end of the function and +the comments describing the following function. There should be a single +empty line after the initial variable declarations of a function. If a +function has no local variables, there should be no initial empty line. If +a long function consists of several complex steps, those steps might be +separated by an empty line (possibly followed by a comment describing the +following step). The code should not contain empty lines in arbitrary +places; if in doubt, it is likely better to NOT have an empty line (this +way, more code will fit on the screen). +@end itemize + +@c *********************************************************************** +@node Build-system +@section Build-system + +If you have code that is likely not to compile or build rules you might +want to not trigger for most developers, use @code{if HAVE_EXPERIMENTAL} +in your @file{Makefile.am}. +Then it is OK to (temporarily) add non-compiling (or known-to-not-port) +code. + +If you want to compile all testcases but NOT run them, run configure with +the @code{--enable-test-suppression} option. + +If you want to run all testcases, including those that take a while, run +configure with the @code{--enable-expensive-testcases} option. + +If you want to compile and run benchmarks, run configure with the +@code{--enable-benchmarks} option. + +If you want to obtain code coverage results, run configure with the +@code{--enable-coverage} option and run the @file{coverage.sh} script in +the @file{contrib/} directory. + +@cindex gnunet-ext +@node Developing extensions for GNUnet using the gnunet-ext template +@section Developing extensions for GNUnet using the gnunet-ext template + +For developers who want to write extensions for GNUnet we provide the +gnunet-ext template to provide an easy to use skeleton. + +gnunet-ext contains the build environment and template files for the +development of GNUnet services, command line tools, APIs and tests. + +First of all you have to obtain gnunet-ext from git: + +@example +git clone https://gnunet.org/git/gnunet-ext.git +@end example + +The next step is to bootstrap and configure it. For configure you have to +provide the path containing GNUnet with +@code{--with-gnunet=/path/to/gnunet} and the prefix where you want the +install the extension using @code{--prefix=/path/to/install}: + +@example +./bootstrap +./configure --prefix=/path/to/install --with-gnunet=/path/to/gnunet +@end example + +When your GNUnet installation is not included in the default linker search +path, you have to add @code{/path/to/gnunet} to the file +@file{/etc/ld.so.conf} and run @code{ldconfig} or your add it to the +environmental variable @code{LD_LIBRARY_PATH} by using + +@example +export LD_LIBRARY_PATH=/path/to/gnunet/lib +@end example + +@cindex writing testcases +@node Writing testcases +@section Writing testcases + +Ideally, any non-trivial GNUnet code should be covered by automated +testcases. Testcases should reside in the same place as the code that is +being tested. The name of source files implementing tests should begin +with @code{test_} followed by the name of the file that contains +the code that is being tested. + +Testcases in GNUnet should be integrated with the autotools build system. +This way, developers and anyone building binary packages will be able to +run all testcases simply by running @code{make check}. The final +testcases shipped with the distribution should output at most some brief +progress information and not display debug messages by default. The +success or failure of a testcase must be indicated by returning zero +(success) or non-zero (failure) from the main method of the testcase. +The integration with the autotools is relatively straightforward and only +requires modifications to the @file{Makefile.am} in the directory +containing the testcase. For a testcase testing the code in @file{foo.c} +the @file{Makefile.am} would contain the following lines: + +@example +check_PROGRAMS = test_foo +TESTS = $(check_PROGRAMS) +test_foo_SOURCES = test_foo.c +test_foo_LDADD = $(top_builddir)/src/util/libgnunetutil.la +@end example + +Naturally, other libraries used by the testcase may be specified in the +@code{LDADD} directive as necessary. + +Often testcases depend on additional input files, such as a configuration +file. These support files have to be listed using the @code{EXTRA_DIST} +directive in order to ensure that they are included in the distribution. + +Example: + +@example +EXTRA_DIST = test_foo_data.conf +@end example + +Executing @code{make check} will run all testcases in the current +directory and all subdirectories. Testcases can be compiled individually +by running @code{make test_foo} and then invoked directly using +@code{./test_foo}. Note that due to the use of plugins in GNUnet, it is +typically necessary to run @code{make install} before running any +testcases. Thus the canonical command @code{make check install} has to be +changed to @code{make install check} for GNUnet. + +@cindex TESTING library +@node TESTING library +@section TESTING library + +The TESTING library is used for writing testcases which involve starting a +single or multiple peers. While peers can also be started by testcases +using the ARM subsystem, using TESTING library provides an elegant way to +do this. The configurations of the peers are auto-generated from a given +template to have non-conflicting port numbers ensuring that peers' +services do not run into bind errors. This is achieved by testing ports' +availability by binding a listening socket to them before allocating them +to services in the generated configurations. + +An another advantage while using TESTING is that it shortens the testcase +startup time as the hostkeys for peers are copied from a pre-computed set +of hostkeys instead of generating them at peer startup which may take a +considerable amount of time when starting multiple peers or on an embedded +processor. + +TESTING also allows for certain services to be shared among peers. This +feature is invaluable when testing with multiple peers as it helps to +reduce the number of services run per each peer and hence the total +number of processes run per testcase. + +TESTING library only handles creating, starting and stopping peers. +Features useful for testcases such as connecting peers in a topology are +not available in TESTING but are available in the TESTBED subsystem. +Furthermore, TESTING only creates peers on the localhost, however by +using TESTBED testcases can benefit from creating peers across multiple +hosts. + +@menu +* API:: +* Finer control over peer stop:: +* Helper functions:: +* Testing with multiple processes:: +@end menu + +@cindex TESTING API +@node API +@subsection API + +TESTING abstracts a group of peers as a TESTING system. All peers in a +system have common hostname and no two services of these peers have a +same port or a UNIX domain socket path. + +TESTING system can be created with the function +@code{GNUNET_TESTING_system_create()} which returns a handle to the +system. This function takes a directory path which is used for generating +the configurations of peers, an IP address from which connections to the +peers' services should be allowed, the hostname to be used in peers' +configuration, and an array of shared service specifications of type +@code{struct GNUNET_TESTING_SharedService}. + +The shared service specification must specify the name of the service to +share, the configuration pertaining to that shared service and the +maximum number of peers that are allowed to share a single instance of +the shared service. + +TESTING system created with @code{GNUNET_TESTING_system_create()} chooses +ports from the default range @code{12000} - @code{56000} while +auto-generating configurations for peers. +This range can be customised with the function +@code{GNUNET_TESTING_system_create_with_portrange()}. This function is +similar to @code{GNUNET_TESTING_system_create()} except that it take 2 +additional parameters --- the start and end of the port range to use. + +A TESTING system is destroyed with the funciton +@code{GNUNET_TESTING_system_destory()}. This function takes the handle of +the system and a flag to remove the files created in the directory used +to generate configurations. + +A peer is created with the function +@code{GNUNET_TESTING_peer_configure()}. This functions takes the system +handle, a configuration template from which the configuration for the peer +is auto-generated and the index from where the hostkey for the peer has to +be copied from. When successfull, this function returs a handle to the +peer which can be used to start and stop it and to obtain the identity of +the peer. If unsuccessful, a NULL pointer is returned with an error +message. This function handles the generated configuration to have +non-conflicting ports and paths. + +Peers can be started and stopped by calling the functions +@code{GNUNET_TESTING_peer_start()} and @code{GNUNET_TESTING_peer_stop()} +respectively. A peer can be destroyed by calling the function +@code{GNUNET_TESTING_peer_destroy}. When a peer is destroyed, the ports +and paths in allocated in its configuration are reclaimed for usage in new +peers. + +@c *********************************************************************** +@node Finer control over peer stop +@subsection Finer control over peer stop + +Using @code{GNUNET_TESTING_peer_stop()} is normally fine for testcases. +However, calling this function for each peer is inefficient when trying to +shutdown multiple peers as this function sends the termination signal to +the given peer process and waits for it to terminate. It would be faster +in this case to send the termination signals to the peers first and then +wait on them. This is accomplished by the functions +@code{GNUNET_TESTING_peer_kill()} which sends a termination signal to the +peer, and the function @code{GNUNET_TESTING_peer_wait()} which waits on +the peer. + +Further finer control can be achieved by choosing to stop a peer +asynchronously with the function @code{GNUNET_TESTING_peer_stop_async()}. +This function takes a callback parameter and a closure for it in addition +to the handle to the peer to stop. The callback function is called with +the given closure when the peer is stopped. Using this function +eliminates blocking while waiting for the peer to terminate. + +An asynchronous peer stop can be cancelled by calling the function +@code{GNUNET_TESTING_peer_stop_async_cancel()}. Note that calling this +function does not prevent the peer from terminating if the termination +signal has already been sent to it. It does, however, cancels the +callback to be called when the peer is stopped. + +@c *********************************************************************** +@node Helper functions +@subsection Helper functions + +Most of the testcases can benefit from an abstraction which configures a +peer and starts it. This is provided by the function +@code{GNUNET_TESTING_peer_run()}. This function takes the testing +directory pathname, a configuration template, a callback and its closure. +This function creates a peer in the given testing directory by using the +configuration template, starts the peer and calls the given callback with +the given closure. + +The function @code{GNUNET_TESTING_peer_run()} starts the ARM service of +the peer which starts the rest of the configured services. A similar +function @code{GNUNET_TESTING_service_run} can be used to just start a +single service of a peer. In this case, the peer's ARM service is not +started; instead, only the given service is run. + +@c *********************************************************************** +@node Testing with multiple processes +@subsection Testing with multiple processes + +When testing GNUnet, the splitting of the code into a services and clients +often complicates testing. The solution to this is to have the testcase +fork @code{gnunet-service-arm}, ask it to start the required server and +daemon processes and then execute appropriate client actions (to test the +client APIs or the core module or both). If necessary, multiple ARM +services can be forked using different ports (!) to simulate a network. +However, most of the time only one ARM process is needed. Note that on +exit, the testcase should shutdown ARM with a @code{TERM} signal (to give +it the chance to cleanly stop its child processes). + +The following code illustrates spawning and killing an ARM process from a +testcase: + +@example +static void run (void *cls, + char *const *args, + const char *cfgfile, + const struct GNUNET_CONFIGURATION_Handle *cfg) @{ + struct GNUNET_OS_Process *arm_pid; + arm_pid = GNUNET_OS_start_process (NULL, + NULL, + "gnunet-service-arm", + "gnunet-service-arm", + "-c", + cfgname, + NULL); + /* do real test work here */ + if (0 != GNUNET_OS_process_kill (arm_pid, SIGTERM)) + GNUNET_log_strerror + (GNUNET_ERROR_TYPE_WARNING, "kill"); + GNUNET_assert (GNUNET_OK == GNUNET_OS_process_wait (arm_pid)); + GNUNET_OS_process_close (arm_pid); @} + +GNUNET_PROGRAM_run (argc, argv, + "NAME-OF-TEST", + "nohelp", + options, + &run, + cls); +@end example + + +An alternative way that works well to test plugins is to implement a +mock-version of the environment that the plugin expects and then to +simply load the plugin directly. + +@c *********************************************************************** +@node Performance regression analysis with Gauger +@section Performance regression analysis with Gauger + +To help avoid performance regressions, GNUnet uses Gauger. Gauger is a +simple logging tool that allows remote hosts to send performance data to +a central server, where this data can be analyzed and visualized. Gauger +shows graphs of the repository revisions and the performace data recorded +for each revision, so sudden performance peaks or drops can be identified +and linked to a specific revision number. + +In the case of GNUnet, the buildbots log the performance data obtained +during the tests after each build. The data can be accesed on GNUnet's +Gauger page. + +The menu on the left allows to select either the results of just one +build bot (under "Hosts") or review the data from all hosts for a given +test result (under "Metrics"). In case of very different absolute value +of the results, for instance arm vs. amd64 machines, the option +"Normalize" on a metric view can help to get an idea about the +performance evolution across all hosts. + +Using Gauger in GNUnet and having the performance of a module tracked over +time is very easy. First of course, the testcase must generate some +consistent metric, which makes sense to have logged. Highly volatile or +random dependant metrics probably are not ideal candidates for meaningful +regression detection. + +To start logging any value, just include @code{gauger.h} in your testcase +code. Then, use the macro @code{GAUGER()} to make the Buildbots log +whatever value is of interest for you to @code{gnunet.org}'s Gauger +server. No setup is necessary as most Buildbots have already everything +in place and new metrics are created on demand. To delete a metric, you +need to contact a member of the GNUnet development team (a file will need +to be removed manually from the respective directory). + +The code in the test should look like this: + +@example +[other includes] +#include <gauger.h> + +int main (int argc, char *argv[]) @{ + + [run test, generate data] + GAUGER("YOUR_MODULE", + "METRIC_NAME", + (float)value, + "UNIT"); @} +@end example + +Where: + +@table @asis + +@item @strong{YOUR_MODULE} is a category in the gauger page and should be +the name of the module or subsystem like "Core" or "DHT" +@item @strong{METRIC} is +the name of the metric being collected and should be concise and +descriptive, like "PUT operations in sqlite-datastore". +@item @strong{value} is the value +of the metric that is logged for this run. +@item @strong{UNIT} is the unit in +which the value is measured, for instance "kb/s" or "kb of RAM/node". +@end table + +If you wish to use Gauger for your own project, you can grab a copy of the +latest stable release or check out Gauger's Subversion repository. + +@cindex TESTBED Subsystem +@node TESTBED Subsystem +@section TESTBED Subsystem + +The TESTBED subsystem facilitates testing and measuring of multi-peer +deployments on a single host or over multiple hosts. + +The architecture of the testbed module is divided into the following: +@itemize @bullet + +@item Testbed API: An API which is used by the testing driver programs. It +provides with functions for creating, destroying, starting, stopping +peers, etc. + +@item Testbed service (controller): A service which is started through the +Testbed API. This service handles operations to create, destroy, start, +stop peers, connect them, modify their configurations. + +@item Testbed helper: When a controller has to be started on a host, the +testbed API starts the testbed helper on that host which in turn starts +the controller. The testbed helper receives a configuration for the +controller through its stdin and changes it to ensure the controller +doesn't run into any port conflict on that host. +@end itemize + + +The testbed service (controller) is different from the other GNUnet +services in that it is not started by ARM and is not supposed to be run +as a daemon. It is started by the testbed API through a testbed helper. +In a typical scenario involving multiple hosts, a controller is started +on each host. Controllers take up the actual task of creating peers, +starting and stopping them on the hosts they run. + +While running deployments on a single localhost the testbed API starts the +testbed helper directly as a child process. When running deployments on +remote hosts the testbed API starts Testbed Helpers on each remote host +through remote shell. By default testbed API uses SSH as a remote shell. +This can be changed by setting the environmental variable +GNUNET_TESTBED_RSH_CMD to the required remote shell program. This +variable can also contain parameters which are to be passed to the remote +shell program. For e.g: + +@example +export GNUNET_TESTBED_RSH_CMD="ssh -o BatchMode=yes \ +-o NoHostAuthenticationForLocalhost=yes %h" +@end example + +Substitutions are allowed in the command string above, +this allows for substitutions through placemarks which begin with a `%'. +At present the following substitutions are supported + +@itemize @bullet +@item %h: hostname +@item %u: username +@item %p: port +@end itemize + +Note that the substitution placemark is replaced only when the +corresponding field is available and only once. Specifying + +@example +%u@atchar{}%h +@end example + +doesn't work either. If you want to user username substitutions for +@command{SSH}, use the argument @code{-l} before the +username substitution. + +For example: +@example +ssh -l %u -p %p %h +@end example + +The testbed API and the helper communicate through the helpers stdin and +stdout. As the helper is started through a remote shell on remote hosts +any output messages from the remote shell interfere with the communication +and results in a failure while starting the helper. For this reason, it is +suggested to use flags to make the remote shells produce no output +messages and to have password-less logins. The default remote shell, SSH, +the default options are: + +@example +-o BatchMode=yes -o NoHostBasedAuthenticationForLocalhost=yes" +@end example + +Password-less logins should be ensured by using SSH keys. + +Since the testbed API executes the remote shell as a non-interactive +shell, certain scripts like .bashrc, .profiler may not be executed. If +this is the case testbed API can be forced to execute an interactive +shell by setting up the environmental variable +@code{GNUNET_TESTBED_RSH_CMD_SUFFIX} to a shell program. + +An example could be: + +@example +export GNUNET_TESTBED_RSH_CMD_SUFFIX="sh -lc" +@end example + +The testbed API will then execute the remote shell program as: + +@example +$GNUNET_TESTBED_RSH_CMD -p $port $dest $GNUNET_TESTBED_RSH_CMD_SUFFIX \ +gnunet-helper-testbed +@end example + +On some systems, problems may arise while starting testbed helpers if +GNUnet is installed into a custom location since the helper may not be +found in the standard path. This can be addressed by setting the variable +`@code{HELPER_BINARY_PATH}' to the path of the testbed helper. +Testbed API will then use this path to start helper binaries both +locally and remotely. + +Testbed API can accessed by including the +@file{gnunet_testbed_service.h} file and linking with +@code{-lgnunettestbed}. + +@c *********************************************************************** +@menu +* Supported Topologies:: +* Hosts file format:: +* Topology file format:: +* Testbed Barriers:: +* Automatic large-scale deployment in the PlanetLab testbed:: +* TESTBED Caveats:: +@end menu + +@node Supported Topologies +@subsection Supported Topologies + +While testing multi-peer deployments, it is often needed that the peers +are connected in some topology. This requirement is addressed by the +function @code{GNUNET_TESTBED_overlay_connect()} which connects any given +two peers in the testbed. + +The API also provides a helper function +@code{GNUNET_TESTBED_overlay_configure_topology()} to connect a given set +of peers in any of the following supported topologies: + +@itemize @bullet + +@item @code{GNUNET_TESTBED_TOPOLOGY_CLIQUE}: All peers are connected with +each other + +@item @code{GNUNET_TESTBED_TOPOLOGY_LINE}: Peers are connected to form a +line + +@item @code{GNUNET_TESTBED_TOPOLOGY_RING}: Peers are connected to form a +ring topology + +@item @code{GNUNET_TESTBED_TOPOLOGY_2D_TORUS}: Peers are connected to +form a 2 dimensional torus topology. The number of peers may not be a +perfect square, in that case the resulting torus may not have the uniform +poloidal and toroidal lengths + +@item @code{GNUNET_TESTBED_TOPOLOGY_ERDOS_RENYI}: Topology is generated +to form a random graph. The number of links to be present should be given + +@item @code{GNUNET_TESTBED_TOPOLOGY_SMALL_WORLD}: Peers are connected to +form a 2D Torus with some random links among them. The number of random +links are to be given + +@item @code{GNUNET_TESTBED_TOPOLOGY_SMALL_WORLD_RING}: Peers are +connected to form a ring with some random links among them. The number of +random links are to be given + +@item @code{GNUNET_TESTBED_TOPOLOGY_SCALE_FREE}: Connects peers in a +topology where peer connectivity follows power law - new peers are +connected with high probabililty to well connected peers. +@footnote{See Emergence of Scaling in Random Networks. Science 286, +509-512, 1999 +(@uref{https://gnunet.org/git/bibliography.git/plain/docs/emergence_of_scaling_in_random_networks__barabasi_albert_science_286__1999.pdf, pdf})} + +@item @code{GNUNET_TESTBED_TOPOLOGY_FROM_FILE}: The topology information +is loaded from a file. The path to the file has to be given. +@xref{Topology file format}, for the format of this file. + +@item @code{GNUNET_TESTBED_TOPOLOGY_NONE}: No topology +@end itemize + + +The above supported topologies can be specified respectively by setting +the variable @code{OVERLAY_TOPOLOGY} to the following values in the +configuration passed to Testbed API functions +@code{GNUNET_TESTBED_test_run()} and +@code{GNUNET_TESTBED_run()}: + +@itemize @bullet +@item @code{CLIQUE} +@item @code{RING} +@item @code{LINE} +@item @code{2D_TORUS} +@item @code{RANDOM} +@item @code{SMALL_WORLD} +@item @code{SMALL_WORLD_RING} +@item @code{SCALE_FREE} +@item @code{FROM_FILE} +@item @code{NONE} +@end itemize + + +Topologies @code{RANDOM}, @code{SMALL_WORLD} and @code{SMALL_WORLD_RING} +require the option @code{OVERLAY_RANDOM_LINKS} to be set to the number of +random links to be generated in the configuration. The option will be +ignored for the rest of the topologies. + +Topology @code{SCALE_FREE} requires the options +@code{SCALE_FREE_TOPOLOGY_CAP} to be set to the maximum number of peers +which can connect to a peer and @code{SCALE_FREE_TOPOLOGY_M} to be set to +how many peers a peer should be atleast connected to. + +Similarly, the topology @code{FROM_FILE} requires the option +@code{OVERLAY_TOPOLOGY_FILE} to contain the path of the file containing +the topology information. This option is ignored for the rest of the +topologies. @xref{Topology file format}, for the format of this file. + +@c *********************************************************************** +@node Hosts file format +@subsection Hosts file format + +The testbed API offers the function +@code{GNUNET_TESTBED_hosts_load_from_file()} to load from a given file +details about the hosts which testbed can use for deploying peers. +This function is useful to keep the data about hosts +separate instead of hard coding them in code. + +Another helper function from testbed API, @code{GNUNET_TESTBED_run()} +also takes a hosts file name as its parameter. It uses the above +function to populate the hosts data structures and start controllers to +deploy peers. + +These functions require the hosts file to be of the following format: +@itemize @bullet +@item Each line is interpreted to have details about a host +@item Host details should include the username to use for logging into the +host, the hostname of the host and the port number to use for the remote +shell program. All thee values should be given. +@item These details should be given in the following format: +@example +<username>@@<hostname>:<port> +@end example +@end itemize + +Note that having canonical hostnames may cause problems while resolving +the IP addresses (See this bug). Hence it is advised to provide the hosts' +IP numerical addresses as hostnames whenever possible. + +@c *********************************************************************** +@node Topology file format +@subsection Topology file format + +A topology file describes how peers are to be connected. It should adhere +to the following format for testbed to parse it correctly. + +Each line should begin with the target peer id. This should be followed by +a colon(`:') and origin peer ids seperated by `|'. All spaces except for +newline characters are ignored. The API will then try to connect each +origin peer to the target peer. + +For example, the following file will result in 5 overlay connections: +[2->1], [3->1],[4->3], [0->3], [2->0]@ +@code{@ 1:2|3@ 3:4| 0@ 0: 2@ } + +@c *********************************************************************** +@node Testbed Barriers +@subsection Testbed Barriers + +The testbed subsystem's barriers API facilitates coordination among the +peers run by the testbed and the experiment driver. The concept is +similar to the barrier synchronisation mechanism found in parallel +programming or multi-threading paradigms - a peer waits at a barrier upon +reaching it until the barrier is reached by a predefined number of peers. +This predefined number of peers required to cross a barrier is also called +quorum. We say a peer has reached a barrier if the peer is waiting for the +barrier to be crossed. Similarly a barrier is said to be reached if the +required quorum of peers reach the barrier. A barrier which is reached is +deemed as crossed after all the peers waiting on it are notified. + +The barriers API provides the following functions: +@itemize @bullet +@item @strong{@code{GNUNET_TESTBED_barrier_init()}:} function to +initialse a barrier in the experiment +@item @strong{@code{GNUNET_TESTBED_barrier_cancel()}:} function to cancel +a barrier which has been initialised before +@item @strong{@code{GNUNET_TESTBED_barrier_wait()}:} function to signal +barrier service that the caller has reached a barrier and is waiting for +it to be crossed +@item @strong{@code{GNUNET_TESTBED_barrier_wait_cancel()}:} function to +stop waiting for a barrier to be crossed +@end itemize + + +Among the above functions, the first two, namely +@code{GNUNET_TESTBED_barrier_init()} and +@code{GNUNET_TESTBED_barrier_cancel()} are used by experiment drivers. All +barriers should be initialised by the experiment driver by calling +@code{GNUNET_TESTBED_barrier_init()}. This function takes a name to +identify the barrier, the quorum required for the barrier to be crossed +and a notification callback for notifying the experiment driver when the +barrier is crossed. @code{GNUNET_TESTBED_barrier_cancel()} cancels an +initialised barrier and frees the resources allocated for it. This +function can be called upon a initialised barrier before it is crossed. + +The remaining two functions @code{GNUNET_TESTBED_barrier_wait()} and +@code{GNUNET_TESTBED_barrier_wait_cancel()} are used in the peer's +processes. @code{GNUNET_TESTBED_barrier_wait()} connects to the local +barrier service running on the same host the peer is running on and +registers that the caller has reached the barrier and is waiting for the +barrier to be crossed. Note that this function can only be used by peers +which are started by testbed as this function tries to access the local +barrier service which is part of the testbed controller service. Calling +@code{GNUNET_TESTBED_barrier_wait()} on an uninitialised barrier results +in failure. @code{GNUNET_TESTBED_barrier_wait_cancel()} cancels the +notification registered by @code{GNUNET_TESTBED_barrier_wait()}. + + +@c *********************************************************************** +@menu +* Implementation:: +@end menu + +@node Implementation +@subsubsection Implementation + +Since barriers involve coordination between experiment driver and peers, +the barrier service in the testbed controller is split into two +components. The first component responds to the message generated by the +barrier API used by the experiment driver (functions +@code{GNUNET_TESTBED_barrier_init()} and +@code{GNUNET_TESTBED_barrier_cancel()}) and the second component to the +messages generated by barrier API used by peers (functions +@code{GNUNET_TESTBED_barrier_wait()} and +@code{GNUNET_TESTBED_barrier_wait_cancel()}). + +Calling @code{GNUNET_TESTBED_barrier_init()} sends a +@code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_INIT} message to the master +controller. The master controller then registers a barrier and calls +@code{GNUNET_TESTBED_barrier_init()} for each its subcontrollers. In this +way barrier initialisation is propagated to the controller hierarchy. +While propagating initialisation, any errors at a subcontroller such as +timeout during further propagation are reported up the hierarchy back to +the experiment driver. + +Similar to @code{GNUNET_TESTBED_barrier_init()}, +@code{GNUNET_TESTBED_barrier_cancel()} propagates +@code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_CANCEL} message which causes +controllers to remove an initialised barrier. + +The second component is implemented as a separate service in the binary +`gnunet-service-testbed' which already has the testbed controller service. +Although this deviates from the gnunet process architecture of having one +service per binary, it is needed in this case as this component needs +access to barrier data created by the first component. This component +responds to @code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_WAIT} messages from +local peers when they call @code{GNUNET_TESTBED_barrier_wait()}. Upon +receiving @code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_WAIT} message, the +service checks if the requested barrier has been initialised before and +if it was not initialised, an error status is sent through +@code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_STATUS} message to the local +peer and the connection from the peer is terminated. If the barrier is +initialised before, the barrier's counter for reached peers is incremented +and a notification is registered to notify the peer when the barrier is +reached. The connection from the peer is left open. + +When enough peers required to attain the quorum send +@code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_WAIT} messages, the controller +sends a @code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_STATUS} message to its +parent informing that the barrier is crossed. If the controller has +started further subcontrollers, it delays this message until it receives +a similar notification from each of those subcontrollers. Finally, the +barriers API at the experiment driver receives the +@code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_STATUS} when the barrier is +reached at all the controllers. + +The barriers API at the experiment driver responds to the +@code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_STATUS} message by echoing it +back to the master controller and notifying the experiment controller +through the notification callback that a barrier has been crossed. The +echoed @code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_STATUS} message is +propagated by the master controller to the controller hierarchy. This +propagation triggers the notifications registered by peers at each of the +controllers in the hierarchy. Note the difference between this downward +propagation of the @code{GNUNET_MESSAGE_TYPE_TESTBED_BARRIER_STATUS} +message from its upward propagation --- the upward propagation is needed +for ensuring that the barrier is reached by all the controllers and the +downward propagation is for triggering that the barrier is crossed. + +@cindex PlanetLab testbed +@node Automatic large-scale deployment in the PlanetLab testbed +@subsection Automatic large-scale deployment in the PlanetLab testbed + +PlanetLab is a testbed for computer networking and distributed systems +research. It was established in 2002 and as of June 2010 was composed of +1090 nodes at 507 sites worldwide. + +To automate the GNUnet we created a set of automation tools to simplify +the large-scale deployment. We provide you a set of scripts you can use +to deploy GNUnet on a set of nodes and manage your installation. + +Please also check @uref{https://gnunet.org/installation-fedora8-svn} and +@uref{https://gnunet.org/installation-fedora12-svn} to find detailled +instructions how to install GNUnet on a PlanetLab node. + + +@c *********************************************************************** +@menu +* PlanetLab Automation for Fedora8 nodes:: +* Install buildslave on PlanetLab nodes running fedora core 8:: +* Setup a new PlanetLab testbed using GPLMT:: +* Why do i get an ssh error when using the regex profiler?:: +@end menu + +@node PlanetLab Automation for Fedora8 nodes +@subsubsection PlanetLab Automation for Fedora8 nodes + +@c *********************************************************************** +@node Install buildslave on PlanetLab nodes running fedora core 8 +@subsubsection Install buildslave on PlanetLab nodes running fedora core 8 +@c ** Actually this is a subsubsubsection, but must be fixed differently +@c ** as subsubsection is the lowest. + +Since most of the PlanetLab nodes are running the very old Fedora core 8 +image, installing the buildslave software is quite some pain. For our +PlanetLab testbed we figured out how to install the buildslave software +best. + +@c This is a vvery terrible way to suggest installing software. +@c FIXME: Is there an official, safer way instead of blind-piping a +@c script? +@c FIXME: Use newer pypi URLs below. +Install Distribute for Python: + +@example +curl http://python-distribute.org/distribute_setup.py | sudo python +@end example + +Install Distribute for zope.interface <= 3.8.0 (4.0 and 4.0.1 will not +work): + +@example +export PYPI=@value{PYPI-URL} +wget $PYPI/z/zope.interface/zope.interface-3.8.0.tar.gz +tar zvfz zope.interface-3.8.0.tar.gz +cd zope.interface-3.8.0 +sudo python setup.py install +@end example + +Install the buildslave software (0.8.6 was the latest version): + +@example +export GCODE="http://buildbot.googlecode.com/files" +wget $GCODE/buildbot-slave-0.8.6p1.tar.gz +tar xvfz buildbot-slave-0.8.6p1.tar.gz +cd buildslave-0.8.6p1 +sudo python setup.py install +@end example + +The setup will download the matching twisted package and install it. +It will also try to install the latest version of zope.interface which +will fail to install. Buildslave will work anyway since version 3.8.0 +was installed before! + +@c *********************************************************************** +@node Setup a new PlanetLab testbed using GPLMT +@subsubsection Setup a new PlanetLab testbed using GPLMT + +@itemize @bullet +@item Get a new slice and assign nodes +Ask your PlanetLab PI to give you a new slice and assign the nodes you +need +@item Install a buildmaster +You can stick to the buildbot documentation:@ +@uref{http://buildbot.net/buildbot/docs/current/manual/installation.html} +@item Install the buildslave software on all nodes +To install the buildslave on all nodes assigned to your slice you can use +the tasklist @code{install_buildslave_fc8.xml} provided with GPLMT: + +@example +./gplmt.py -c contrib/tumple_gnunet.conf -t \ +contrib/tasklists/install_buildslave_fc8.xml -a -p <planetlab password> +@end example + +@item Create the buildmaster configuration and the slave setup commands + +The master and the and the slaves have need to have credentials and the +master has to have all nodes configured. This can be done with the +@file{create_buildbot_configuration.py} script in the @file{scripts} +directory. + +This scripts takes a list of nodes retrieved directly from PlanetLab or +read from a file and a configuration template and creates: + +@itemize @bullet +@item a tasklist which can be executed with gplmt to setup the slaves +@item a master.cfg file containing a PlanetLab nodes +@end itemize + +A configuration template is included in the <contrib>, most important is +that the script replaces the following tags in the template: + +%GPLMT_BUILDER_DEFINITION :@ GPLMT_BUILDER_SUMMARY@ GPLMT_SLAVES@ +%GPLMT_SCHEDULER_BUILDERS + +Create configuration for all nodes assigned to a slice: + +@example +./create_buildbot_configuration.py -u <planetlab username> \ +-p <planetlab password> -s <slice> -m <buildmaster+port> \ +-t <template> +@end example + +Create configuration for some nodes in a file: + +@example +./create_buildbot_configuration.p -f <node_file> \ +-m <buildmaster+port> -t <template> +@end example + +@item Copy the @file{master.cfg} to the buildmaster and start it +Use @code{buildbot start <basedir>} to start the server +@item Setup the buildslaves +@end itemize + +@c *********************************************************************** +@node Why do i get an ssh error when using the regex profiler? +@subsubsection Why do i get an ssh error when using the regex profiler? + +Why do i get an ssh error "Permission denied (publickey,password)." when +using the regex profiler although passwordless ssh to localhost works +using publickey and ssh-agent? + +You have to generate a public/private-key pair with no password:@ +@code{ssh-keygen -t rsa -b 4096 -f ~/.ssh/id_localhost}@ +and then add the following to your ~/.ssh/config file: + +@code{Host 127.0.0.1@ IdentityFile ~/.ssh/id_localhost} + +now make sure your hostsfile looks like + +@example +[USERNAME]@@127.0.0.1:22@ +[USERNAME]@@127.0.0.1:22 +@end example + +You can test your setup by running @code{ssh 127.0.0.1} in a +terminal and then in the opened session run it again. +If you were not asked for a password on either login, +then you should be good to go. + +@cindex TESTBED Caveats +@node TESTBED Caveats +@subsection TESTBED Caveats + +This section documents a few caveats when using the GNUnet testbed +subsystem. + +@c *********************************************************************** +@menu +* CORE must be started:: +* ATS must want the connections:: +@end menu + +@node CORE must be started +@subsubsection CORE must be started + +A simple issue is #3993: Your configuration MUST somehow ensure that for +each peer the CORE service is started when the peer is setup, otherwise +TESTBED may fail to connect peers when the topology is initialized, as +TESTBED will start some CORE services but not necessarily all (but it +relies on all of them running). The easiest way is to set +'FORCESTART = YES' in the '[core]' section of the configuration file. +Alternatively, having any service that directly or indirectly depends on +CORE being started with FORCESTART will also do. This issue largely arises +if users try to over-optimize by not starting any services with +FORCESTART. + +@c *********************************************************************** +@node ATS must want the connections +@subsubsection ATS must want the connections + +When TESTBED sets up connections, it only offers the respective HELLO +information to the TRANSPORT service. It is then up to the ATS service to +@strong{decide} to use the connection. The ATS service will typically +eagerly establish any connection if the number of total connections is +low (relative to bandwidth). Details may further depend on the +specific ATS backend that was configured. If ATS decides to NOT establish +a connection (even though TESTBED provided the required information), then +that connection will count as failed for TESTBED. Note that you can +configure TESTBED to tolerate a certain number of connection failures +(see '-e' option of gnunet-testbed-profiler). This issue largely arises +for dense overlay topologies, especially if you try to create cliques +with more than 20 peers. + +@cindex libgnunetutil +@node libgnunetutil +@section libgnunetutil + +libgnunetutil is the fundamental library that all GNUnet code builds upon. +Ideally, this library should contain most of the platform dependent code +(except for user interfaces and really special needs that only few +applications have). It is also supposed to offer basic services that most +if not all GNUnet binaries require. The code of libgnunetutil is in the +@file{src/util/} directory. The public interface to the library is in the +gnunet_util.h header. The functions provided by libgnunetutil fall +roughly into the following categories (in roughly the order of importance +for new developers): + +@itemize @bullet +@item logging (common_logging.c) +@item memory allocation (common_allocation.c) +@item endianess conversion (common_endian.c) +@item internationalization (common_gettext.c) +@item String manipulation (string.c) +@item file access (disk.c) +@item buffered disk IO (bio.c) +@item time manipulation (time.c) +@item configuration parsing (configuration.c) +@item command-line handling (getopt*.c) +@item cryptography (crypto_*.c) +@item data structures (container_*.c) +@item CPS-style scheduling (scheduler.c) +@item Program initialization (program.c) +@item Networking (network.c, client.c, server*.c, service.c) +@item message queueing (mq.c) +@item bandwidth calculations (bandwidth.c) +@item Other OS-related (os*.c, plugin.c, signal.c) +@item Pseudonym management (pseudonym.c) +@end itemize + +It should be noted that only developers that fully understand this entire +API will be able to write good GNUnet code. + +Ideally, porting GNUnet should only require porting the gnunetutil +library. More testcases for the gnunetutil APIs are therefore a great +way to make porting of GNUnet easier. + +@menu +* Logging:: +* Interprocess communication API (IPC):: +* Cryptography API:: +* Message Queue API:: +* Service API:: +* Optimizing Memory Consumption of GNUnet's (Multi-) Hash Maps:: +* CONTAINER_MDLL API:: +@end menu + +@cindex Logging +@cindex log levels +@node Logging +@subsection Logging + +GNUnet is able to log its activity, mostly for the purposes of debugging +the program at various levels. + +@file{gnunet_common.h} defines several @strong{log levels}: +@table @asis + +@item ERROR for errors (really problematic situations, often leading to +crashes) +@item WARNING for warnings (troubling situations that might have +negative consequences, although not fatal) +@item INFO for various information. +Used somewhat rarely, as GNUnet statistics is used to hold and display +most of the information that users might find interesting. +@item DEBUG for debugging. +Does not produce much output on normal builds, but when extra logging is +enabled at compile time, a staggering amount of data is outputted under +this log level. +@end table + + +Normal builds of GNUnet (configured with @code{--enable-logging[=yes]}) +are supposed to log nothing under DEBUG level. The +@code{--enable-logging=verbose} configure option can be used to create a +build with all logging enabled. However, such build will produce large +amounts of log data, which is inconvenient when one tries to hunt down a +specific problem. + +To mitigate this problem, GNUnet provides facilities to apply a filter to +reduce the logs: +@table @asis + +@item Logging by default When no log levels are configured in any other +way (see below), GNUnet will default to the WARNING log level. This +mostly applies to GNUnet command line utilities, services and daemons; +tests will always set log level to WARNING or, if +@code{--enable-logging=verbose} was passed to configure, to DEBUG. The +default level is suggested for normal operation. +@item The -L option Most GNUnet executables accept an "-L loglevel" or +"--log=loglevel" option. If used, it makes the process set a global log +level to "loglevel". Thus it is possible to run some processes +with -L DEBUG, for example, and others with -L ERROR to enable specific +settings to diagnose problems with a particular process. +@item Configuration files. Because GNUnet +service and deamon processes are usually launched by gnunet-arm, it is not +possible to pass different custom command line options directly to every +one of them. The options passed to @code{gnunet-arm} only affect +gnunet-arm and not the rest of GNUnet. However, one can specify a +configuration key "OPTIONS" in the section that corresponds to a service +or a daemon, and put a value of "-L loglevel" there. This will make the +respective service or daemon set its log level to "loglevel" (as the +value of OPTIONS will be passed as a command-line argument). + +To specify the same log level for all services without creating separate +"OPTIONS" entries in the configuration for each one, the user can specify +a config key "GLOBAL_POSTFIX" in the [arm] section of the configuration +file. The value of GLOBAL_POSTFIX will be appended to all command lines +used by the ARM service to run other services. It can contain any option +valid for all GNUnet commands, thus in particular the "-L loglevel" +option. The ARM service itself is, however, unaffected by GLOBAL_POSTFIX; +to set log level for it, one has to specify "OPTIONS" key in the [arm] +section. +@item Environment variables. +Setting global per-process log levels with "-L loglevel" does not offer +sufficient log filtering granularity, as one service will call interface +libraries and supporting libraries of other GNUnet services, potentially +producing lots of debug log messages from these libraries. Also, changing +the config file is not always convenient (especially when running the +GNUnet test suite).@ To fix that, and to allow GNUnet to use different +log filtering at runtime without re-compiling the whole source tree, the +log calls were changed to be configurable at run time. To configure them +one has to define environment variables "GNUNET_FORCE_LOGFILE", +"GNUNET_LOG" and/or "GNUNET_FORCE_LOG": +@itemize @bullet + +@item "GNUNET_LOG" only affects the logging when no global log level is +configured by any other means (that is, the process does not explicitly +set its own log level, there are no "-L loglevel" options on command line +or in configuration files), and can be used to override the default +WARNING log level. + +@item "GNUNET_FORCE_LOG" will completely override any other log +configuration options given. + +@item "GNUNET_FORCE_LOGFILE" will completely override the location of the +file to log messages to. It should contain a relative or absolute file +name. Setting GNUNET_FORCE_LOGFILE is equivalent to passing +"--log-file=logfile" or "-l logfile" option (see below). It supports "[]" +format in file names, but not "@{@}" (see below). +@end itemize + + +Because environment variables are inherited by child processes when they +are launched, starting or re-starting the ARM service with these +variables will propagate them to all other services. + +"GNUNET_LOG" and "GNUNET_FORCE_LOG" variables must contain a specially +formatted @strong{logging definition} string, which looks like this:@ + +@c FIXME: Can we close this with [/component] instead? +@example +[component];[file];[function];[from_line[-to_line]];loglevel[/component...] +@end example + +That is, a logging definition consists of definition entries, separated by +slashes ('/'). If only one entry is present, there is no need to add a +slash to its end (although it is not forbidden either).@ All definition +fields (component, file, function, lines and loglevel) are mandatory, but +(except for the loglevel) they can be empty. An empty field means +"match anything". Note that even if fields are empty, the semicolon (';') +separators must be present.@ The loglevel field is mandatory, and must +contain one of the log level names (ERROR, WARNING, INFO or DEBUG).@ +The lines field might contain one non-negative number, in which case it +matches only one line, or a range "from_line-to_line", in which case it +matches any line in the interval [from_line;to_line] (that is, including +both start and end line).@ GNUnet mostly defaults component name to the +name of the service that is implemented in a process ('transport', +'core', 'peerinfo', etc), but logging calls can specify custom component +names using @code{GNUNET_log_from}.@ File name and function name are +provided by the compiler (__FILE__ and __FUNCTION__ built-ins). + +Component, file and function fields are interpreted as non-extended +regular expressions (GNU libc regex functions are used). Matching is +case-sensitive, "^" and "$" will match the beginning and the end of the +text. If a field is empty, its contents are automatically replaced with +a ".*" regular expression, which matches anything. Matching is done in +the default way, which means that the expression matches as long as it's +contained anywhere in the string. Thus "GNUNET_" will match both +"GNUNET_foo" and "BAR_GNUNET_BAZ". Use '^' and/or '$' to make sure that +the expression matches at the start and/or at the end of the string. +The semicolon (';') can't be escaped, and GNUnet will not use it in +component names (it can't be used in function names and file names +anyway). + +@end table + + +Every logging call in GNUnet code will be (at run time) matched against +the log definitions passed to the process. If a log definition fields are +matching the call arguments, then the call log level is compared the the +log level of that definition. If the call log level is less or equal to +the definition log level, the call is allowed to proceed. Otherwise the +logging call is forbidden, and nothing is logged. If no definitions +matched at all, GNUnet will use the global log level or (if a global log +level is not specified) will default to WARNING (that is, it will allow +the call to proceed, if its level is less or equal to the global log +level or to WARNING). + +That is, definitions are evaluated from left to right, and the first +matching definition is used to allow or deny the logging call. Thus it is +advised to place narrow definitions at the beginning of the logdef +string, and generic definitions - at the end. + +Whether a call is allowed or not is only decided the first time this +particular call is made. The evaluation result is then cached, so that +any attempts to make the same call later will be allowed or disallowed +right away. Because of that runtime log level evaluation should not +significantly affect the process performance. +Log definition parsing is only done once, at the first call to +GNUNET_log_setup () made by the process (which is usually done soon after +it starts). + +At the moment of writing there is no way to specify logging definitions +from configuration files, only via environment variables. + +At the moment GNUnet will stop processing a log definition when it +encounters an error in definition formatting or an error in regular +expression syntax, and will not report the failure in any way. + + +@c *********************************************************************** +@menu +* Examples:: +* Log files:: +* Updated behavior of GNUNET_log:: +@end menu + +@node Examples +@subsubsection Examples + +@table @asis + +@item @code{GNUNET_FORCE_LOG=";;;;DEBUG" gnunet-arm -s} Start GNUnet +process tree, running all processes with DEBUG level (one should be +careful with it, as log files will grow at alarming rate!) +@item @code{GNUNET_FORCE_LOG="core;;;;DEBUG" gnunet-arm -s} Start GNUnet +process tree, running the core service under DEBUG level (everything else +will use configured or default level). + +@item Start GNUnet process tree, allowing any logging calls from +gnunet-service-transport_validation.c (everything else will use +configured or default level). + +@example +GNUNET_FORCE_LOG=";gnunet-service-transport_validation.c;;; DEBUG" \ +gnunet-arm -s +@end example + +@item Start GNUnet process tree, allowing any logging calls from +gnunet-gnunet-service-fs_push.c (everything else will use configured or +default level). + +@example +GNUNET_FORCE_LOG="fs;gnunet-service-fs_push.c;;;DEBUG" gnunet-arm -s +@end example + +@item Start GNUnet process tree, allowing any logging calls from the +GNUNET_NETWORK_socket_select function (everything else will use +configured or default level). + +@example +GNUNET_FORCE_LOG=";;GNUNET_NETWORK_socket_select;;DEBUG" gnunet-arm -s +@end example + +@item Start GNUnet process tree, allowing any logging calls from the +components that have "transport" in their names, and are made from +function that have "send" in their names. Everything else will be allowed +to be logged only if it has WARNING level. + +@example +GNUNET_FORCE_LOG="transport.*;;.*send.*;;DEBUG/;;;;WARNING" gnunet-arm -s +@end example + +@end table + + +On Windows, one can use batch files to run GNUnet processes with special +environment variables, without affecting the whole system. Such batch +file will look like this: + +@example +set GNUNET_FORCE_LOG=;;do_transmit;;DEBUG@ gnunet-arm -s +@end example + +(note the absence of double quotes in the environment variable definition, +as opposed to earlier examples, which use the shell). +Another limitation, on Windows, GNUNET_FORCE_LOGFILE @strong{MUST} be set +in order to GNUNET_FORCE_LOG to work. + + +@cindex Log files +@node Log files +@subsubsection Log files + +GNUnet can be told to log everything into a file instead of stderr (which +is the default) using the "--log-file=logfile" or "-l logfile" option. +This option can also be passed via command line, or from the "OPTION" and +"GLOBAL_POSTFIX" configuration keys (see above). The file name passed +with this option is subject to GNUnet filename expansion. If specified in +"GLOBAL_POSTFIX", it is also subject to ARM service filename expansion, +in particular, it may contain "@{@}" (left and right curly brace) +sequence, which will be replaced by ARM with the name of the service. +This is used to keep logs from more than one service separate, while only +specifying one template containing "@{@}" in GLOBAL_POSTFIX. + +As part of a secondary file name expansion, the first occurrence of "[]" +sequence ("left square brace" followed by "right square brace") in the +file name will be replaced with a process identifier or the process when +it initializes its logging subsystem. As a result, all processes will log +into different files. This is convenient for isolating messages of a +particular process, and prevents I/O races when multiple processes try to +write into the file at the same time. This expansion is done +independently of "@{@}" expansion that ARM service does (see above). + +The log file name that is specified via "-l" can contain format characters +from the 'strftime' function family. For example, "%Y" will be replaced +with the current year. Using "basename-%Y-%m-%d.log" would include the +current year, month and day in the log file. If a GNUnet process runs for +long enough to need more than one log file, it will eventually clean up +old log files. Currently, only the last three log files (plus the current +log file) are preserved. So once the fifth log file goes into use (so +after 4 days if you use "%Y-%m-%d" as above), the first log file will be +automatically deleted. Note that if your log file name only contains "%Y", +then log files would be kept for 4 years and the logs from the first year +would be deleted once year 5 begins. If you do not use any date-related +string format codes, logs would never be automatically deleted by GNUnet. + + +@c *********************************************************************** + +@node Updated behavior of GNUNET_log +@subsubsection Updated behavior of GNUNET_log + +It's currently quite common to see constructions like this all over the +code: + +@example +#if MESH_DEBUG +GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "MESH: client disconnected\n"); +#endif +@end example + +The reason for the #if is not to avoid displaying the message when +disabled (GNUNET_ERROR_TYPE takes care of that), but to avoid the +compiler including it in the binary at all, when compiling GNUnet for +platforms with restricted storage space / memory (MIPS routers, +ARM plug computers / dev boards, etc). + +This presents several problems: the code gets ugly, hard to write and it +is very easy to forget to include the #if guards, creating non-consistent +code. A new change in GNUNET_log aims to solve these problems. + +@strong{This change requires to @file{./configure} with at least +@code{--enable-logging=verbose} to see debug messages.} + +Here is an example of code with dense debug statements: + +@example +switch (restrict_topology) @{ +case GNUNET_TESTING_TOPOLOGY_CLIQUE:#if VERBOSE_TESTING +GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, _("Blacklisting all but clique +topology\n")); #endif unblacklisted_connections = create_clique (pg, +&remove_connections, BLACKLIST, GNUNET_NO); break; case +GNUNET_TESTING_TOPOLOGY_SMALL_WORLD_RING: #if VERBOSE_TESTING GNUNET_log +(GNUNET_ERROR_TYPE_DEBUG, _("Blacklisting all but small world (ring) +topology\n")); #endif unblacklisted_connections = create_small_world_ring +(pg,&remove_connections, BLACKLIST); break; +@end example + + +Pretty hard to follow, huh? + +From now on, it is not necessary to include the #if / #endif statements to +achieve the same behavior. The GNUNET_log and GNUNET_log_from macros take +care of it for you, depending on the configure option: + +@itemize @bullet +@item If @code{--enable-logging} is set to @code{no}, the binary will +contain no log messages at all. +@item If @code{--enable-logging} is set to @code{yes}, the binary will +contain no DEBUG messages, and therefore running with -L DEBUG will have +no effect. Other messages (ERROR, WARNING, INFO, etc) will be included. +@item If @code{--enable-logging} is set to @code{verbose}, or +@code{veryverbose} the binary will contain DEBUG messages (still, it will +be neccessary to run with -L DEBUG or set the DEBUG config option to show +them). +@end itemize + + +If you are a developer: +@itemize @bullet +@item please make sure that you @code{./configure +--enable-logging=@{verbose,veryverbose@}}, so you can see DEBUG messages. +@item please remove the @code{#if} statements around @code{GNUNET_log +(GNUNET_ERROR_TYPE_DEBUG, ...)} lines, to improve the readibility of your +code. +@end itemize + +Since now activating DEBUG automatically makes it VERBOSE and activates +@strong{all} debug messages by default, you probably want to use the +https://gnunet.org/logging functionality to filter only relevant messages. +A suitable configuration could be: + +@example +$ export GNUNET_FORCE_LOG="^YOUR_SUBSYSTEM$;;;;DEBUG/;;;;WARNING" +@end example + +Which will behave almost like enabling DEBUG in that subsytem before the +change. Of course you can adapt it to your particular needs, this is only +a quick example. + +@cindex Interprocess communication API +@cindex ICP +@node Interprocess communication API (IPC) +@subsection Interprocess communication API (IPC) + +In GNUnet a variety of new message types might be defined and used in +interprocess communication, in this tutorial we use the +@code{struct AddressLookupMessage} as a example to introduce how to +construct our own message type in GNUnet and how to implement the message +communication between service and client. +(Here, a client uses the @code{struct AddressLookupMessage} as a request +to ask the server to return the address of any other peer connecting to +the service.) + + +@c *********************************************************************** +@menu +* Define new message types:: +* Define message struct:: +* Client - Establish connection:: +* Client - Initialize request message:: +* Client - Send request and receive response:: +* Server - Startup service:: +* Server - Add new handles for specified messages:: +* Server - Process request message:: +* Server - Response to client:: +* Server - Notification of clients:: +* Conversion between Network Byte Order (Big Endian) and Host Byte Order:: +@end menu + +@node Define new message types +@subsubsection Define new message types + +First of all, you should define the new message type in +@file{gnunet_protocols.h}: + +@example + // Request to look addresses of peers in server. +#define GNUNET_MESSAGE_TYPE_TRANSPORT_ADDRESS_LOOKUP 29 + // Response to the address lookup request. +#define GNUNET_MESSAGE_TYPE_TRANSPORT_ADDRESS_REPLY 30 +@end example + +@c *********************************************************************** +@node Define message struct +@subsubsection Define message struct + +After the type definition, the specified message structure should also be +described in the header file, e.g. transport.h in our case. + +@example +struct AddressLookupMessage @{ + struct GNUNET_MessageHeader header; + int32_t numeric_only GNUNET_PACKED; + struct GNUNET_TIME_AbsoluteNBO timeout; + uint32_t addrlen GNUNET_PACKED; + /* followed by 'addrlen' bytes of the actual address, then + followed by the 0-terminated name of the transport */ @}; +GNUNET_NETWORK_STRUCT_END +@end example + + +Please note @code{GNUNET_NETWORK_STRUCT_BEGIN} and @code{GNUNET_PACKED} +which both ensure correct alignment when sending structs over the network. + +@menu +@end menu + +@c *********************************************************************** +@node Client - Establish connection +@subsubsection Client - Establish connection +@c %**end of header + + +At first, on the client side, the underlying API is employed to create a +new connection to a service, in our example the transport service would be +connected. + +@example +struct GNUNET_CLIENT_Connection *client; +client = GNUNET_CLIENT_connect ("transport", cfg); +@end example + +@c *********************************************************************** +@node Client - Initialize request message +@subsubsection Client - Initialize request message +@c %**end of header + +When the connection is ready, we initialize the message. In this step, +all the fields of the message should be properly initialized, namely the +size, type, and some extra user-defined data, such as timeout, name of +transport, address and name of transport. + +@example +struct AddressLookupMessage *msg; +size_t len = sizeof (struct AddressLookupMessage) + + addressLen + + strlen (nameTrans) + + 1; +msg->header->size = htons (len); +msg->header->type = htons +(GNUNET_MESSAGE_TYPE_TRANSPORT_ADDRESS_LOOKUP); +msg->timeout = GNUNET_TIME_absolute_hton (abs_timeout); +msg->addrlen = htonl (addressLen); +char *addrbuf = (char *) &msg[1]; +memcpy (addrbuf, address, addressLen); +char *tbuf = &addrbuf[addressLen]; +memcpy (tbuf, nameTrans, strlen (nameTrans) + 1); +@end example + +Note that, here the functions @code{htonl}, @code{htons} and +@code{GNUNET_TIME_absolute_hton} are applied to convert little endian +into big endian, about the usage of the big/small edian order and the +corresponding conversion function please refer to Introduction of +Big Endian and Little Endian. + +@c *********************************************************************** +@node Client - Send request and receive response +@subsubsection Client - Send request and receive response +@c %**end of header + +@b{FIXME: This is very outdated, see the tutorial for the current API!} + +Next, the client would send the constructed message as a request to the +service and wait for the response from the service. To accomplish this +goal, there are a number of API calls that can be used. In this example, +@code{GNUNET_CLIENT_transmit_and_get_response} is chosen as the most +appropriate function to use. + +@example +GNUNET_CLIENT_transmit_and_get_response +(client, msg->header, timeout, GNUNET_YES, &address_response_processor, +arp_ctx); +@end example + +the argument @code{address_response_processor} is a function with +@code{GNUNET_CLIENT_MessageHandler} type, which is used to process the +reply message from the service. + +@node Server - Startup service +@subsubsection Server - Startup service + +After receiving the request message, we run a standard GNUnet service +startup sequence using @code{GNUNET_SERVICE_run}, as follows, + +@example +int main(int argc, char**argv) @{ + GNUNET_SERVICE_run(argc, argv, "transport" + GNUNET_SERVICE_OPTION_NONE, &run, NULL)); @} +@end example + +@c *********************************************************************** +@node Server - Add new handles for specified messages +@subsubsection Server - Add new handles for specified messages +@c %**end of header + +in the function above the argument @code{run} is used to initiate +transport service,and defined like this: + +@example +static void run (void *cls, +struct GNUNET_SERVER_Handle *serv, +const struct GNUNET_CONFIGURATION_Handle *cfg) @{ + GNUNET_SERVER_add_handlers (serv, handlers); @} +@end example + + +Here, @code{GNUNET_SERVER_add_handlers} must be called in the run +function to add new handlers in the service. The parameter +@code{handlers} is a list of @code{struct GNUNET_SERVER_MessageHandler} +to tell the service which function should be called when a particular +type of message is received, and should be defined in this way: + +@example +static struct GNUNET_SERVER_MessageHandler handlers[] = @{ + @{&handle_start, + NULL, + GNUNET_MESSAGE_TYPE_TRANSPORT_START, + 0@}, + @{&handle_send, + NULL, + GNUNET_MESSAGE_TYPE_TRANSPORT_SEND, + 0@}, + @{&handle_try_connect, + NULL, + GNUNET_MESSAGE_TYPE_TRANSPORT_TRY_CONNECT, + sizeof (struct TryConnectMessage) + @}, + @{&handle_address_lookup, + NULL, + GNUNET_MESSAGE_TYPE_TRANSPORT_ADDRESS_LOOKUP, + 0@}, + @{NULL, + NULL, + 0, + 0@} +@}; +@end example + + +As shown, the first member of the struct in the first area is a callback +function, which is called to process the specified message types, given +as the third member. The second parameter is the closure for the callback +function, which is set to @code{NULL} in most cases, and the last +parameter is the expected size of the message of this type, usually we +set it to 0 to accept variable size, for special cases the exact size of +the specified message also can be set. In addition, the terminator sign +depicted as @code{@{NULL, NULL, 0, 0@}} is set in the last aera. + +@c *********************************************************************** +@node Server - Process request message +@subsubsection Server - Process request message +@c %**end of header + +After the initialization of transport service, the request message would +be processed. Before handling the main message data, the validity of this +message should be checked out, e.g., to check whether the size of message +is correct. + +@example +size = ntohs (message->size); +if (size < sizeof (struct AddressLookupMessage)) @{ + GNUNET_break_op (0); + GNUNET_SERVER_receive_done (client, GNUNET_SYSERR); + return; @} +@end example + + +Note that, opposite to the construction method of the request message in +the client, in the server the function @code{nothl} and @code{ntohs} +should be employed during the extraction of the data from the message, so +that the data in big endian order can be converted back into little +endian order. See more in detail please refer to Introduction of +Big Endian and Little Endian. + +Moreover in this example, the name of the transport stored in the message +is a 0-terminated string, so we should also check whether the name of the +transport in the received message is 0-terminated: + +@example +nameTransport = (const char *) &address[addressLen]; +if (nameTransport[size - sizeof + (struct AddressLookupMessage) + - addressLen - 1] != '\0') @{ + GNUNET_break_op (0); + GNUNET_SERVER_receive_done (client, + GNUNET_SYSERR); + return; @} +@end example + +Here, @code{GNUNET_SERVER_receive_done} should be called to tell the +service that the request is done and can receive the next message. The +argument @code{GNUNET_SYSERR} here indicates that the service didn't +understand the request message, and the processing of this request would +be terminated. + +In comparison to the aforementioned situation, when the argument is equal +to @code{GNUNET_OK}, the service would continue to process the requst +message. + +@c *********************************************************************** +@node Server - Response to client +@subsubsection Server - Response to client +@c %**end of header + +Once the processing of current request is done, the server should give the +response to the client. A new @code{struct AddressLookupMessage} would be +produced by the server in a similar way as the client did and sent to the +client, but here the type should be +@code{GNUNET_MESSAGE_TYPE_TRANSPORT_ADDRESS_REPLY} rather than +@code{GNUNET_MESSAGE_TYPE_TRANSPORT_ADDRESS_LOOKUP} in client. +@example +struct AddressLookupMessage *msg; +size_t len = sizeof (struct AddressLookupMessage) + + addressLen + + strlen (nameTrans) + 1; +msg->header->size = htons (len); +msg->header->type = htons + (GNUNET_MESSAGE_TYPE_TRANSPORT_ADDRESS_REPLY); + +// ... + +struct GNUNET_SERVER_TransmitContext *tc; +tc = GNUNET_SERVER_transmit_context_create (client); +GNUNET_SERVER_transmit_context_append_data +(tc, + NULL, + 0, + GNUNET_MESSAGE_TYPE_TRANSPORT_ADDRESS_REPLY); +GNUNET_SERVER_transmit_context_run (tc, rtimeout); +@end example + + +Note that, there are also a number of other APIs provided to the service +to send the message. + +@c *********************************************************************** +@node Server - Notification of clients +@subsubsection Server - Notification of clients +@c %**end of header + +Often a service needs to (repeatedly) transmit notifications to a client +or a group of clients. In these cases, the client typically has once +registered for a set of events and then needs to receive a message +whenever such an event happens (until the client disconnects). The use of +a notification context can help manage message queues to clients and +handle disconnects. Notification contexts can be used to send +individualized messages to a particular client or to broadcast messages +to a group of clients. An individualized notification might look like +this: + +@example +GNUNET_SERVER_notification_context_unicast(nc, + client, + msg, + GNUNET_YES); +@end example + + +Note that after processing the original registration message for +notifications, the server code still typically needs to call +@code{GNUNET_SERVER_receive_done} so that the client can transmit further +messages to the server. + +@c *********************************************************************** +@node Conversion between Network Byte Order (Big Endian) and Host Byte Order +@subsubsection Conversion between Network Byte Order (Big Endian) and Host Byte Order +@c %** subsub? it's a referenced page on the ipc document. +@c %**end of header + +Here we can simply comprehend big endian and little endian as Network Byte +Order and Host Byte Order respectively. What is the difference between +both two? + +Usually in our host computer we store the data byte as Host Byte Order, +for example, we store a integer in the RAM which might occupies 4 Byte, +as Host Byte Order the higher Byte would be stored at the lower address +of RAM, and the lower Byte would be stored at the higher address of RAM. +However, contrast to this, Network Byte Order just take the totally +opposite way to store the data, says, it will store the lower Byte at the +lower address, and the higher Byte will stay at higher address. + +For the current communication of network, we normally exchange the +information by surveying the data package, every two host wants to +communicate with each other must send and receive data package through +network. In order to maintain the identity of data through the +transmission in the network, the order of the Byte storage must changed +before sending and after receiving the data. + +There ten convenient functions to realize the conversion of Byte Order in +GNUnet, as following: + +@table @asis + +@item uint16_t htons(uint16_t hostshort) Convert host byte order to net +byte order with short int +@item uint32_t htonl(uint32_t hostlong) Convert host byte +order to net byte order with long int +@item uint16_t ntohs(uint16_t netshort) +Convert net byte order to host byte order with short int +@item uint32_t +ntohl(uint32_t netlong) Convert net byte order to host byte order with +long int +@item unsigned long long GNUNET_ntohll (unsigned long long netlonglong) +Convert net byte order to host byte order with long long int +@item unsigned long long GNUNET_htonll (unsigned long long hostlonglong) +Convert host byte order to net byte order with long long int +@item struct GNUNET_TIME_RelativeNBO GNUNET_TIME_relative_hton +(struct GNUNET_TIME_Relative a) Convert relative time to network byte +order. +@item struct GNUNET_TIME_Relative GNUNET_TIME_relative_ntoh +(struct GNUNET_TIME_RelativeNBO a) Convert relative time from network +byte order. +@item struct GNUNET_TIME_AbsoluteNBO GNUNET_TIME_absolute_hton +(struct GNUNET_TIME_Absolute a) Convert relative time to network byte +order. +@item struct GNUNET_TIME_Absolute GNUNET_TIME_absolute_ntoh +(struct GNUNET_TIME_AbsoluteNBO a) Convert relative time from network +byte order. +@end table + +@cindex Cryptography API +@node Cryptography API +@subsection Cryptography API +@c %**end of header + +The gnunetutil APIs provides the cryptographic primitives used in GNUnet. +GNUnet uses 2048 bit RSA keys for the session key exchange and for signing +messages by peers and most other public-key operations. Most researchers +in cryptography consider 2048 bit RSA keys as secure and practically +unbreakable for a long time. The API provides functions to create a fresh +key pair, read a private key from a file (or create a new file if the +file does not exist), encrypt, decrypt, sign, verify and extraction of +the public key into a format suitable for network transmission. + +For the encryption of files and the actual data exchanged between peers +GNUnet uses 256-bit AES encryption. Fresh, session keys are negotiated +for every new connection.@ Again, there is no published technique to +break this cipher in any realistic amount of time. The API provides +functions for generation of keys, validation of keys (important for +checking that decryptions using RSA succeeded), encryption and decryption. + +GNUnet uses SHA-512 for computing one-way hash codes. The API provides +functions to compute a hash over a block in memory or over a file on disk. + +The crypto API also provides functions for randomizing a block of memory, +obtaining a single random number and for generating a permuation of the +numbers 0 to n-1. Random number generation distinguishes between WEAK and +STRONG random number quality; WEAK random numbers are pseudo-random +whereas STRONG random numbers use entropy gathered from the operating +system. + +Finally, the crypto API provides a means to deterministically generate a +1024-bit RSA key from a hash code. These functions should most likely not +be used by most applications; most importantly, +GNUNET_CRYPTO_rsa_key_create_from_hash does not create an RSA-key that +should be considered secure for traditional applications of RSA. + +@cindex Message Queue API +@node Message Queue API +@subsection Message Queue API +@c %**end of header + +@strong{ Introduction }@ +Often, applications need to queue messages that +are to be sent to other GNUnet peers, clients or services. As all of +GNUnet's message-based communication APIs, by design, do not allow +messages to be queued, it is common to implement custom message queues +manually when they are needed. However, writing very similar code in +multiple places is tedious and leads to code duplication. + +MQ (for Message Queue) is an API that provides the functionality to +implement and use message queues. We intend to eventually replace all of +the custom message queue implementations in GNUnet with MQ. + +@strong{ Basic Concepts }@ +The two most important entities in MQ are queues and envelopes. + +Every queue is backed by a specific implementation (e.g. for mesh, stream, +connection, server client, etc.) that will actually deliver the queued +messages. For convenience,@ some queues also allow to specify a list of +message handlers. The message queue will then also wait for incoming +messages and dispatch them appropriately. + +An envelope holds the the memory for a message, as well as metadata +(Where is the envelope queued? What should happen after it has been +sent?). Any envelope can only be queued in one message queue. + +@strong{ Creating Queues }@ +The following is a list of currently available message queues. Note that +to avoid layering issues, message queues for higher level APIs are not +part of @code{libgnunetutil}, but@ the respective API itself provides the +queue implementation. + +@table @asis + +@item @code{GNUNET_MQ_queue_for_connection_client} +Transmits queued messages over a @code{GNUNET_CLIENT_Connection} handle. +Also supports receiving with message handlers. + +@item @code{GNUNET_MQ_queue_for_server_client} +Transmits queued messages over a @code{GNUNET_SERVER_Client} handle. Does +not support incoming message handlers. + +@item @code{GNUNET_MESH_mq_create} Transmits queued messages over a +@code{GNUNET_MESH_Tunnel} handle. Does not support incoming message +handlers. + +@item @code{GNUNET_MQ_queue_for_callbacks} This is the most general +implementation. Instead of delivering and receiving messages with one of +GNUnet's communication APIs, implementation callbacks are called. Refer to +"Implementing Queues" for a more detailed explanation. +@end table + + +@strong{ Allocating Envelopes }@ +A GNUnet message (as defined by the GNUNET_MessageHeader) has three +parts: The size, the type, and the body. + +MQ provides macros to allocate an envelope containing a message +conveniently, automatically setting the size and type fields of the +message. + +Consider the following simple message, with the body consisting of a +single number value. +@c why the empy code function? +@code{} + +@example +struct NumberMessage @{ + /** Type: GNUNET_MESSAGE_TYPE_EXAMPLE_1 */ + struct GNUNET_MessageHeader header; + uint32_t number GNUNET_PACKED; +@}; +@end example + +An envelope containing an instance of the NumberMessage can be +constructed like this: + +@example +struct GNUNET_MQ_Envelope *ev; +struct NumberMessage *msg; +ev = GNUNET_MQ_msg (msg, GNUNET_MESSAGE_TYPE_EXAMPLE_1); +msg->number = htonl (42); +@end example + +In the above code, @code{GNUNET_MQ_msg} is a macro. The return value is +the newly allocated envelope. The first argument must be a pointer to some +@code{struct} containing a @code{struct GNUNET_MessageHeader header} +field, while the second argument is the desired message type, in host +byte order. + +The @code{msg} pointer now points to an allocated message, where the +message type and the message size are already set. The message's size is +inferred from the type of the @code{msg} pointer: It will be set to +'sizeof(*msg)', properly converted to network byte order. + +If the message body's size is dynamic, the the macro +@code{GNUNET_MQ_msg_extra} can be used to allocate an envelope whose +message has additional space allocated after the @code{msg} structure. + +If no structure has been defined for the message, +@code{GNUNET_MQ_msg_header_extra} can be used to allocate additional space +after the message header. The first argument then must be a pointer to a +@code{GNUNET_MessageHeader}. + +@strong{Envelope Properties}@ +A few functions in MQ allow to set additional properties on envelopes: + +@table @asis + +@item @code{GNUNET_MQ_notify_sent} Allows to specify a function that will +be called once the envelope's message has been sent irrevocably. +An envelope can be canceled precisely up to the@ point where the notify +sent callback has been called. + +@item @code{GNUNET_MQ_disable_corking} No corking will be used when +sending the message. Not every@ queue supports this flag, per default, +envelopes are sent with corking.@ + +@end table + + +@strong{Sending Envelopes}@ +Once an envelope has been constructed, it can be queued for sending with +@code{GNUNET_MQ_send}. + +Note that in order to avoid memory leaks, an envelope must either be sent +(the queue will free it) or destroyed explicitly with +@code{GNUNET_MQ_discard}. + +@strong{Canceling Envelopes}@ +An envelope queued with @code{GNUNET_MQ_send} can be canceled with +@code{GNUNET_MQ_cancel}. Note that after the notify sent callback has +been called, canceling a message results in undefined behavior. +Thus it is unsafe to cancel an envelope that does not have a notify sent +callback. When canceling an envelope, it is not necessary@ to call +@code{GNUNET_MQ_discard}, and the envelope can't be sent again. + +@strong{ Implementing Queues }@ +@code{TODO} + +@cindex Service API +@node Service API +@subsection Service API +@c %**end of header + +Most GNUnet code lives in the form of services. Services are processes +that offer an API for other components of the system to build on. Those +other components can be command-line tools for users, graphical user +interfaces or other services. Services provide their API using an IPC +protocol. For this, each service must listen on either a TCP port or a +UNIX domain socket; for this, the service implementation uses the server +API. This use of server is exposed directly to the users of the service +API. Thus, when using the service API, one is usually also often using +large parts of the server API. The service API provides various +convenience functions, such as parsing command-line arguments and the +configuration file, which are not found in the server API. +The dual to the service/server API is the client API, which can be used to +access services. + +The most common way to start a service is to use the +@code{GNUNET_SERVICE_run} function from the program's main function. +@code{GNUNET_SERVICE_run} will then parse the command line and +configuration files and, based on the options found there, +start the server. It will then give back control to the main +program, passing the server and the configuration to the +@code{GNUNET_SERVICE_Main} callback. @code{GNUNET_SERVICE_run} +will also take care of starting the scheduler loop. +If this is inappropriate (for example, because the scheduler loop +is already running), @code{GNUNET_SERVICE_start} and +related functions provide an alternative to @code{GNUNET_SERVICE_run}. + +When starting a service, the service_name option is used to determine +which sections in the configuration file should be used to configure the +service. A typical value here is the name of the @file{src/} +sub-directory, for example @file{statistics}. +The same string would also be given to +@code{GNUNET_CLIENT_connect} to access the service. + +Once a service has been initialized, the program should use the +@code{GNUNET_SERVICE_Main} callback to register message handlers +using @code{GNUNET_SERVER_add_handlers}. +The service will already have registered a handler for the +"TEST" message. + +@fnindex GNUNET_SERVICE_Options +The option bitfield (@code{enum GNUNET_SERVICE_Options}) +determines how a service should behave during shutdown. +There are three key strategies: + +@table @asis + +@item instant (@code{GNUNET_SERVICE_OPTION_NONE}) +Upon receiving the shutdown +signal from the scheduler, the service immediately terminates the server, +closing all existing connections with clients. +@item manual (@code{GNUNET_SERVICE_OPTION_MANUAL_SHUTDOWN}) +The service does nothing by itself +during shutdown. The main program will need to take the appropriate +action by calling GNUNET_SERVER_destroy or GNUNET_SERVICE_stop (depending +on how the service was initialized) to terminate the service. This method +is used by gnunet-service-arm and rather uncommon. +@item soft (@code{GNUNET_SERVICE_OPTION_SOFT_SHUTDOWN}) +Upon receiving the shutdown signal from the scheduler, +the service immediately tells the server to stop +listening for incoming clients. Requests from normal existing clients are +still processed and the server/service terminates once all normal clients +have disconnected. Clients that are not expected to ever disconnect (such +as clients that monitor performance values) can be marked as 'monitor' +clients using GNUNET_SERVER_client_mark_monitor. Those clients will +continue to be processed until all 'normal' clients have disconnected. +Then, the server will terminate, closing the monitor connections. +This mode is for example used by 'statistics', allowing existing 'normal' +clients to set (possibly persistent) statistic values before terminating. + +@end table + +@c *********************************************************************** +@node Optimizing Memory Consumption of GNUnet's (Multi-) Hash Maps +@subsection Optimizing Memory Consumption of GNUnet's (Multi-) Hash Maps +@c %**end of header + +A commonly used data structure in GNUnet is a (multi-)hash map. It is most +often used to map a peer identity to some data structure, but also to map +arbitrary keys to values (for example to track requests in the distributed +hash table or in file-sharing). As it is commonly used, the DHT is +actually sometimes responsible for a large share of GNUnet's overall +memory consumption (for some processes, 30% is not uncommon). The +following text documents some API quirks (and their implications for +applications) that were recently introduced to minimize the footprint of +the hash map. + + +@c *********************************************************************** +@menu +* Analysis:: +* Solution:: +* Migration:: +* Conclusion:: +* Availability:: +@end menu + +@node Analysis +@subsubsection Analysis +@c %**end of header + +The main reason for the "excessive" memory consumption by the hash map is +that GNUnet uses 512-bit cryptographic hash codes --- and the +(multi-)hash map also uses the same 512-bit 'struct GNUNET_HashCode'. As +a result, storing just the keys requires 64 bytes of memory for each key. +As some applications like to keep a large number of entries in the hash +map (after all, that's what maps are good for), 64 bytes per hash is +significant: keeping a pointer to the value and having a linked list for +collisions consume between 8 and 16 bytes, and 'malloc' may add about the +same overhead per allocation, putting us in the 16 to 32 byte per entry +ballpark. Adding a 64-byte key then triples the overall memory +requirement for the hash map. + +To make things "worse", most of the time storing the key in the hash map +is not required: it is typically already in memory elsewhere! In most +cases, the values stored in the hash map are some application-specific +struct that _also_ contains the hash. Here is a simplified example: + +@example +struct MyValue @{ +struct GNUNET_HashCode key; +unsigned int my_data; @}; + +// ... +val = GNUNET_malloc (sizeof (struct MyValue)); +val->key = key; +val->my_data = 42; +GNUNET_CONTAINER_multihashmap_put (map, &key, val, ...); +@end example + +This is a common pattern as later the entries might need to be removed, +and at that time it is convenient to have the key immediately at hand: + +@example +GNUNET_CONTAINER_multihashmap_remove (map, &val->key, val); +@end example + + +Note that here we end up with two times 64 bytes for the key, plus maybe +64 bytes total for the rest of the 'struct MyValue' and the map entry in +the hash map. The resulting redundant storage of the key increases +overall memory consumption per entry from the "optimal" 128 bytes to 192 +bytes. This is not just an extreme example: overheads in practice are +actually sometimes close to those highlighted in this example. This is +especially true for maps with a significant number of entries, as there +we tend to really try to keep the entries small. + +@c *********************************************************************** +@node Solution +@subsubsection Solution +@c %**end of header + +The solution that has now been implemented is to @strong{optionally} +allow the hash map to not make a (deep) copy of the hash but instead have +a pointer to the hash/key in the entry. This reduces the memory +consumption for the key from 64 bytes to 4 to 8 bytes. However, it can +also only work if the key is actually stored in the entry (which is the +case most of the time) and if the entry does not modify the key (which in +all of the code I'm aware of has been always the case if there key is +stored in the entry). Finally, when the client stores an entry in the +hash map, it @strong{must} provide a pointer to the key within the entry, +not just a pointer to a transient location of the key. If +the client code does not meet these requirements, the result is a dangling +pointer and undefined behavior of the (multi-)hash map API. + +@c *********************************************************************** +@node Migration +@subsubsection Migration +@c %**end of header + +To use the new feature, first check that the values contain the respective +key (and never modify it). Then, all calls to +@code{GNUNET_CONTAINER_multihashmap_put} on the respective map must be +audited and most likely changed to pass a pointer into the value's struct. +For the initial example, the new code would look like this: + +@example +struct MyValue @{ +struct GNUNET_HashCode key; +unsigned int my_data; @}; + +// ... +val = GNUNET_malloc (sizeof (struct MyValue)); +val->key = key; val->my_data = 42; +GNUNET_CONTAINER_multihashmap_put (map, &val->key, val, ...); +@end example + + +Note that @code{&val} was changed to @code{&val->key} in the argument to +the @code{put} call. This is critical as often @code{key} is on the stack +or in some other transient data structure and thus having the hash map +keep a pointer to @code{key} would not work. Only the key inside of +@code{val} has the same lifetime as the entry in the map (this must of +course be checked as well). Naturally, @code{val->key} must be +intiialized before the @code{put} call. Once all @code{put} calls have +been converted and double-checked, you can change the call to create the +hash map from + +@example +map = +GNUNET_CONTAINER_multihashmap_create (SIZE, GNUNET_NO); +@end example + +to + +@example +map = GNUNET_CONTAINER_multihashmap_create (SIZE, GNUNET_YES); +@end example + +If everything was done correctly, you now use about 60 bytes less memory +per entry in @code{map}. However, if now (or in the future) any call to +@code{put} does not ensure that the given key is valid until the entry is +removed from the map, undefined behavior is likely to be observed. + +@c *********************************************************************** +@node Conclusion +@subsubsection Conclusion +@c %**end of header + +The new optimization can is often applicable and can result in a +reduction in memory consumption of up to 30% in practice. However, it +makes the code less robust as additional invariants are imposed on the +multi hash map client. Thus applications should refrain from enabling the +new mode unless the resulting performance increase is deemed significant +enough. In particular, it should generally not be used in new code (wait +at least until benchmarks exist). + +@c *********************************************************************** +@node Availability +@subsubsection Availability +@c %**end of header + +The new multi hash map code was committed in SVN 24319 (will be in GNUnet +0.9.4). Various subsystems (transport, core, dht, file-sharing) were +previously audited and modified to take advantage of the new capability. +In particular, memory consumption of the file-sharing service is expected +to drop by 20-30% due to this change. + + +@cindex CONTAINER_MDLL API +@node CONTAINER_MDLL API +@subsection CONTAINER_MDLL API +@c %**end of header + +This text documents the GNUNET_CONTAINER_MDLL API. The +GNUNET_CONTAINER_MDLL API is similar to the GNUNET_CONTAINER_DLL API in +that it provides operations for the construction and manipulation of +doubly-linked lists. The key difference to the (simpler) DLL-API is that +the MDLL-version allows a single element (instance of a "struct") to be +in multiple linked lists at the same time. + +Like the DLL API, the MDLL API stores (most of) the data structures for +the doubly-linked list with the respective elements; only the 'head' and +'tail' pointers are stored "elsewhere" --- and the application needs to +provide the locations of head and tail to each of the calls in the +MDLL API. The key difference for the MDLL API is that the "next" and +"previous" pointers in the struct can no longer be simply called "next" +and "prev" --- after all, the element may be in multiple doubly-linked +lists, so we cannot just have one "next" and one "prev" pointer! + +The solution is to have multiple fields that must have a name of the +format "next_XX" and "prev_XX" where "XX" is the name of one of the +doubly-linked lists. Here is a simple example: + +@example +struct MyMultiListElement @{ + struct MyMultiListElement *next_ALIST; + struct MyMultiListElement *prev_ALIST; + struct MyMultiListElement *next_BLIST; + struct MyMultiListElement *prev_BLIST; + void + *data; +@}; +@end example + + +Note that by convention, we use all-uppercase letters for the list names. +In addition, the program needs to have a location for the head and tail +pointers for both lists, for example: + +@example +static struct MyMultiListElement *head_ALIST; +static struct MyMultiListElement *tail_ALIST; +static struct MyMultiListElement *head_BLIST; +static struct MyMultiListElement *tail_BLIST; +@end example + + +Using the MDLL-macros, we can now insert an element into the ALIST: + +@example +GNUNET_CONTAINER_MDLL_insert (ALIST, head_ALIST, tail_ALIST, element); +@end example + + +Passing "ALIST" as the first argument to MDLL specifies which of the +next/prev fields in the 'struct MyMultiListElement' should be used. The +extra "ALIST" argument and the "_ALIST" in the names of the +next/prev-members are the only differences between the MDDL and DLL-API. +Like the DLL-API, the MDLL-API offers functions for inserting (at head, +at tail, after a given element) and removing elements from the list. +Iterating over the list should be done by directly accessing the +"next_XX" and/or "prev_XX" members. + +@cindex Automatic Restart Manager +@cindex ARM +@node Automatic Restart Manager (ARM) +@section Automatic Restart Manager (ARM) +@c %**end of header + +GNUnet's Automated Restart Manager (ARM) is the GNUnet service responsible +for system initialization and service babysitting. ARM starts and halts +services, detects configuration changes and restarts services impacted by +the changes as needed. It's also responsible for restarting services in +case of crashes and is planned to incorporate automatic debugging for +diagnosing service crashes providing developers insights about crash +reasons. The purpose of this document is to give GNUnet developer an idea +about how ARM works and how to interact with it. + +@menu +* Basic functionality:: +* Key configuration options:: +* ARM - Availability:: +* Reliability:: +@end menu + +@c *********************************************************************** +@node Basic functionality +@subsection Basic functionality +@c %**end of header + +@itemize @bullet +@item ARM source code can be found under "src/arm".@ Service processes are +managed by the functions in "gnunet-service-arm.c" which is controlled +with "gnunet-arm.c" (main function in that file is ARM's entry point). + +@item The functions responsible for communicating with ARM , starting and +stopping services -including ARM service itself- are provided by the +ARM API "arm_api.c".@ Function: GNUNET_ARM_connect() returns to the caller +an ARM handle after setting it to the caller's context (configuration and +scheduler in use). This handle can be used afterwards by the caller to +communicate with ARM. Functions GNUNET_ARM_start_service() and +GNUNET_ARM_stop_service() are used for starting and stopping services +respectively. + +@item A typical example of using these basic ARM services can be found in +file test_arm_api.c. The test case connects to ARM, starts it, then uses +it to start a service "resolver", stops the "resolver" then stops "ARM". +@end itemize + +@c *********************************************************************** +@node Key configuration options +@subsection Key configuration options +@c %**end of header + +Configurations for ARM and services should be available in a .conf file +(As an example, see test_arm_api_data.conf). When running ARM, the +configuration file to use should be passed to the command: + +@example +$ gnunet-arm -s -c configuration_to_use.conf +@end example + +If no configuration is passed, the default configuration file will be used +(see GNUNET_PREFIX/share/gnunet/defaults.conf which is created from +contrib/defaults.conf).@ Each of the services is having a section starting +by the service name between square brackets, for example: "[arm]". +The following options configure how ARM configures or interacts with the +various services: + +@table @asis + +@item PORT Port number on which the service is listening for incoming TCP +connections. ARM will start the services should it notice a request at +this port. + +@item HOSTNAME Specifies on which host the service is deployed. Note +that ARM can only start services that are running on the local system +(but will not check that the hostname matches the local machine name). +This option is used by the @code{gnunet_client_lib.h} implementation to +determine which system to connect to. The default is "localhost". + +@item BINARY The name of the service binary file. + +@item OPTIONS To be passed to the service. + +@item PREFIX A command to pre-pend to the actual command, for example, +running a service with "valgrind" or "gdb" + +@item DEBUG Run in debug mode (much verbosity). + +@item AUTOSTART ARM will listen to UNIX domain socket and/or TCP port of +the service and start the service on-demand. + +@item FORCESTART ARM will always start this service when the peer +is started. + +@item ACCEPT_FROM IPv4 addresses the service accepts connections from. + +@item ACCEPT_FROM6 IPv6 addresses the service accepts connections from. + +@end table + + +Options that impact the operation of ARM overall are in the "[arm]" +section. ARM is a normal service and has (except for AUTOSTART) all of the +options that other services do. In addition, ARM has the +following options: + +@table @asis + +@item GLOBAL_PREFIX Command to be pre-pended to all services that are +going to run. + +@item GLOBAL_POSTFIX Global option that will be supplied to all the +services that are going to run. + +@end table + +@c *********************************************************************** +@node ARM - Availability +@subsection ARM - Availability +@c %**end of header + +As mentioned before, one of the features provided by ARM is starting +services on demand. Consider the example of one service "client" that +wants to connect to another service a "server". The "client" will ask ARM +to run the "server". ARM starts the "server". The "server" starts +listening to incoming connections. The "client" will establish a +connection with the "server". And then, they will start to communicate +together.@ One problem with that scheme is that it's slow!@ +The "client" service wants to communicate with the "server" service at +once and is not willing wait for it to be started and listening to +incoming connections before serving its request.@ One solution for that +problem will be that ARM starts all services as default services. That +solution will solve the problem, yet, it's not quite practical, for some +services that are going to be started can never be used or are going to +be used after a relatively long time.@ +The approach followed by ARM to solve this problem is as follows: + +@itemize @bullet + +@item For each service having a PORT field in the configuration file and +that is not one of the default services ( a service that accepts incoming +connections from clients), ARM creates listening sockets for all addresses +associated with that service. + +@item The "client" will immediately establish a connection with +the "server". + +@item ARM --- pretending to be the "server" --- will listen on the +respective port and notice the incoming connection from the "client" +(but not accept it), instead + +@item Once there is an incoming connection, ARM will start the "server", +passing on the listen sockets (now, the service is started and can do its +work). + +@item Other client services now can directly connect directly to the +"server". + +@end itemize + +@c *********************************************************************** +@node Reliability +@subsection Reliability + +One of the features provided by ARM, is the automatic restart of crashed +services.@ ARM needs to know which of the running services died. Function +"gnunet-service-arm.c/maint_child_death()" is responsible for that. The +function is scheduled to run upon receiving a SIGCHLD signal. The +function, then, iterates ARM's list of services running and monitors +which service has died (crashed). For all crashing services, ARM restarts +them.@ +Now, considering the case of a service having a serious problem causing it +to crash each time it's started by ARM. If ARM keeps blindly restarting +such a service, we are going to have the pattern: +start-crash-restart-crash-restart-crash and so forth!! Which is of course +not practical.@ +For that reason, ARM schedules the service to be restarted after waiting +for some delay that grows exponentially with each crash/restart of that +service.@ To clarify the idea, considering the following example: + +@itemize @bullet + +@item Service S crashed. + +@item ARM receives the SIGCHLD and inspects its list of services to find +the dead one(s). + +@item ARM finds S dead and schedules it for restarting after "backoff" +time which is initially set to 1ms. ARM will double the backoff time +correspondent to S (now backoff(S) = 2ms) + +@item Because there is a severe problem with S, it crashed again. + +@item Again ARM receives the SIGCHLD and detects that it's S again that's +crashed. ARM schedules it for restarting but after its new backoff time +(which became 2ms), and doubles its backoff time (now backoff(S) = 4). + +@item and so on, until backoff(S) reaches a certain threshold +(@code{EXPONENTIAL_BACKOFF_THRESHOLD} is set to half an hour), +after reaching it, backoff(S) will remain half an hour, +hence ARM won't be busy for a lot of time trying to restart a +problematic service. +@end itemize + +@cindex TRANSPORT Subsystem +@node TRANSPORT Subsystem +@section TRANSPORT Subsystem +@c %**end of header + +This chapter documents how the GNUnet transport subsystem works. The +GNUnet transport subsystem consists of three main components: the +transport API (the interface used by the rest of the system to access the +transport service), the transport service itself (most of the interesting +functions, such as choosing transports, happens here) and the transport +plugins. A transport plugin is a concrete implementation for how two +GNUnet peers communicate; many plugins exist, for example for +communication via TCP, UDP, HTTP, HTTPS and others. Finally, the +transport subsystem uses supporting code, especially the NAT/UPnP +library to help with tasks such as NAT traversal. + +Key tasks of the transport service include: + +@itemize @bullet + +@item Create our HELLO message, notify clients and neighbours if our HELLO +changes (using NAT library as necessary) + +@item Validate HELLOs from other peers (send PING), allow other peers to +validate our HELLO's addresses (send PONG) + +@item Upon request, establish connections to other peers (using address +selection from ATS subsystem) and maintain them (again using PINGs and +PONGs) as long as desired + +@item Accept incoming connections, give ATS service the opportunity to +switch communication channels + +@item Notify clients about peers that have connected to us or that have +been disconnected from us + +@item If a (stateful) connection goes down unexpectedly (without explicit +DISCONNECT), quickly attempt to recover (without notifying clients) but do +notify clients quickly if reconnecting fails + +@item Send (payload) messages arriving from clients to other peers via +transport plugins and receive messages from other peers, forwarding +those to clients + +@item Enforce inbound traffic limits (using flow-control if it is +applicable); outbound traffic limits are enforced by CORE, not by us (!) + +@item Enforce restrictions on P2P connection as specified by the blacklist +configuration and blacklisting clients +@end itemize + +Note that the term "clients" in the list above really refers to the +GNUnet-CORE service, as CORE is typically the only client of the +transport service. + +@menu +* Address validation protocol:: +@end menu + +@node Address validation protocol +@subsection Address validation protocol +@c %**end of header + +This section documents how the GNUnet transport service validates +connections with other peers. It is a high-level description of the +protocol necessary to understand the details of the implementation. It +should be noted that when we talk about PING and PONG messages in this +section, we refer to transport-level PING and PONG messages, which are +different from core-level PING and PONG messages (both in implementation +and function). + +The goal of transport-level address validation is to minimize the chances +of a successful man-in-the-middle attack against GNUnet peers on the +transport level. Such an attack would not allow the adversary to decrypt +the P2P transmissions, but a successful attacker could at least measure +traffic volumes and latencies (raising the adversaries capablities by +those of a global passive adversary in the worst case). The scenarios we +are concerned about is an attacker, Mallory, giving a @code{HELLO} to +Alice that claims to be for Bob, but contains Mallory's IP address +instead of Bobs (for some transport). +Mallory would then forward the traffic to Bob (by initiating a +connection to Bob and claiming to be Alice). As a further +complication, the scheme has to work even if say Alice is behind a NAT +without traversal support and hence has no address of her own (and thus +Alice must always initiate the connection to Bob). + +An additional constraint is that @code{HELLO} messages do not contain a +cryptographic signature since other peers must be able to edit +(i.e. remove) addresses from the @code{HELLO} at any time (this was +not true in GNUnet 0.8.x). A basic @strong{assumption} is that each peer +knows the set of possible network addresses that it @strong{might} +be reachable under (so for example, the external IP address of the +NAT plus the LAN address(es) with the respective ports). + +The solution is the following. If Alice wants to validate that a given +address for Bob is valid (i.e. is actually established @strong{directly} +with the intended target), she sends a PING message over that connection +to Bob. Note that in this case, Alice initiated the connection so only +Alice knows which address was used for sure (Alice may be behind NAT, so +whatever address Bob sees may not be an address Alice knows she has). +Bob checks that the address given in the @code{PING} is actually one +of Bob's addresses (ie: does not belong to Mallory), and if it is, +sends back a @code{PONG} (with a signature that says that Bob +owns/uses the address from the @code{PING}). +Alice checks the signature and is happy if it is valid and the address +in the @code{PONG} is the address Alice used. +This is similar to the 0.8.x protocol where the @code{HELLO} contained a +signature from Bob for each address used by Bob. +Here, the purpose code for the signature is +@code{GNUNET_SIGNATURE_PURPOSE_TRANSPORT_PONG_OWN}. After this, Alice will +remember Bob's address and consider the address valid for a while (12h in +the current implementation). Note that after this exchange, Alice only +considers Bob's address to be valid, the connection itself is not +considered 'established'. In particular, Alice may have many addresses +for Bob that Alice considers valid. + +@c TODO: reference Footnotes so that I don't have to duplicate the +@c footnotes or add them to an index at the end. Is this possible at +@c all in Texinfo? +The @code{PONG} message is protected with a nonce/challenge against replay +attacks@footnote{@uref{http://en.wikipedia.org/wiki/Replay_attack, replay}} +and uses an expiration time for the signature (but those are almost +implementation details). + +@cindex NAT library +@node NAT library +@section NAT library +@c %**end of header + +The goal of the GNUnet NAT library is to provide a general-purpose API for +NAT traversal @strong{without} third-party support. So protocols that +involve contacting a third peer to help establish a connection between +two peers are outside of the scope of this API. That does not mean that +GNUnet doesn't support involving a third peer (we can do this with the +distance-vector transport or using application-level protocols), it just +means that the NAT API is not concerned with this possibility. The API is +written so that it will work for IPv6-NAT in the future as well as +current IPv4-NAT. Furthermore, the NAT API is always used, even for peers +that are not behind NAT --- in that case, the mapping provided is simply +the identity. + +NAT traversal is initiated by calling @code{GNUNET_NAT_register}. Given a +set of addresses that the peer has locally bound to (TCP or UDP), the NAT +library will return (via callback) a (possibly longer) list of addresses +the peer @strong{might} be reachable under. Internally, depending on the +configuration, the NAT library will try to punch a hole (using UPnP) or +just "know" that the NAT was manually punched and generate the respective +external IP address (the one that should be globally visible) based on +the given information. + +The NAT library also supports ICMP-based NAT traversal. Here, the other +peer can request connection-reversal by this peer (in this special case, +the peer is even allowed to configure a port number of zero). If the NAT +library detects a connection-reversal request, it returns the respective +target address to the client as well. It should be noted that +connection-reversal is currently only intended for TCP, so other plugins +@strong{must} pass @code{NULL} for the reversal callback. Naturally, the +NAT library also supports requesting connection reversal from a remote +peer (@code{GNUNET_NAT_run_client}). + +Once initialized, the NAT handle can be used to test if a given address is +possibly a valid address for this peer (@code{GNUNET_NAT_test_address}). +This is used for validating our addresses when generating PONGs. + +Finally, the NAT library contains an API to test if our NAT configuration +is correct. Using @code{GNUNET_NAT_test_start} @strong{before} binding to +the respective port, the NAT library can be used to test if the +configuration works. The test function act as a local client, initialize +the NAT traversal and then contact a @code{gnunet-nat-server} (running by +default on @code{gnunet.org}) and ask for a connection to be established. +This way, it is easy to test if the current NAT configuration is valid. + +@node Distance-Vector plugin +@section Distance-Vector plugin +@c %**end of header + +The Distance Vector (DV) transport is a transport mechanism that allows +peers to act as relays for each other, thereby connecting peers that would +otherwise be unable to connect. This gives a larger connection set to +applications that may work better with more peers to choose from (for +example, File Sharing and/or DHT). + +The Distance Vector transport essentially has two functions. The first is +"gossiping" connection information about more distant peers to directly +connected peers. The second is taking messages intended for non-directly +connected peers and encapsulating them in a DV wrapper that contains the +required information for routing the message through forwarding peers. Via +gossiping, optimal routes through the known DV neighborhood are discovered +and utilized and the message encapsulation provides some benefits in +addition to simply getting the message from the correct source to the +proper destination. + +The gossiping function of DV provides an up to date routing table of +peers that are available up to some number of hops. We call this a +fisheye view of the network (like a fish, nearby objects are known while +more distant ones unknown). Gossip messages are sent only to directly +connected peers, but they are sent about other knowns peers within the +"fisheye distance". Whenever two peers connect, they immediately gossip +to each other about their appropriate other neighbors. They also gossip +about the newly connected peer to previously +connected neighbors. In order to keep the routing tables up to date, +disconnect notifications are propogated as gossip as well (because +disconnects may not be sent/received, timeouts are also used remove +stagnant routing table entries). + +Routing of messages via DV is straightforward. When the DV transport is +notified of a message destined for a non-direct neighbor, the appropriate +forwarding peer is selected, and the base message is encapsulated in a DV +message which contains information about the initial peer and the intended +recipient. At each forwarding hop, the initial peer is validated (the +forwarding peer ensures that it has the initial peer in its neighborhood, +otherwise the message is dropped). Next the base message is +re-encapsulated in a new DV message for the next hop in the forwarding +chain (or delivered to the current peer, if it has arrived at the +destination). + +Assume a three peer network with peers Alice, Bob and Carol. Assume that + +@example +Alice <-> Bob and Bob <-> Carol +@end example + +@noindent +are direct (e.g. over TCP or UDP transports) connections, but that +Alice cannot directly connect to Carol. +This may be the case due to NAT or firewall restrictions, or perhaps +based on one of the peers respective configurations. If the Distance +Vector transport is enabled on all three peers, it will automatically +discover (from the gossip protocol) that Alice and Carol can connect via +Bob and provide a "virtual" Alice <-> Carol connection. Routing between +Alice and Carol happens as follows; Alice creates a message destined for +Carol and notifies the DV transport about it. The DV transport at Alice +looks up Carol in the routing table and finds that the message must be +sent through Bob for Carol. The message is encapsulated setting Alice as +the initiator and Carol as the destination and sent to Bob. Bob receives +the messages, verifies that both Alice and Carol are known to Bob, and +re-wraps the message in a new DV message for Carol. +The DV transport at Carol receives this message, unwraps the original +message, and delivers it to Carol as though it came directly from Alice. + +@cindex SMTP plugin +@node SMTP plugin +@section SMTP plugin +@c %**end of header + +This section describes the new SMTP transport plugin for GNUnet as it +exists in the 0.7.x and 0.8.x branch. SMTP support is currently not +available in GNUnet 0.9.x. This page also describes the transport layer +abstraction (as it existed in 0.7.x and 0.8.x) in more detail and gives +some benchmarking results. The performance results presented are quite +old and maybe outdated at this point. + +@itemize @bullet +@item Why use SMTP for a peer-to-peer transport? +@item SMTPHow does it work? +@item How do I configure my peer? +@item How do I test if it works? +@item How fast is it? +@item Is there any additional documentation? +@end itemize + + +@menu +* Why use SMTP for a peer-to-peer transport?:: +* How does it work?:: +* How do I configure my peer?:: +* How do I test if it works?:: +* How fast is it?:: +@end menu + +@node Why use SMTP for a peer-to-peer transport? +@subsection Why use SMTP for a peer-to-peer transport? +@c %**end of header + +There are many reasons why one would not want to use SMTP: + +@itemize @bullet +@item SMTP is using more bandwidth than TCP, UDP or HTTP +@item SMTP has a much higher latency. +@item SMTP requires significantly more computation (encoding and decoding +time) for the peers. +@item SMTP is significantly more complicated to configure. +@item SMTP may be abused by tricking GNUnet into sending mail to@ +non-participating third parties. +@end itemize + +So why would anybody want to use SMTP? +@itemize @bullet +@item SMTP can be used to contact peers behind NAT boxes (in virtual +private networks). +@item SMTP can be used to circumvent policies that limit or prohibit +peer-to-peer traffic by masking as "legitimate" traffic. +@item SMTP uses E-mail addresses which are independent of a specific IP, +which can be useful to address peers that use dynamic IP addresses. +@item SMTP can be used to initiate a connection (e.g. initial address +exchange) and peers can then negotiate the use of a more efficient +protocol (e.g. TCP) for the actual communication. +@end itemize + +In summary, SMTP can for example be used to send a message to a peer +behind a NAT box that has a dynamic IP to tell the peer to establish a +TCP connection to a peer outside of the private network. Even an +extraordinary overhead for this first message would be irrelevant in this +type of situation. + +@node How does it work? +@subsection How does it work? +@c %**end of header + +When a GNUnet peer needs to send a message to another GNUnet peer that has +advertised (only) an SMTP transport address, GNUnet base64-encodes the +message and sends it in an E-mail to the advertised address. The +advertisement contains a filter which is placed in the E-mail header, +such that the receiving host can filter the tagged E-mails and forward it +to the GNUnet peer process. The filter can be specified individually by +each peer and be changed over time. This makes it impossible to censor +GNUnet E-mail messages by searching for a generic filter. + +@node How do I configure my peer? +@subsection How do I configure my peer? +@c %**end of header + +First, you need to configure @code{procmail} to filter your inbound E-mail +for GNUnet traffic. The GNUnet messages must be delivered into a pipe, for +example @code{/tmp/gnunet.smtp}. You also need to define a filter that is +used by @command{procmail} to detect GNUnet messages. You are free to +choose whichever filter you like, but you should make sure that it does +not occur in your other E-mail. In our example, we will use +@code{X-mailer: GNUnet}. The @code{~/.procmailrc} configuration file then +looks like this: + +@example +:0: +* ^X-mailer: GNUnet +/tmp/gnunet.smtp +# where do you want your other e-mail delivered to +# (default: /var/spool/mail/) +:0: /var/spool/mail/ +@end example + +After adding this file, first make sure that your regular E-mail still +works (e.g. by sending an E-mail to yourself). Then edit the GNUnet +configuration. In the section @code{SMTP} you need to specify your E-mail +address under @code{EMAIL}, your mail server (for outgoing mail) under +@code{SERVER}, the filter (X-mailer: GNUnet in the example) under +@code{FILTER} and the name of the pipe under @code{PIPE}.@ The completed +section could then look like this: + +@example +EMAIL = me@@mail.gnu.org MTU = 65000 SERVER = mail.gnu.org:25 FILTER = +"X-mailer: GNUnet" PIPE = /tmp/gnunet.smtp +@end example + +Finally, you need to add @code{smtp} to the list of @code{TRANSPORTS} in +the @code{GNUNETD} section. GNUnet peers will use the E-mail address that +you specified to contact your peer until the advertisement times out. +Thus, if you are not sure if everything works properly or if you are not +planning to be online for a long time, you may want to configure this +timeout to be short, e.g. just one hour. For this, set +@code{HELLOEXPIRES} to @code{1} in the @code{GNUNETD} section. + +This should be it, but you may probably want to test it first. + +@node How do I test if it works? +@subsection How do I test if it works? +@c %**end of header + +Any transport can be subjected to some rudimentary tests using the +@code{gnunet-transport-check} tool. The tool sends a message to the local +node via the transport and checks that a valid message is received. While +this test does not involve other peers and can not check if firewalls or +other network obstacles prohibit proper operation, this is a great +testcase for the SMTP transport since it tests pretty much nearly all of +the functionality. + +@code{gnunet-transport-check} should only be used without running +@code{gnunetd} at the same time. By default, @code{gnunet-transport-check} +tests all transports that are specified in the configuration file. But +you can specifically test SMTP by giving the option +@code{--transport=smtp}. + +Note that this test always checks if a transport can receive and send. +While you can configure most transports to only receive or only send +messages, this test will only work if you have configured the transport +to send and receive messages. + +@node How fast is it? +@subsection How fast is it? +@c %**end of header + +We have measured the performance of the UDP, TCP and SMTP transport layer +directly and when used from an application using the GNUnet core. +Measureing just the transport layer gives the better view of the actual +overhead of the protocol, whereas evaluating the transport from the +application puts the overhead into perspective from a practical point of +view. + +The loopback measurements of the SMTP transport were performed on three +different machines spanning a range of modern SMTP configurations. We +used a PIII-800 running RedHat 7.3 with the Purdue Computer Science +configuration which includes filters for spam. We also used a Xenon 2 GHZ +with a vanilla RedHat 8.0 sendmail configuration. Furthermore, we used +qmail on a PIII-1000 running Sorcerer GNU Linux (SGL). The numbers for +UDP and TCP are provided using the SGL configuration. The qmail benchmark +uses qmail's internal filtering whereas the sendmail benchmarks relies on +procmail to filter and deliver the mail. We used the transport layer to +send a message of b bytes (excluding transport protocol headers) directly +to the local machine. This way, network latency and packet loss on the +wire have no impact on the timings. n messages were sent sequentially over +the transport layer, sending message i+1 after the i-th message was +received. All messages were sent over the same connection and the time to +establish the connection was not taken into account since this overhead is +miniscule in practice --- as long as a connection is used for a +significant number of messages. + +@multitable @columnfractions .20 .15 .15 .15 .15 .15 +@headitem Transport @tab UDP @tab TCP @tab SMTP (Purdue sendmail) +@tab SMTP (RH 8.0) @tab SMTP (SGL qmail) +@item 11 bytes @tab 31 ms @tab 55 ms @tab 781 s @tab 77 s @tab 24 s +@item 407 bytes @tab 37 ms @tab 62 ms @tab 789 s @tab 78 s @tab 25 s +@item 1,221 bytes @tab 46 ms @tab 73 ms @tab 804 s @tab 78 s @tab 25 s +@end multitable + +The benchmarks show that UDP and TCP are, as expected, both significantly +faster compared with any of the SMTP services. Among the SMTP +implementations, there can be significant differences depending on the +SMTP configuration. Filtering with an external tool like procmail that +needs to re-parse its configuration for each mail can be very expensive. +Applying spam filters can also significantly impact the performance of +the underlying SMTP implementation. The microbenchmark shows that SMTP +can be a viable solution for initiating peer-to-peer sessions: a couple of +seconds to connect to a peer are probably not even going to be noticed by +users. The next benchmark measures the possible throughput for a +transport. Throughput can be measured by sending multiple messages in +parallel and measuring packet loss. Note that not only UDP but also the +TCP transport can actually loose messages since the TCP implementation +drops messages if the @code{write} to the socket would block. While the +SMTP protocol never drops messages itself, it is often so +slow that only a fraction of the messages can be sent and received in the +given time-bounds. For this benchmark we report the message loss after +allowing t time for sending m messages. If messages were not sent (or +received) after an overall timeout of t, they were considered lost. The +benchmark was performed using two Xeon 2 GHZ machines running RedHat 8.0 +with sendmail. The machines were connected with a direct 100 MBit ethernet +connection.@ Figures udp1200, tcp1200 and smtp-MTUs show that the +throughput for messages of size 1,200 octects is 2,343 kbps, 3,310 kbps +and 6 kbps for UDP, TCP and SMTP respectively. The high per-message +overhead of SMTP can be improved by increasing the MTU, for example, an +MTU of 12,000 octets improves the throughput to 13 kbps as figure +smtp-MTUs shows. Our research paper) has some more details on the +benchmarking results. + +@cindex Bluetooth plugin +@node Bluetooth plugin +@section Bluetooth plugin +@c %**end of header + +This page describes the new Bluetooth transport plugin for GNUnet. The +plugin is still in the testing stage so don't expect it to work +perfectly. If you have any questions or problems just post them here or +ask on the IRC channel. + +@itemize @bullet +@item What do I need to use the Bluetooth plugin transport? +@item BluetoothHow does it work? +@item What possible errors should I be aware of? +@item How do I configure my peer? +@item How can I test it? +@end itemize + +@menu +* What do I need to use the Bluetooth plugin transport?:: +* How does it work2?:: +* What possible errors should I be aware of?:: +* How do I configure my peer2?:: +* How can I test it?:: +* The implementation of the Bluetooth transport plugin:: +@end menu + +@node What do I need to use the Bluetooth plugin transport? +@subsection What do I need to use the Bluetooth plugin transport? +@c %**end of header + +If you are a GNU/Linux user and you want to use the Bluetooth +transport plugin you should install the +@command{BlueZ development libraries} (if they aren't already +installed). +For instructions about how to install the libraries you should +check out the BlueZ site +(@uref{http://www.bluez.org/, http://www.bluez.org}). If you don't know if +you have the necesarry libraries, don't worry, just run the GNUnet +configure script and you will be able to see a notification at the end +which will warn you if you don't have the necessary libraries. + +If you are a Windows user you should have installed the +@emph{MinGW}/@emph{MSys2} with the latest updates (especially the +@emph{ws2bth} header). If this is your first build of GNUnet on Windows +you should check out the SBuild repository. It will semi-automatically +assembles a @emph{MinGW}/@emph{MSys2} installation with a lot of extra +packages which are needed for the GNUnet build. So this will ease your +work!@ Finally you just have to be sure that you have the correct drivers +for your Bluetooth device installed and that your device is on and in a +discoverable mode. The Windows Bluetooth Stack supports only the RFCOMM +protocol so we cannot turn on your device programatically! + +@c FIXME: Change to unique title +@node How does it work2? +@subsection How does it work2? +@c %**end of header + +The Bluetooth transport plugin uses virtually the same code as the WLAN +plugin and only the helper binary is different. The helper takes a single +argument, which represents the interface name and is specified in the +configuration file. Here are the basic steps that are followed by the +helper binary used on GNU/Linux: + +@itemize @bullet +@item it verifies if the name corresponds to a Bluetooth interface name +@item it verifies if the iterface is up (if it is not, it tries to bring +it up) +@item it tries to enable the page and inquiry scan in order to make the +device discoverable and to accept incoming connection requests +@emph{The above operations require root access so you should start the +transport plugin with root privileges.} +@item it finds an available port number and registers a SDP service which +will be used to find out on which port number is the server listening on +and switch the socket in listening mode +@item it sends a HELLO message with its address +@item finally it forwards traffic from the reading sockets to the STDOUT +and from the STDIN to the writing socket +@end itemize + +Once in a while the device will make an inquiry scan to discover the +nearby devices and it will send them randomly HELLO messages for peer +discovery. + +@node What possible errors should I be aware of? +@subsection What possible errors should I be aware of? +@c %**end of header + +@emph{This section is dedicated for GNU/Linux users} + +Well there are many ways in which things could go wrong but I will try to +present some tools that you could use to debug and some scenarios. + +@itemize @bullet + +@item @code{bluetoothd -n -d} : use this command to enable logging in the +foreground and to print the logging messages + +@item @code{hciconfig}: can be used to configure the Bluetooth devices. +If you run it without any arguments it will print information about the +state of the interfaces. So if you receive an error that the device +couldn't be brought up you should try to bring it manually and to see if +it works (use @code{hciconfig -a hciX up}). If you can't and the +Bluetooth address has the form 00:00:00:00:00:00 it means that there is +something wrong with the D-Bus daemon or with the Bluetooth daemon. Use +@code{bluetoothd} tool to see the logs + +@item @code{sdptool} can be used to control and interogate SDP servers. +If you encounter problems regarding the SDP server (like the SDP server is +down) you should check out if the D-Bus daemon is running correctly and to +see if the Bluetooth daemon started correctly(use @code{bluetoothd} tool). +Also, sometimes the SDP service could work but somehow the device couldn't +register his service. Use @code{sdptool browse [dev-address]} to see if +the service is registered. There should be a service with the name of the +interface and GNUnet as provider. + +@item @code{hcitool} : another useful tool which can be used to configure +the device and to send some particular commands to it. + +@item @code{hcidump} : could be used for low level debugging +@end itemize + +@c FIXME: A more unique name +@node How do I configure my peer2? +@subsection How do I configure my peer2? +@c %**end of header + +On GNU/Linux, you just have to be sure that the interface name +corresponds to the one that you want to use. +Use the @code{hciconfig} tool to check that. +By default it is set to hci0 but you can change it. + +A basic configuration looks like this: + +@example +[transport-bluetooth] +# Name of the interface (typically hciX) +INTERFACE = hci0 +# Real hardware, no testing +TESTMODE = 0 TESTING_IGNORE_KEYS = ACCEPT_FROM; +@end example + +In order to use the Bluetooth transport plugin when the transport service +is started, you must add the plugin name to the default transport service +plugins list. For example: + +@example +[transport] ... PLUGINS = dns bluetooth ... +@end example + +If you want to use only the Bluetooth plugin set +@emph{PLUGINS = bluetooth} + +On Windows, you cannot specify which device to use. The only thing that +you should do is to add @emph{bluetooth} on the plugins list of the +transport service. + +@node How can I test it? +@subsection How can I test it? +@c %**end of header + +If you have two Bluetooth devices on the same machine and you are using +GNU/Linux you must: + +@itemize @bullet + +@item create two different file configuration (one which will use the +first interface (@emph{hci0}) and the other which will use the second +interface (@emph{hci1})). Let's name them @emph{peer1.conf} and +@emph{peer2.conf}. + +@item run @emph{gnunet-peerinfo -c peerX.conf -s} in order to generate the +peers private keys. The @strong{X} must be replace with 1 or 2. + +@item run @emph{gnunet-arm -c peerX.conf -s -i=transport} in order to +start the transport service. (Make sure that you have "bluetooth" on the +transport plugins list if the Bluetooth transport service doesn't start.) + +@item run @emph{gnunet-peerinfo -c peer1.conf -s} to get the first peer's +ID. If you already know your peer ID (you saved it from the first +command), this can be skipped. + +@item run @emph{gnunet-transport -c peer2.conf -p=PEER1_ID -s} to start +sending data for benchmarking to the other peer. + +@end itemize + + +This scenario will try to connect the second peer to the first one and +then start sending data for benchmarking. + +On Windows you cannot test the plugin functionality using two Bluetooth +devices from the same machine because after you install the drivers there +will occur some conflicts between the Bluetooth stacks. (At least that is +what happend on my machine : I wasn't able to use the Bluesoleil stack and +the WINDCOMM one in the same time). + +If you have two different machines and your configuration files are good +you can use the same scenario presented on the begining of this section. + +Another way to test the plugin functionality is to create your own +application which will use the GNUnet framework with the Bluetooth +transport service. + +@node The implementation of the Bluetooth transport plugin +@subsection The implementation of the Bluetooth transport plugin +@c %**end of header + +This page describes the implementation of the Bluetooth transport plugin. + +First I want to remind you that the Bluetooth transport plugin uses +virtually the same code as the WLAN plugin and only the helper binary is +different. Also the scope of the helper binary from the Bluetooth +transport plugin is the same as the one used for the wlan transport +plugin: it acceses the interface and then it forwards traffic in both +directions between the Bluetooth interface and stdin/stdout of the +process involved. + +The Bluetooth plugin transport could be used both on GNU/Linux and Windows +platforms. + +@itemize @bullet +@item Linux functionality +@item Windows functionality +@item Pending Features +@end itemize + + + +@menu +* Linux functionality:: +* THE INITIALIZATION:: +* THE LOOP:: +* Details about the broadcast implementation:: +* Windows functionality:: +* Pending features:: +@end menu + +@node Linux functionality +@subsubsection Linux functionality +@c %**end of header + +In order to implement the plugin functionality on GNU/Linux I +used the BlueZ stack. +For the communication with the other devices I used the RFCOMM +protocol. Also I used the HCI protocol to gain some control over the +device. The helper binary takes a single argument (the name of the +Bluetooth interface) and is separated in two stages: + +@c %** 'THE INITIALIZATION' should be in bigger letters or stand out, not +@c %** starting a new section? +@node THE INITIALIZATION +@subsubsection THE INITIALIZATION + +@itemize @bullet +@item first, it checks if we have root privilegies +(@emph{Remember that we need to have root privilegies in order to be able +to bring the interface up if it is down or to change its state.}). + +@item second, it verifies if the interface with the given name exists. + +@strong{If the interface with that name exists and it is a Bluetooth +interface:} + +@item it creates a RFCOMM socket which will be used for listening and call +the @emph{open_device} method + +On the @emph{open_device} method: +@itemize @bullet +@item creates a HCI socket used to send control events to the the device +@item searches for the device ID using the interface name +@item saves the device MAC address +@item checks if the interface is down and tries to bring it UP +@item checks if the interface is in discoverable mode and tries to make it +discoverable +@item closes the HCI socket and binds the RFCOMM one +@item switches the RFCOMM socket in listening mode +@item registers the SDP service (the service will be used by the other +devices to get the port on which this device is listening on) +@end itemize + +@item drops the root privilegies + +@strong{If the interface is not a Bluetooth interface the helper exits +with a suitable error} +@end itemize + +@c %** Same as for @node entry above +@node THE LOOP +@subsubsection THE LOOP + +The helper binary uses a list where it saves all the connected neighbour +devices (@emph{neighbours.devices}) and two buffers (@emph{write_pout} and +@emph{write_std}). The first message which is send is a control message +with the device's MAC address in order to announce the peer presence to +the neighbours. Here are a short description of what happens in the main +loop: + +@itemize @bullet +@item Every time when it receives something from the STDIN it processes +the data and saves the message in the first buffer (@emph{write_pout}). +When it has something in the buffer, it gets the destination address from +the buffer, searches the destination address in the list (if there is no +connection with that device, it creates a new one and saves it to the +list) and sends the message. +@item Every time when it receives something on the listening socket it +accepts the connection and saves the socket on a list with the reading +sockets. @item Every time when it receives something from a reading +socket it parses the message, verifies the CRC and saves it in the +@emph{write_std} buffer in order to be sent later to the STDOUT. +@end itemize + +So in the main loop we use the select function to wait until one of the +file descriptor saved in one of the two file descriptors sets used is +ready to use. The first set (@emph{rfds}) represents the reading set and +it could contain the list with the reading sockets, the STDIN file +descriptor or the listening socket. The second set (@emph{wfds}) is the +writing set and it could contain the sending socket or the STDOUT file +descriptor. After the select function returns, we check which file +descriptor is ready to use and we do what is supposed to do on that kind +of event. @emph{For example:} if it is the listening socket then we +accept a new connection and save the socket in the reading list; if it is +the STDOUT file descriptor, then we write to STDOUT the message from the +@emph{write_std} buffer. + +To find out on which port a device is listening on we connect to the local +SDP server and searche the registered service for that device. + +@emph{You should be aware of the fact that if the device fails to connect +to another one when trying to send a message it will attempt one more +time. If it fails again, then it skips the message.} +@emph{Also you should know that the transport Bluetooth plugin has +support for @strong{broadcast messages}.} + +@node Details about the broadcast implementation +@subsubsection Details about the broadcast implementation +@c %**end of header + +First I want to point out that the broadcast functionality for the CONTROL +messages is not implemented in a conventional way. Since the inquiry scan +time is too big and it will take some time to send a message to all the +discoverable devices I decided to tackle the problem in a different way. +Here is how I did it: + +@itemize @bullet +@item If it is the first time when I have to broadcast a message I make an +inquiry scan and save all the devices' addresses to a vector. +@item After the inquiry scan ends I take the first address from the list +and I try to connect to it. If it fails, I try to connect to the next one. +If it succeeds, I save the socket to a list and send the message to the +device. +@item When I have to broadcast another message, first I search on the list +for a new device which I'm not connected to. If there is no new device on +the list I go to the beginning of the list and send the message to the +old devices. After 5 cycles I make a new inquiry scan to check out if +there are new discoverable devices and save them to the list. If there +are no new discoverable devices I reset the cycling counter and go again +through the old list and send messages to the devices saved in it. +@end itemize + +@strong{Therefore}: + +@itemize @bullet +@item every time when I have a broadcast message I look up on the list +for a new device and send the message to it +@item if I reached the end of the list for 5 times and I'm connected to +all the devices from the list I make a new inquiry scan. +@emph{The number of the list's cycles after an inquiry scan could be +increased by redefining the MAX_LOOPS variable} +@item when there are no new devices I send messages to the old ones. +@end itemize + +Doing so, the broadcast control messages will reach the devices but with +delay. + +@emph{NOTICE:} When I have to send a message to a certain device first I +check on the broadcast list to see if we are connected to that device. If +not we try to connect to it and in case of success we save the address and +the socket on the list. If we are already connected to that device we +simply use the socket. + +@node Windows functionality +@subsubsection Windows functionality +@c %**end of header + +For Windows I decided to use the Microsoft Bluetooth stack which has the +advantage of coming standard from Windows XP SP2. The main disadvantage is +that it only supports the RFCOMM protocol so we will not be able to have +a low level control over the Bluetooth device. Therefore it is the user +responsability to check if the device is up and in the discoverable mode. +Also there are no tools which could be used for debugging in order to read +the data coming from and going to a Bluetooth device, which obviously +hindered my work. Another thing that slowed down the implementation of the +plugin (besides that I wasn't too accomodated with the win32 API) was that +there were some bugs on MinGW regarding the Bluetooth. Now they are solved +but you should keep in mind that you should have the latest updates +(especially the @emph{ws2bth} header). + +Besides the fact that it uses the Windows Sockets, the Windows +implemenation follows the same principles as the GNU/Linux one: + +@itemize @bullet +@item It has a initalization part where it initializes the +Windows Sockets, creates a RFCOMM socket which will be binded and switched +to the listening mode and registers a SDP service. In the Microsoft +Bluetooth API there are two ways to work with the SDP: +@itemize @bullet +@item an easy way which works with very simple service records +@item a hard way which is useful when you need to update or to delete the +record +@end itemize +@end itemize + +Since I only needed the SDP service to find out on which port the device +is listening on and that did not change, I decided to use the easy way. +In order to register the service I used the @emph{WSASetService} function +and I generated the @emph{Universally Unique Identifier} with the +@emph{guidgen.exe} Windows's tool. + +In the loop section the only difference from the GNU/Linux implementation +is that I used the @code{GNUNET_NETWORK} library for +functions like @emph{accept}, @emph{bind}, @emph{connect} or +@emph{select}. I decided to use the +@code{GNUNET_NETWORK} library because I also needed to interact +with the STDIN and STDOUT handles and on Windows +the select function is only defined for sockets, +and it will not work for arbitrary file handles. + +Another difference between GNU/Linux and Windows implementation is that in +GNU/Linux, the Bluetooth address is represented in 48 bits +while in Windows is represented in 64 bits. +Therefore I had to do some changes on @emph{plugin_transport_wlan} header. + +Also, currently on Windows the Bluetooth plugin doesn't have support for +broadcast messages. When it receives a broadcast message it will skip it. + +@node Pending features +@subsubsection Pending features +@c %**end of header + +@itemize @bullet +@item Implement the broadcast functionality on Windows @emph{(currently +working on)} +@item Implement a testcase for the helper :@ @emph{The testcase +consists of a program which emaluates the plugin and uses the helper. It +will simulate connections, disconnections and data transfers.} +@end itemize + +If you have a new idea about a feature of the plugin or suggestions about +how I could improve the implementation you are welcome to comment or to +contact me. + +@node WLAN plugin +@section WLAN plugin +@c %**end of header + +This section documents how the wlan transport plugin works. Parts which +are not implemented yet or could be better implemented are described at +the end. + +@cindex ATS Subsystem +@node ATS Subsystem +@section ATS Subsystem +@c %**end of header + +ATS stands for "automatic transport selection", and the function of ATS in +GNUnet is to decide on which address (and thus transport plugin) should +be used for two peers to communicate, and what bandwidth limits should be +imposed on such an individual connection. To help ATS make an informed +decision, higher-level services inform the ATS service about their +requirements and the quality of the service rendered. The ATS service +also interacts with the transport service to be appraised of working +addresses and to communicate its resource allocation decisions. Finally, +the ATS service's operation can be observed using a monitoring API. + +The main logic of the ATS service only collects the available addresses, +their performance characteristics and the applications requirements, but +does not make the actual allocation decision. This last critical step is +left to an ATS plugin, as we have implemented (currently three) different +allocation strategies which differ significantly in their performance and +maturity, and it is still unclear if any particular plugin is generally +superior. + +@cindex CORE Subsystem +@node CORE Subsystem +@section CORE Subsystem +@c %**end of header + +The CORE subsystem in GNUnet is responsible for securing link-layer +communications between nodes in the GNUnet overlay network. CORE builds +on the TRANSPORT subsystem which provides for the actual, insecure, +unreliable link-layer communication (for example, via UDP or WLAN), and +then adds fundamental security to the connections: + +@itemize @bullet +@item confidentiality with so-called perfect forward secrecy; we use +ECDHE@footnote{@uref{http://en.wikipedia.org/wiki/Elliptic_curve_Diffie%E2%80%93Hellman, Elliptic-curve Diffie---Hellman}} +powered by Curve25519 +@footnote{@uref{http://cr.yp.to/ecdh.html, Curve25519}} for the key +exchange and then use symmetric encryption, encrypting with both AES-256 +@footnote{@uref{http://en.wikipedia.org/wiki/Rijndael, AES-256}} and +Twofish @footnote{@uref{http://en.wikipedia.org/wiki/Twofish, Twofish}} +@item @uref{http://en.wikipedia.org/wiki/Authentication, authentication} +is achieved by signing the ephemeral keys using Ed25519 +@footnote{@uref{http://ed25519.cr.yp.to/, Ed25519}}, a deterministic +variant of ECDSA +@footnote{@uref{http://en.wikipedia.org/wiki/ECDSA, ECDSA}} +@item integrity protection (using SHA-512 +@footnote{@uref{http://en.wikipedia.org/wiki/SHA-2, SHA-512}} to do +encrypt-then-MAC +@footnote{@uref{http://en.wikipedia.org/wiki/Authenticated_encryption, encrypt-then-MAC}}) +@item Replay +@footnote{@uref{http://en.wikipedia.org/wiki/Replay_attack, replay}} +protection (using nonces, timestamps, challenge-response, +message counters and ephemeral keys) +@item liveness (keep-alive messages, timeout) +@end itemize + +@menu +* Limitations:: +* When is a peer "connected"?:: +* libgnunetcore:: +* The CORE Client-Service Protocol:: +* The CORE Peer-to-Peer Protocol:: +@end menu + +@cindex core subsystem limitations +@node Limitations +@subsection Limitations +@c %**end of header + +CORE does not perform +@uref{http://en.wikipedia.org/wiki/Routing, routing}; using CORE it is +only possible to communicate with peers that happen to already be +"directly" connected with each other. CORE also does not have an +API to allow applications to establish such "direct" connections --- for +this, applications can ask TRANSPORT, but TRANSPORT might not be able to +establish a "direct" connection. The TOPOLOGY subsystem is responsible for +trying to keep a few "direct" connections open at all times. Applications +that need to talk to particular peers should use the CADET subsystem, as +it can establish arbitrary "indirect" connections. + +Because CORE does not perform routing, CORE must only be used directly by +applications that either perform their own routing logic (such as +anonymous file-sharing) or that do not require routing, for example +because they are based on flooding the network. CORE communication is +unreliable and delivery is possibly out-of-order. Applications that +require reliable communication should use the CADET service. Each +application can only queue one message per target peer with the CORE +service at any time; messages cannot be larger than approximately +63 kilobytes. If messages are small, CORE may group multiple messages +(possibly from different applications) prior to encryption. If permitted +by the application (using the @uref{http://baus.net/on-tcp_cork/, cork} +option), CORE may delay transmissions to facilitate grouping of multiple +small messages. If cork is not enabled, CORE will transmit the message as +soon as TRANSPORT allows it (TRANSPORT is responsible for limiting +bandwidth and congestion control). CORE does not allow flow control; +applications are expected to process messages at line-speed. If flow +control is needed, applications should use the CADET service. + +@cindex when is a peer connected +@node When is a peer "connected"? +@subsection When is a peer "connected"? +@c %**end of header + +In addition to the security features mentioned above, CORE also provides +one additional key feature to applications using it, and that is a +limited form of protocol-compatibility checking. CORE distinguishes +between TRANSPORT-level connections (which enable communication with other +peers) and application-level connections. Applications using the CORE API +will (typically) learn about application-level connections from CORE, and +not about TRANSPORT-level connections. When a typical application uses +CORE, it will specify a set of message types +(from @code{gnunet_protocols.h}) that it understands. CORE will then +notify the application about connections it has with other peers if and +only if those applications registered an intersecting set of message +types with their CORE service. Thus, it is quite possible that CORE only +exposes a subset of the established direct connections to a particular +application --- and different applications running above CORE might see +different sets of connections at the same time. + +A special case are applications that do not register a handler for any +message type. +CORE assumes that these applications merely want to monitor connections +(or "all" messages via other callbacks) and will notify those applications +about all connections. This is used, for example, by the +@code{gnunet-core} command-line tool to display the active connections. +Note that it is also possible that the TRANSPORT service has more active +connections than the CORE service, as the CORE service first has to +perform a key exchange with connecting peers before exchanging information +about supported message types and notifying applications about the new +connection. + +@cindex libgnunetcore +@node libgnunetcore +@subsection libgnunetcore +@c %**end of header + +The CORE API (defined in @file{gnunet_core_service.h}) is the basic +messaging API used by P2P applications built using GNUnet. It provides +applications the ability to send and receive encrypted messages to the +peer's "directly" connected neighbours. + +As CORE connections are generally "direct" connections,@ applications must +not assume that they can connect to arbitrary peers this way, as "direct" +connections may not always be possible. Applications using CORE are +notified about which peers are connected. Creating new "direct" +connections must be done using the TRANSPORT API. + +The CORE API provides unreliable, out-of-order delivery. While the +implementation tries to ensure timely, in-order delivery, both message +losses and reordering are not detected and must be tolerated by the +application. Most important, the core will NOT perform retransmission if +messages could not be delivered. + +Note that CORE allows applications to queue one message per connected +peer. The rate at which each connection operates is influenced by the +preferences expressed by local application as well as restrictions +imposed by the other peer. Local applications can express their +preferences for particular connections using the "performance" API of the +ATS service. + +Applications that require more sophisticated transmission capabilities +such as TCP-like behavior, or if you intend to send messages to arbitrary +remote peers, should use the CADET API. + +The typical use of the CORE API is to connect to the CORE service using +@code{GNUNET_CORE_connect}, process events from the CORE service (such as +peers connecting, peers disconnecting and incoming messages) and send +messages to connected peers using +@code{GNUNET_CORE_notify_transmit_ready}. Note that applications must +cancel pending transmission requests if they receive a disconnect event +for a peer that had a transmission pending; furthermore, queueing more +than one transmission request per peer per application using the +service is not permitted. + +The CORE API also allows applications to monitor all communications of the +peer prior to encryption (for outgoing messages) or after decryption (for +incoming messages). This can be useful for debugging, diagnostics or to +establish the presence of cover traffic (for anonymity). As monitoring +applications are often not interested in the payload, the monitoring +callbacks can be configured to only provide the message headers (including +the message type and size) instead of copying the full data stream to the +monitoring client. + +The init callback of the @code{GNUNET_CORE_connect} function is called +with the hash of the public key of the peer. This public key is used to +identify the peer globally in the GNUnet network. Applications are +encouraged to check that the provided hash matches the hash that they are +using (as theoretically the application may be using a different +configuration file with a different private key, which would result in +hard to find bugs). + +As with most service APIs, the CORE API isolates applications from crashes +of the CORE service. If the CORE service crashes, the application will see +disconnect events for all existing connections. Once the connections are +re-established, the applications will be receive matching connect events. + +@cindex core clinet-service protocol +@node The CORE Client-Service Protocol +@subsection The CORE Client-Service Protocol +@c %**end of header + +This section describes the protocol between an application using the CORE +service (the client) and the CORE service process itself. + + +@menu +* Setup2:: +* Notifications:: +* Sending:: +@end menu + +@node Setup2 +@subsubsection Setup2 +@c %**end of header + +When a client connects to the CORE service, it first sends a +@code{InitMessage} which specifies options for the connection and a set of +message type values which are supported by the application. The options +bitmask specifies which events the client would like to be notified about. +The options include: + +@table @asis +@item GNUNET_CORE_OPTION_NOTHING No notifications +@item GNUNET_CORE_OPTION_STATUS_CHANGE Peers connecting and disconnecting +@item GNUNET_CORE_OPTION_FULL_INBOUND All inbound messages (after +decryption) with full payload +@item GNUNET_CORE_OPTION_HDR_INBOUND Just the @code{MessageHeader} +of all inbound messages +@item GNUNET_CORE_OPTION_FULL_OUTBOUND All outbound +messages (prior to encryption) with full payload +@item GNUNET_CORE_OPTION_HDR_OUTBOUND Just the @code{MessageHeader} of all +outbound messages +@end table + +Typical applications will only monitor for connection status changes. + +The CORE service responds to the @code{InitMessage} with an +@code{InitReplyMessage} which contains the peer's identity. Afterwards, +both CORE and the client can send messages. + +@node Notifications +@subsubsection Notifications +@c %**end of header + +The CORE will send @code{ConnectNotifyMessage}s and +@code{DisconnectNotifyMessage}s whenever peers connect or disconnect from +the CORE (assuming their type maps overlap with the message types +registered by the client). When the CORE receives a message that matches +the set of message types specified during the @code{InitMessage} (or if +monitoring is enabled in for inbound messages in the options), it sends a +@code{NotifyTrafficMessage} with the peer identity of the sender and the +decrypted payload. The same message format (except with +@code{GNUNET_MESSAGE_TYPE_CORE_NOTIFY_OUTBOUND} for the message type) is +used to notify clients monitoring outbound messages; here, the peer +identity given is that of the receiver. + +@node Sending +@subsubsection Sending +@c %**end of header + +When a client wants to transmit a message, it first requests a +transmission slot by sending a @code{SendMessageRequest} which specifies +the priority, deadline and size of the message. Note that these values +may be ignored by CORE. When CORE is ready for the message, it answers +with a @code{SendMessageReady} response. The client can then transmit the +payload with a @code{SendMessage} message. Note that the actual message +size in the @code{SendMessage} is allowed to be smaller than the size in +the original request. A client may at any time send a fresh +@code{SendMessageRequest}, which then superceeds the previous +@code{SendMessageRequest}, which is then no longer valid. The client can +tell which @code{SendMessageRequest} the CORE service's +@code{SendMessageReady} message is for as all of these messages contain a +"unique" request ID (based on a counter incremented by the client +for each request). + +@cindex CORE Peer-to-Peer Protocol +@node The CORE Peer-to-Peer Protocol +@subsection The CORE Peer-to-Peer Protocol +@c %**end of header + + +@menu +* Creating the EphemeralKeyMessage:: +* Establishing a connection:: +* Encryption and Decryption:: +* Type maps:: +@end menu + +@cindex EphemeralKeyMessage creation +@node Creating the EphemeralKeyMessage +@subsubsection Creating the EphemeralKeyMessage +@c %**end of header + +When the CORE service starts, each peer creates a fresh ephemeral (ECC) +public-private key pair and signs the corresponding +@code{EphemeralKeyMessage} with its long-term key (which we usually call +the peer's identity; the hash of the public long term key is what results +in a @code{struct GNUNET_PeerIdentity} in all GNUnet APIs. The ephemeral +key is ONLY used for an ECDHE@footnote{@uref{http://en.wikipedia.org/wiki/Elliptic_curve_Diffie%E2%80%93Hellman, Elliptic-curve Diffie---Hellman}} +exchange by the CORE service to establish symmetric session keys. A peer +will use the same @code{EphemeralKeyMessage} for all peers for +@code{REKEY_FREQUENCY}, which is usually 12 hours. After that time, it +will create a fresh ephemeral key (forgetting the old one) and broadcast +the new @code{EphemeralKeyMessage} to all connected peers, resulting in +fresh symmetric session keys. Note that peers independently decide on +when to discard ephemeral keys; it is not a protocol violation to discard +keys more often. Ephemeral keys are also never stored to disk; restarting +a peer will thus always create a fresh ephemeral key. The use of ephemeral +keys is what provides @uref{http://en.wikipedia.org/wiki/Forward_secrecy, forward secrecy}. + +Just before transmission, the @code{EphemeralKeyMessage} is patched to +reflect the current sender_status, which specifies the current state of +the connection from the point of view of the sender. The possible values +are: + +@itemize @bullet +@item @code{KX_STATE_DOWN} Initial value, never used on the network +@item @code{KX_STATE_KEY_SENT} We sent our ephemeral key, do not know the +key of the other peer +@item @code{KX_STATE_KEY_RECEIVED} This peer has received a valid +ephemeral key of the other peer, but we are waiting for the other peer to +confirm it's authenticity (ability to decode) via challenge-response. +@item @code{KX_STATE_UP} The connection is fully up from the point of +view of the sender (now performing keep-alives) +@item @code{KX_STATE_REKEY_SENT} The sender has initiated a rekeying +operation; the other peer has so far failed to confirm a working +connection using the new ephemeral key +@end itemize + +@node Establishing a connection +@subsubsection Establishing a connection +@c %**end of header + +Peers begin their interaction by sending a @code{EphemeralKeyMessage} to +the other peer once the TRANSPORT service notifies the CORE service about +the connection. +A peer receiving an @code{EphemeralKeyMessage} with a status +indicating that the sender does not have the receiver's ephemeral key, the +receiver's @code{EphemeralKeyMessage} is sent in response. +Additionally, if the receiver has not yet confirmed the authenticity of +the sender, it also sends an (encrypted)@code{PingMessage} with a +challenge (and the identity of the target) to the other peer. Peers +receiving a @code{PingMessage} respond with an (encrypted) +@code{PongMessage} which includes the challenge. Peers receiving a +@code{PongMessage} check the challenge, and if it matches set the +connection to @code{KX_STATE_UP}. + +@node Encryption and Decryption +@subsubsection Encryption and Decryption +@c %**end of header + +All functions related to the key exchange and encryption/decryption of +messages can be found in @file{gnunet-service-core_kx.c} (except for the +cryptographic primitives, which are in @file{util/crypto*.c}). +Given the key material from ECDHE, a Key derivation function +@footnote{@uref{https://en.wikipedia.org/wiki/Key_derivation_function, Key derivation function}} +is used to derive two pairs of encryption and decryption keys for AES-256 +and TwoFish, as well as initialization vectors and authentication keys +(for HMAC@footnote{@uref{https://en.wikipedia.org/wiki/HMAC, HMAC}}). +The HMAC is computed over the encrypted payload. +Encrypted messages include an iv_seed and the HMAC in the header. + +Each encrypted message in the CORE service includes a sequence number and +a timestamp in the encrypted payload. The CORE service remembers the +largest observed sequence number and a bit-mask which represents which of +the previous 32 sequence numbers were already used. +Messages with sequence numbers lower than the largest observed sequence +number minus 32 are discarded. Messages with a timestamp that is less +than @code{REKEY_TOLERANCE} off (5 minutes) are also discarded. This of +course means that system clocks need to be reasonably synchronized for +peers to be able to communicate. Additionally, as the ephemeral key +changes every 12 hours, a peer would not even be able to decrypt messages +older than 12 hours. + +@node Type maps +@subsubsection Type maps +@c %**end of header + +Once an encrypted connection has been established, peers begin to exchange +type maps. Type maps are used to allow the CORE service to determine which +(encrypted) connections should be shown to which applications. A type map +is an array of 65536 bits representing the different types of messages +understood by applications using the CORE service. Each CORE service +maintains this map, simply by setting the respective bit for each message +type supported by any of the applications using the CORE service. Note +that bits for message types embedded in higher-level protocols (such as +MESH) will not be included in these type maps. + +Typically, the type map of a peer will be sparse. Thus, the CORE service +attempts to compress its type map using @code{gzip}-style compression +("deflate") prior to transmission. However, if the compression fails to +compact the map, the map may also be transmitted without compression +(resulting in @code{GNUNET_MESSAGE_TYPE_CORE_COMPRESSED_TYPE_MAP} or +@code{GNUNET_MESSAGE_TYPE_CORE_BINARY_TYPE_MAP} messages respectively). +Upon receiving a type map, the respective CORE service notifies +applications about the connection to the other peer if they support any +message type indicated in the type map (or no message type at all). +If the CORE service experience a connect or disconnect event from an +application, it updates its type map (setting or unsetting the respective +bits) and notifies its neighbours about the change. +The CORE services of the neighbours then in turn generate connect and +disconnect events for the peer that sent the type map for their respective +applications. As CORE messages may be lost, the CORE service confirms +receiving a type map by sending back a +@code{GNUNET_MESSAGE_TYPE_CORE_CONFIRM_TYPE_MAP}. If such a confirmation +(with the correct hash of the type map) is not received, the sender will +retransmit the type map (with exponential back-off). + +@cindex CADET Subsystem +@node CADET Subsystem +@section CADET Subsystem + +The CADET subsystem in GNUnet is responsible for secure end-to-end +communications between nodes in the GNUnet overlay network. CADET builds +on the CORE subsystem which provides for the link-layer communication and +then adds routing, forwarding and additional security to the connections. +CADET offers the same cryptographic services as CORE, but on an +end-to-end level. This is done so peers retransmitting traffic on behalf +of other peers cannot access the payload data. + +@itemize @bullet +@item CADET provides confidentiality with so-called perfect forward +secrecy; we use ECDHE powered by Curve25519 for the key exchange and then +use symmetric encryption, encrypting with both AES-256 and Twofish +@item authentication is achieved by signing the ephemeral keys using +Ed25519, a deterministic variant of ECDSA +@item integrity protection (using SHA-512 to do encrypt-then-MAC, although +only 256 bits are sent to reduce overhead) +@item replay protection (using nonces, timestamps, challenge-response, +message counters and ephemeral keys) +@item liveness (keep-alive messages, timeout) +@end itemize + +Additional to the CORE-like security benefits, CADET offers other +properties that make it a more universal service than CORE. + +@itemize @bullet +@item CADET can establish channels to arbitrary peers in GNUnet. If a +peer is not immediately reachable, CADET will find a path through the +network and ask other peers to retransmit the traffic on its behalf. +@item CADET offers (optional) reliability mechanisms. In a reliable +channel traffic is guaranteed to arrive complete, unchanged and in-order. +@item CADET takes care of flow and congestion control mechanisms, not +allowing the sender to send more traffic than the receiver or the network +are able to process. +@end itemize + +@menu +* libgnunetcadet:: +@end menu + +@cindex libgnunetcadet +@node libgnunetcadet +@subsection libgnunetcadet + + +The CADET API (defined in @file{gnunet_cadet_service.h}) is the +messaging API used by P2P applications built using GNUnet. +It provides applications the ability to send and receive encrypted +messages to any peer participating in GNUnet. +The API is heavily base on the CORE API. + +CADET delivers messages to other peers in "channels". +A channel is a permanent connection defined by a destination peer +(identified by its public key) and a port number. +Internally, CADET tunnels all channels towards a destiantion peer +using one session key and relays the data on multiple "connections", +independent from the channels. + +Each channel has optional paramenters, the most important being the +reliability flag. +Should a message get lost on TRANSPORT/CORE level, if a channel is +created with as reliable, CADET will retransmit the lost message and +deliver it in order to the destination application. + +To communicate with other peers using CADET, it is necessary to first +connect to the service using @code{GNUNET_CADET_connect}. +This function takes several parameters in form of callbacks, to allow the +client to react to various events, like incoming channels or channels that +terminate, as well as specify a list of ports the client wishes to listen +to (at the moment it is not possible to start listening on further ports +once connected, but nothing prevents a client to connect several times to +CADET, even do one connection per listening port). +The function returns a handle which has to be used for any further +interaction with the service. + +To connect to a remote peer a client has to call the +@code{GNUNET_CADET_channel_create} function. The most important parameters +given are the remote peer's identity (it public key) and a port, which +specifies which application on the remote peer to connect to, similar to +TCP/UDP ports. CADET will then find the peer in the GNUnet network and +establish the proper low-level connections and do the necessary key +exchanges to assure and authenticated, secure and verified communication. +Similar to @code{GNUNET_CADET_connect},@code{GNUNET_CADET_create_channel} +returns a handle to interact with the created channel. + +For every message the client wants to send to the remote application, +@code{GNUNET_CADET_notify_transmit_ready} must be called, indicating the +channel on which the message should be sent and the size of the message +(but not the message itself!). Once CADET is ready to send the message, +the provided callback will fire, and the message contents are provided to +this callback. + +Please note the CADET does not provide an explicit notification of when a +channel is connected. In loosely connected networks, like big wireless +mesh networks, this can take several seconds, even minutes in the worst +case. To be alerted when a channel is online, a client can call +@code{GNUNET_CADET_notify_transmit_ready} immediately after +@code{GNUNET_CADET_create_channel}. When the callback is activated, it +means that the channel is online. The callback can give 0 bytes to CADET +if no message is to be sent, this is ok. + +If a transmission was requested but before the callback fires it is no +longer needed, it can be cancelled with +@code{GNUNET_CADET_notify_transmit_ready_cancel}, which uses the handle +given back by @code{GNUNET_CADET_notify_transmit_ready}. +As in the case of CORE, only one message can be requested at a time: a +client must not call @code{GNUNET_CADET_notify_transmit_ready} again until +the callback is called or the request is cancelled. + +When a channel is no longer needed, a client can call +@code{GNUNET_CADET_channel_destroy} to get rid of it. +Note that CADET will try to transmit all pending traffic before notifying +the remote peer of the destruction of the channel, including +retransmitting lost messages if the channel was reliable. + +Incoming channels, channels being closed by the remote peer, and traffic +on any incoming or outgoing channels are given to the client when CADET +executes the callbacks given to it at the time of +@code{GNUNET_CADET_connect}. + +Finally, when an application no longer wants to use CADET, it should call +@code{GNUNET_CADET_disconnect}, but first all channels and pending +transmissions must be closed (otherwise CADET will complain). + +@cindex NSE Subsystem +@node NSE Subsystem +@section NSE Subsystem + + +NSE stands for @dfn{Network Size Estimation}. The NSE subsystem provides +other subsystems and users with a rough estimate of the number of peers +currently participating in the GNUnet overlay. +The computed value is not a precise number as producing a precise number +in a decentralized, efficient and secure way is impossible. +While NSE's estimate is inherently imprecise, NSE also gives the expected +range. For a peer that has been running in a stable network for a +while, the real network size will typically (99.7% of the time) be in the +range of [2/3 estimate, 3/2 estimate]. We will now give an overview of the +algorithm used to calculate the estimate; +all of the details can be found in this technical report. + +@c FIXME: link to the report. + +@menu +* Motivation:: +* Principle:: +* libgnunetnse:: +* The NSE Client-Service Protocol:: +* The NSE Peer-to-Peer Protocol:: +@end menu + +@node Motivation +@subsection Motivation + + +Some subsytems, like DHT, need to know the size of the GNUnet network to +optimize some parameters of their own protocol. The decentralized nature +of GNUnet makes efficient and securely counting the exact number of peers +infeasable. Although there are several decentralized algorithms to count +the number of peers in a system, so far there is none to do so securely. +Other protocols may allow any malicious peer to manipulate the final +result or to take advantage of the system to perform +@dfn{Denial of Service} (DoS) attacks against the network. +GNUnet's NSE protocol avoids these drawbacks. + + + +@menu +* Security:: +@end menu + +@cindex NSE security +@cindex nse security +@node Security +@subsubsection Security + + +The NSE subsystem is designed to be resilient against these attacks. +It uses @uref{http://en.wikipedia.org/wiki/Proof-of-work_system, proofs of work} +to prevent one peer from impersonating a large number of participants, +which would otherwise allow an adversary to artifically inflate the +estimate. +The DoS protection comes from the time-based nature of the protocol: +the estimates are calculated periodically and out-of-time traffic is +either ignored or stored for later retransmission by benign peers. +In particular, peers cannot trigger global network communication at will. + +@cindex NSE principle +@cindex nse principle +@node Principle +@subsection Principle + + +The algorithm calculates the estimate by finding the globally closest +peer ID to a random, time-based value. + +The idea is that the closer the ID is to the random value, the more +"densely packed" the ID space is, and therefore, more peers are in the +network. + + + +@menu +* Example:: +* Algorithm:: +* Target value:: +* Timing:: +* Controlled Flooding:: +* Calculating the estimate:: +@end menu + +@node Example +@subsubsection Example + + +Suppose all peers have IDs between 0 and 100 (our ID space), and the +random value is 42. +If the closest peer has the ID 70 we can imagine that the average +"distance" between peers is around 30 and therefore the are around 3 +peers in the whole ID space. On the other hand, if the closest peer has +the ID 44, we can imagine that the space is rather packed with peers, +maybe as much as 50 of them. +Naturally, we could have been rather unlucky, and there is only one peer +and happens to have the ID 44. Thus, the current estimate is calculated +as the average over multiple rounds, and not just a single sample. + +@node Algorithm +@subsubsection Algorithm + + +Given that example, one can imagine that the job of the subsystem is to +efficiently communicate the ID of the closest peer to the target value +to all the other peers, who will calculate the estimate from it. + +@node Target value +@subsubsection Target value + +@c %**end of header + +The target value itself is generated by hashing the current time, rounded +down to an agreed value. If the rounding amount is 1h (default) and the +time is 12:34:56, the time to hash would be 12:00:00. The process is +repeated each rouning amount (in this example would be every hour). +Every repetition is called a round. + +@node Timing +@subsubsection Timing +@c %**end of header + +The NSE subsystem has some timing control to avoid everybody broadcasting +its ID all at one. Once each peer has the target random value, it +compares its own ID to the target and calculates the hypothetical size of +the network if that peer were to be the closest. +Then it compares the hypothetical size with the estimate from the previous +rounds. For each value there is an assiciated point in the period, +let's call it "broadcast time". If its own hypothetical estimate +is the same as the previous global estimate, its "broadcast time" will be +in the middle of the round. If its bigger it will be earlier and if its +smaller (the most likely case) it will be later. This ensures that the +peers closests to the target value start broadcasting their ID the first. + +@node Controlled Flooding +@subsubsection Controlled Flooding + +@c %**end of header + +When a peer receives a value, first it verifies that it is closer than the +closest value it had so far, otherwise it answers the incoming message +with a message containing the better value. Then it checks a proof of +work that must be included in the incoming message, to ensure that the +other peer's ID is not made up (otherwise a malicious peer could claim to +have an ID of exactly the target value every round). Once validated, it +compares the brodcast time of the received value with the current time +and if it's not too early, sends the received value to its neighbors. +Otherwise it stores the value until the correct broadcast time comes. +This prevents unnecessary traffic of sub-optimal values, since a better +value can come before the broadcast time, rendering the previous one +obsolete and saving the traffic that would have been used to broadcast it +to the neighbors. + +@node Calculating the estimate +@subsubsection Calculating the estimate + +@c %**end of header + +Once the closest ID has been spread across the network each peer gets the +exact distance betweed this ID and the target value of the round and +calculates the estimate with a mathematical formula described in the tech +report. The estimate generated with this method for a single round is not +very precise. Remember the case of the example, where the only peer is the +ID 44 and we happen to generate the target value 42, thinking there are +50 peers in the network. Therefore, the NSE subsystem remembers the last +64 estimates and calculates an average over them, giving a result of which +usually has one bit of uncertainty (the real size could be half of the +estimate or twice as much). Note that the actual network size is +calculated in powers of two of the raw input, thus one bit of uncertainty +means a factor of two in the size estimate. + +@cindex libgnunetnse +@node libgnunetnse +@subsection libgnunetnse + +@c %**end of header + +The NSE subsystem has the simplest API of all services, with only two +calls: @code{GNUNET_NSE_connect} and @code{GNUNET_NSE_disconnect}. + +The connect call gets a callback function as a parameter and this function +is called each time the network agrees on an estimate. This usually is +once per round, with some exceptions: if the closest peer has a late +local clock and starts spreading his ID after everyone else agreed on a +value, the callback might be activated twice in a round, the second value +being always bigger than the first. The default round time is set to +1 hour. + +The disconnect call disconnects from the NSE subsystem and the callback +is no longer called with new estimates. + + + +@menu +* Results:: +* libgnunetnse - Examples:: +@end menu + +@node Results +@subsubsection Results + +@c %**end of header + +The callback provides two values: the average and the +@uref{http://en.wikipedia.org/wiki/Standard_deviation, standard deviation} +of the last 64 rounds. The values provided by the callback function are +logarithmic, this means that the real estimate numbers can be obtained by +calculating 2 to the power of the given value (2average). From a +statistics point of view this means that: + +@itemize @bullet +@item 68% of the time the real size is included in the interval +[(2average-stddev), 2] +@item 95% of the time the real size is included in the interval +[(2average-2*stddev, 2^average+2*stddev] +@item 99.7% of the time the real size is included in the interval +[(2average-3*stddev, 2average+3*stddev] +@end itemize + +The expected standard variation for 64 rounds in a network of stable size +is 0.2. Thus, we can say that normally: + +@itemize @bullet +@item 68% of the time the real size is in the range [-13%, +15%] +@item 95% of the time the real size is in the range [-24%, +32%] +@item 99.7% of the time the real size is in the range [-34%, +52%] +@end itemize + +As said in the introduction, we can be quite sure that usually the real +size is between one third and three times the estimate. This can of +course vary with network conditions. +Thus, applications may want to also consider the provided standard +deviation value, not only the average (in particular, if the standard +veriation is very high, the average maybe meaningless: the network size is +changing rapidly). + +@node libgnunetnse - Examples +@subsubsection libgnunetnse -Examples + +@c %**end of header + +Let's close with a couple examples. + +@table @asis + +@item Average: 10, std dev: 1 Here the estimate would be +2^10 = 1024 peers. @footnote{The range in which we can be 95% sure is: +[2^8, 2^12] = [256, 4096]. We can be very (>99.7%) sure that the network +is not a hundred peers and absolutely sure that it is not a million peers, +but somewhere around a thousand.} + +@item Average 22, std dev: 0.2 Here the estimate would be +2^22 = 4 Million peers. @footnote{The range in which we can be 99.7% sure +is: [2^21.4, 2^22.6] = [2.8M, 6.3M]. We can be sure that the network size +is around four million, with absolutely way of it being 1 million.} + +@end table + +To put this in perspective, if someone remembers the LHC Higgs boson +results, were announced with "5 sigma" and "6 sigma" certainties. In this +case a 5 sigma minimum would be 2 million and a 6 sigma minimum, +1.8 million. + +@node The NSE Client-Service Protocol +@subsection The NSE Client-Service Protocol + +@c %**end of header + +As with the API, the client-service protocol is very simple, only has 2 +different messages, defined in @code{src/nse/nse.h}: + +@itemize @bullet +@item @code{GNUNET_MESSAGE_TYPE_NSE_START}@ This message has no parameters +and is sent from the client to the service upon connection. +@item @code{GNUNET_MESSAGE_TYPE_NSE_ESTIMATE}@ This message is sent from +the service to the client for every new estimate and upon connection. +Contains a timestamp for the estimate, the average and the standard +deviation for the respective round. +@end itemize + +When the @code{GNUNET_NSE_disconnect} API call is executed, the client +simply disconnects from the service, with no message involved. + +@cindex NSE Peer-to-Peer Protocol +@node The NSE Peer-to-Peer Protocol +@subsection The NSE Peer-to-Peer Protocol + +@c %**end of header + +The NSE subsystem only has one message in the P2P protocol, the +@code{GNUNET_MESSAGE_TYPE_NSE_P2P_FLOOD} message. + +This message key contents are the timestamp to identify the round +(differences in system clocks may cause some peers to send messages way +too early or way too late, so the timestamp allows other peers to +identify such messages easily), the +@uref{http://en.wikipedia.org/wiki/Proof-of-work_system, proof of work} +used to make it difficult to mount a +@uref{http://en.wikipedia.org/wiki/Sybil_attack, Sybil attack}, and the +public key, which is used to verify the signature on the message. + +Every peer stores a message for the previous, current and next round. The +messages for the previous and current round are given to peers that +connect to us. The message for the next round is simply stored until our +system clock advances to the next round. The message for the current round +is what we are flooding the network with right now. +At the beginning of each round the peer does the following: + +@itemize @bullet +@item calculates his own distance to the target value +@item creates, signs and stores the message for the current round (unless +it has a better message in the "next round" slot which came early in the +previous round) +@item calculates, based on the stored round message (own or received) when +to stard flooding it to its neighbors +@end itemize + +Upon receiving a message the peer checks the validity of the message +(round, proof of work, signature). The next action depends on the +contents of the incoming message: + +@itemize @bullet +@item if the message is worse than the current stored message, the peer +sends the current message back immediately, to stop the other peer from +spreading suboptimal results +@item if the message is better than the current stored message, the peer +stores the new message and calculates the new target time to start +spreading it to its neighbors (excluding the one the message came from) +@item if the message is for the previous round, it is compared to the +message stored in the "previous round slot", which may then be updated +@item if the message is for the next round, it is compared to the message +stored in the "next round slot", which again may then be updated +@end itemize + +Finally, when it comes to send the stored message for the current round to +the neighbors there is a random delay added for each neighbor, to avoid +traffic spikes and minimize cross-messages. + +@cindex HOSTLIST Subsystem +@node HOSTLIST Subsystem +@section HOSTLIST Subsystem + +@c %**end of header + +Peers in the GNUnet overlay network need address information so that they +can connect with other peers. GNUnet uses so called HELLO messages to +store and exchange peer addresses. +GNUnet provides several methods for peers to obtain this information: + +@itemize @bullet +@item out-of-band exchange of HELLO messages (manually, using for example +gnunet-peerinfo) +@item HELLO messages shipped with GNUnet (automatic with distribution) +@item UDP neighbor discovery in LAN (IPv4 broadcast, IPv6 multicast) +@item topology gossiping (learning from other peers we already connected +to), and +@item the HOSTLIST daemon covered in this section, which is particularly +relevant for bootstrapping new peers. +@end itemize + +New peers have no existing connections (and thus cannot learn from gossip +among peers), may not have other peers in their LAN and might be started +with an outdated set of HELLO messages from the distribution. +In this case, getting new peers to connect to the network requires either +manual effort or the use of a HOSTLIST to obtain HELLOs. + +@menu +* HELLOs:: +* Overview for the HOSTLIST subsystem:: +* Interacting with the HOSTLIST daemon:: +* Hostlist security address validation:: +* The HOSTLIST daemon:: +* The HOSTLIST server:: +* The HOSTLIST client:: +* Usage:: +@end menu + +@node HELLOs +@subsection HELLOs + +@c %**end of header + +The basic information peers require to connect to other peers are +contained in so called HELLO messages you can think of as a business card. +Besides the identity of the peer (based on the cryptographic public key) a +HELLO message may contain address information that specifies ways to +contact a peer. By obtaining HELLO messages, a peer can learn how to +contact other peers. + +@node Overview for the HOSTLIST subsystem +@subsection Overview for the HOSTLIST subsystem + +@c %**end of header + +The HOSTLIST subsystem provides a way to distribute and obtain contact +information to connect to other peers using a simple HTTP GET request. +It's implementation is split in three parts, the main file for the daemon +itself (@file{gnunet-daemon-hostlist.c}), the HTTP client used to download +peer information (@file{hostlist-client.c}) and the server component used +to provide this information to other peers (@file{hostlist-server.c}). +The server is basically a small HTTP web server (based on GNU +libmicrohttpd) which provides a list of HELLOs known to the local peer for +download. The client component is basically a HTTP client +(based on libcurl) which can download hostlists from one or more websites. +The hostlist format is a binary blob containing a sequence of HELLO +messages. Note that any HTTP server can theoretically serve a hostlist, +the build-in hostlist server makes it simply convenient to offer this +service. + + +@menu +* Features:: +* HOSTLIST - Limitations:: +@end menu + +@node Features +@subsubsection Features + +@c %**end of header + +The HOSTLIST daemon can: + +@itemize @bullet +@item provide HELLO messages with validated addresses obtained from +PEERINFO to download for other peers +@item download HELLO messages and forward these message to the TRANSPORT +subsystem for validation +@item advertises the URL of this peer's hostlist address to other peers +via gossip +@item automatically learn about hostlist servers from the gossip of other +peers +@end itemize + +@node HOSTLIST - Limitations +@subsubsection HOSTLIST - Limitations + +@c %**end of header + +The HOSTLIST daemon does not: + +@itemize @bullet +@item verify the cryptographic information in the HELLO messages +@item verify the address information in the HELLO messages +@end itemize + +@node Interacting with the HOSTLIST daemon +@subsection Interacting with the HOSTLIST daemon + +@c %**end of header + +The HOSTLIST subsystem is currently implemented as a daemon, so there is +no need for the user to interact with it and therefore there is no +command line tool and no API to communicate with the daemon. In the +future, we can envision changing this to allow users to manually trigger +the download of a hostlist. + +Since there is no command line interface to interact with HOSTLIST, the +only way to interact with the hostlist is to use STATISTICS to obtain or +modify information about the status of HOSTLIST: + +@example +$ gnunet-statistics -s hostlist +@end example + +@noindent +In particular, HOSTLIST includes a @strong{persistent} value in statistics +that specifies when the hostlist server might be queried next. As this +value is exponentially increasing during runtime, developers may want to +reset or manually adjust it. Note that HOSTLIST (but not STATISTICS) needs +to be shutdown if changes to this value are to have any effect on the +daemon (as HOSTLIST does not monitor STATISTICS for changes to the +download frequency). + +@node Hostlist security address validation +@subsection Hostlist security address validation + +@c %**end of header + +Since information obtained from other parties cannot be trusted without +validation, we have to distinguish between @emph{validated} and +@emph{not validated} addresses. Before using (and so trusting) +information from other parties, this information has to be double-checked +(validated). Address validation is not done by HOSTLIST but by the +TRANSPORT service. + +The HOSTLIST component is functionally located between the PEERINFO and +the TRANSPORT subsystem. When acting as a server, the daemon obtains valid +(@emph{validated}) peer information (HELLO messages) from the PEERINFO +service and provides it to other peers. When acting as a client, it +contacts the HOSTLIST servers specified in the configuration, downloads +the (unvalidated) list of HELLO messages and forwards these information +to the TRANSPORT server to validate the addresses. + +@cindex HOSTLIST daemon +@node The HOSTLIST daemon +@subsection The HOSTLIST daemon + +@c %**end of header + +The hostlist daemon is the main component of the HOSTLIST subsystem. It is +started by the ARM service and (if configured) starts the HOSTLIST client +and server components. + +If the daemon provides a hostlist itself it can advertise it's own +hostlist to other peers. To do so it sends a +@code{GNUNET_MESSAGE_TYPE_HOSTLIST_ADVERTISEMENT} message to other peers +when they connect to this peer on the CORE level. This hostlist +advertisement message contains the URL to access the HOSTLIST HTTP +server of the sender. The daemon may also subscribe to this type of +message from CORE service, and then forward these kind of message to the +HOSTLIST client. The client then uses all available URLs to download peer +information when necessary. + +When starting, the HOSTLIST daemon first connects to the CORE subsystem +and if hostlist learning is enabled, registers a CORE handler to receive +this kind of messages. Next it starts (if configured) the client and +server. It passes pointers to CORE connect and disconnect and receive +handlers where the client and server store their functions, so the daemon +can notify them about CORE events. + +To clean up on shutdown, the daemon has a cleaning task, shutting down all +subsystems and disconnecting from CORE. + +@cindex HOSTLIST server +@node The HOSTLIST server +@subsection The HOSTLIST server + +@c %**end of header + +The server provides a way for other peers to obtain HELLOs. Basically it +is a small web server other peers can connect to and download a list of +HELLOs using standard HTTP; it may also advertise the URL of the hostlist +to other peers connecting on CORE level. + + +@menu +* The HTTP Server:: +* Advertising the URL:: +@end menu + +@node The HTTP Server +@subsubsection The HTTP Server + +@c %**end of header + +During startup, the server starts a web server listening on the port +specified with the HTTPPORT value (default 8080). In addition it connects +to the PEERINFO service to obtain peer information. The HOSTLIST server +uses the GNUNET_PEERINFO_iterate function to request HELLO information for +all peers and adds their information to a new hostlist if they are +suitable (expired addresses and HELLOs without addresses are both not +suitable) and the maximum size for a hostlist is not exceeded +(MAX_BYTES_PER_HOSTLISTS = 500000). +When PEERINFO finishes (with a last NULL callback), the server destroys +the previous hostlist response available for download on the web server +and replaces it with the updated hostlist. The hostlist format is +basically a sequence of HELLO messages (as obtained from PEERINFO) without +any special tokenization. Since each HELLO message contains a size field, +the response can easily be split into separate HELLO messages by the +client. + +A HOSTLIST client connecting to the HOSTLIST server will receive the +hostlist as a HTTP response and the the server will terminate the +connection with the result code @code{HTTP 200 OK}. +The connection will be closed immediately if no hostlist is available. + +@node Advertising the URL +@subsubsection Advertising the URL + +@c %**end of header + +The server also advertises the URL to download the hostlist to other peers +if hostlist advertisement is enabled. +When a new peer connects and has hostlist learning enabled, the server +sends a @code{GNUNET_MESSAGE_TYPE_HOSTLIST_ADVERTISEMENT} message to this +peer using the CORE service. + +@cindex HOSTLIST client +@node The HOSTLIST client +@subsection The HOSTLIST client + +@c %**end of header + +The client provides the functionality to download the list of HELLOs from +a set of URLs. +It performs a standard HTTP request to the URLs configured and learned +from advertisement messages received from other peers. When a HELLO is +downloaded, the HOSTLIST client forwards the HELLO to the TRANSPORT +service for validation. + +The client supports two modes of operation: + +@itemize @bullet +@item download of HELLOs (bootstrapping) +@item learning of URLs +@end itemize + +@menu +* Bootstrapping:: +* Learning:: +@end menu + +@node Bootstrapping +@subsubsection Bootstrapping + +@c %**end of header + +For bootstrapping, it schedules a task to download the hostlist from the +set of known URLs. +The downloads are only performed if the number of current +connections is smaller than a minimum number of connections +(at the moment 4). +The interval between downloads increases exponentially; however, the +exponential growth is limited if it becomes longer than an hour. +At that point, the frequency growth is capped at +(#number of connections * 1h). + +Once the decision has been taken to download HELLOs, the daemon chooses a +random URL from the list of known URLs. URLs can be configured in the +configuration or be learned from advertisement messages. +The client uses a HTTP client library (libcurl) to initiate the download +using the libcurl multi interface. +Libcurl passes the data to the callback_download function which +stores the data in a buffer if space is available and the maximum size for +a hostlist download is not exceeded (MAX_BYTES_PER_HOSTLISTS = 500000). +When a full HELLO was downloaded, the HOSTLIST client offers this +HELLO message to the TRANSPORT service for validation. +When the download is finished or failed, statistical information about the +quality of this URL is updated. + +@cindex HOSTLIST learning +@node Learning +@subsubsection Learning + +@c %**end of header + +The client also manages hostlist advertisements from other peers. The +HOSTLIST daemon forwards @code{GNUNET_MESSAGE_TYPE_HOSTLIST_ADVERTISEMENT} +messages to the client subsystem, which extracts the URL from the message. +Next, a test of the newly obtained URL is performed by triggering a +download from the new URL. If the URL works correctly, it is added to the +list of working URLs. + +The size of the list of URLs is restricted, so if an additional server is +added and the list is full, the URL with the worst quality ranking +(determined through successful downloads and number of HELLOs e.g.) is +discarded. During shutdown the list of URLs is saved to a file for +persistance and loaded on startup. URLs from the configuration file are +never discarded. + +@node Usage +@subsection Usage + +@c %**end of header + +To start HOSTLIST by default, it has to be added to the DEFAULTSERVICES +section for the ARM services. This is done in the default configuration. + +For more information on how to configure the HOSTLIST subsystem see the +installation handbook:@ +Configuring the hostlist to bootstrap@ +Configuring your peer to provide a hostlist + +@cindex IDENTITY Subsystem +@node IDENTITY Subsystem +@section IDENTITY Subsystem + +@c %**end of header + +Identities of "users" in GNUnet are called egos. +Egos can be used as pseudonyms ("fake names") or be tied to an +organization (for example, "GNU") or even the actual identity of a human. +GNUnet users are expected to have many egos. They might have one tied to +their real identity, some for organizations they manage, and more for +different domains where they want to operate under a pseudonym. + +The IDENTITY service allows users to manage their egos. The identity +service manages the private keys egos of the local user; it does not +manage identities of other users (public keys). Public keys for other +users need names to become manageable. GNUnet uses the +@dfn{GNU Name System} (GNS) to give names to other users and manage their +public keys securely. This chapter is about the IDENTITY service, +which is about the management of private keys. + +On the network, an ego corresponds to an ECDSA key (over Curve25519, +using RFC 6979, as required by GNS). Thus, users can perform actions +under a particular ego by using (signing with) a particular private key. +Other users can then confirm that the action was really performed by that +ego by checking the signature against the respective public key. + +The IDENTITY service allows users to associate a human-readable name with +each ego. This way, users can use names that will remind them of the +purpose of a particular ego. +The IDENTITY service will store the respective private keys and +allows applications to access key information by name. +Users can change the name that is locally (!) associated with an ego. +Egos can also be deleted, which means that the private key will be removed +and it thus will not be possible to perform actions with that ego in the +future. + +Additionally, the IDENTITY subsystem can associate service functions with +egos. +For example, GNS requires the ego that should be used for the shorten +zone. GNS will ask IDENTITY for an ego for the "gns-short" service. +The IDENTITY service has a mapping of such service strings to the name of +the ego that the user wants to use for this service, for example +"my-short-zone-ego". + +Finally, the IDENTITY API provides access to a special ego, the +anonymous ego. The anonymous ego is special in that its private key is not +really private, but fixed and known to everyone. +Thus, anyone can perform actions as anonymous. This can be useful as with +this trick, code does not have to contain a special case to distinguish +between anonymous and pseudonymous egos. + +@menu +* libgnunetidentity:: +* The IDENTITY Client-Service Protocol:: +@end menu + +@cindex libgnunetidentity +@node libgnunetidentity +@subsection libgnunetidentity +@c %**end of header + + +@menu +* Connecting to the service:: +* Operations on Egos:: +* The anonymous Ego:: +* Convenience API to lookup a single ego:: +* Associating egos with service functions:: +@end menu + +@node Connecting to the service +@subsubsection Connecting to the service + +@c %**end of header + +First, typical clients connect to the identity service using +@code{GNUNET_IDENTITY_connect}. This function takes a callback as a +parameter. +If the given callback parameter is non-null, it will be invoked to notify +the application about the current state of the identities in the system. + +@itemize @bullet +@item First, it will be invoked on all known egos at the time of the +connection. For each ego, a handle to the ego and the user's name for the +ego will be passed to the callback. Furthermore, a @code{void **} context +argument will be provided which gives the client the opportunity to +associate some state with the ego. +@item Second, the callback will be invoked with NULL for the ego, the name +and the context. This signals that the (initial) iteration over all egos +has completed. +@item Then, the callback will be invoked whenever something changes about +an ego. +If an ego is renamed, the callback is invoked with the ego handle of the +ego that was renamed, and the new name. If an ego is deleted, the callback +is invoked with the ego handle and a name of NULL. In the deletion case, +the application should also release resources stored in the context. +@item When the application destroys the connection to the identity service +using @code{GNUNET_IDENTITY_disconnect}, the callback is again invoked +with the ego and a name of NULL (equivalent to deletion of the egos). +This should again be used to clean up the per-ego context. +@end itemize + +The ego handle passed to the callback remains valid until the callback is +invoked with a name of NULL, so it is safe to store a reference to the +ego's handle. + +@node Operations on Egos +@subsubsection Operations on Egos + +@c %**end of header + +Given an ego handle, the main operations are to get its associated private +key using @code{GNUNET_IDENTITY_ego_get_private_key} or its associated +public key using @code{GNUNET_IDENTITY_ego_get_public_key}. + +The other operations on egos are pretty straightforward. +Using @code{GNUNET_IDENTITY_create}, an application can request the +creation of an ego by specifying the desired name. +The operation will fail if that name is +already in use. Using @code{GNUNET_IDENTITY_rename} the name of an +existing ego can be changed. Finally, egos can be deleted using +@code{GNUNET_IDENTITY_delete}. All of these operations will trigger +updates to the callback given to the @code{GNUNET_IDENTITY_connect} +function of all applications that are connected with the identity service +at the time. @code{GNUNET_IDENTITY_cancel} can be used to cancel the +operations before the respective continuations would be called. +It is not guaranteed that the operation will not be completed anyway, +only the continuation will no longer be called. + +@node The anonymous Ego +@subsubsection The anonymous Ego + +@c %**end of header + +A special way to obtain an ego handle is to call +@code{GNUNET_IDENTITY_ego_get_anonymous}, which returns an ego for the +"anonymous" user --- anyone knows and can get the private key for this +user, so it is suitable for operations that are supposed to be anonymous +but require signatures (for example, to avoid a special path in the code). +The anonymous ego is always valid and accessing it does not require a +connection to the identity service. + +@node Convenience API to lookup a single ego +@subsubsection Convenience API to lookup a single ego + + +As applications commonly simply have to lookup a single ego, there is a +convenience API to do just that. Use @code{GNUNET_IDENTITY_ego_lookup} to +lookup a single ego by name. Note that this is the user's name for the +ego, not the service function. The resulting ego will be returned via a +callback and will only be valid during that callback. The operation can +be cancelled via @code{GNUNET_IDENTITY_ego_lookup_cancel} +(cancellation is only legal before the callback is invoked). + +@node Associating egos with service functions +@subsubsection Associating egos with service functions + + +The @code{GNUNET_IDENTITY_set} function is used to associate a particular +ego with a service function. The name used by the service and the ego are +given as arguments. +Afterwards, the service can use its name to lookup the associated ego +using @code{GNUNET_IDENTITY_get}. + +@node The IDENTITY Client-Service Protocol +@subsection The IDENTITY Client-Service Protocol + +@c %**end of header + +A client connecting to the identity service first sends a message with +type +@code{GNUNET_MESSAGE_TYPE_IDENTITY_START} to the service. After that, the +client will receive information about changes to the egos by receiving +messages of type @code{GNUNET_MESSAGE_TYPE_IDENTITY_UPDATE}. +Those messages contain the private key of the ego and the user's name of +the ego (or zero bytes for the name to indicate that the ego was deleted). +A special bit @code{end_of_list} is used to indicate the end of the +initial iteration over the identity service's egos. + +The client can trigger changes to the egos by sending @code{CREATE}, +@code{RENAME} or @code{DELETE} messages. +The CREATE message contains the private key and the desired name.@ +The RENAME message contains the old name and the new name.@ +The DELETE message only needs to include the name of the ego to delete.@ +The service responds to each of these messages with a @code{RESULT_CODE} +message which indicates success or error of the operation, and possibly +a human-readable error message. + +Finally, the client can bind the name of a service function to an ego by +sending a @code{SET_DEFAULT} message with the name of the service function +and the private key of the ego. +Such bindings can then be resolved using a @code{GET_DEFAULT} message, +which includes the name of the service function. The identity service +will respond to a GET_DEFAULT request with a SET_DEFAULT message +containing the respective information, or with a RESULT_CODE to +indicate an error. + +@cindex NAMESTORE Subsystem +@node NAMESTORE Subsystem +@section NAMESTORE Subsystem + +The NAMESTORE subsystem provides persistent storage for local GNS zone +information. All local GNS zone information are managed by NAMESTORE. It +provides both the functionality to administer local GNS information (e.g. +delete and add records) as well as to retrieve GNS information (e.g to +list name information in a client). +NAMESTORE does only manage the persistent storage of zone information +belonging to the user running the service: GNS information from other +users obtained from the DHT are stored by the NAMECACHE subsystem. + +NAMESTORE uses a plugin-based database backend to store GNS information +with good performance. Here sqlite, MySQL and PostgreSQL are supported +database backends. +NAMESTORE clients interact with the IDENTITY subsystem to obtain +cryptographic information about zones based on egos as described with the +IDENTITY subsystem, but internally NAMESTORE refers to zones using the +ECDSA private key. +In addition, it collaborates with the NAMECACHE subsystem and +stores zone information when local information are modified in the +GNS cache to increase look-up performance for local information. + +NAMESTORE provides functionality to look-up and store records, to iterate +over a specific or all zones and to monitor zones for changes. NAMESTORE +functionality can be accessed using the NAMESTORE api or the NAMESTORE +command line tool. + +@menu +* libgnunetnamestore:: +@end menu + +@cindex libgnunetnamestore +@node libgnunetnamestore +@subsection libgnunetnamestore + +To interact with NAMESTORE clients first connect to the NAMESTORE service +using the @code{GNUNET_NAMESTORE_connect} passing a configuration handle. +As a result they obtain a NAMESTORE handle, they can use for operations, +or NULL is returned if the connection failed. + +To disconnect from NAMESTORE, clients use +@code{GNUNET_NAMESTORE_disconnect} and specify the handle to disconnect. + +NAMESTORE internally uses the ECDSA private key to refer to zones. These +private keys can be obtained from the IDENTITY subsytem. +Here @emph{egos} @emph{can be used to refer to zones or the default ego +assigned to the GNS subsystem can be used to obtained the master zone's +private key.} + + +@menu +* Editing Zone Information:: +* Iterating Zone Information:: +* Monitoring Zone Information:: +@end menu + +@node Editing Zone Information +@subsubsection Editing Zone Information + +@c %**end of header + +NAMESTORE provides functions to lookup records stored under a label in a +zone and to store records under a label in a zone. + +To store (and delete) records, the client uses the +@code{GNUNET_NAMESTORE_records_store} function and has to provide +namestore handle to use, the private key of the zone, the label to store +the records under, the records and number of records plus an callback +function. +After the operation is performed NAMESTORE will call the provided +callback function with the result GNUNET_SYSERR on failure +(including timeout/queue drop/failure to validate), GNUNET_NO if content +was already there or not found GNUNET_YES (or other positive value) on +success plus an additional error message. + +Records are deleted by using the store command with 0 records to store. +It is important to note, that records are not merged when records exist +with the label. +So a client has first to retrieve records, merge with existing records +and then store the result. + +To perform a lookup operation, the client uses the +@code{GNUNET_NAMESTORE_records_store} function. Here he has to pass the +namestore handle, the private key of the zone and the label. He also has +to provide a callback function which will be called with the result of +the lookup operation: +the zone for the records, the label, and the records including the +number of records included. + +A special operation is used to set the preferred nickname for a zone. +This nickname is stored with the zone and is automatically merged with +all labels and records stored in a zone. Here the client uses the +@code{GNUNET_NAMESTORE_set_nick} function and passes the private key of +the zone, the nickname as string plus a the callback with the result of +the operation. + +@node Iterating Zone Information +@subsubsection Iterating Zone Information + +@c %**end of header + +A client can iterate over all information in a zone or all zones managed +by NAMESTORE. +Here a client uses the @code{GNUNET_NAMESTORE_zone_iteration_start} +function and passes the namestore handle, the zone to iterate over and a +callback function to call with the result. +If the client wants to iterate over all the, he passes NULL for the zone. +A @code{GNUNET_NAMESTORE_ZoneIterator} handle is returned to be used to +continue iteration. + +NAMESTORE calls the callback for every result and expects the client to +call @code{GNUNET_NAMESTORE_zone_iterator_next} to continue to iterate or +@code{GNUNET_NAMESTORE_zone_iterator_stop} to interrupt the iteration. +When NAMESTORE reached the last item it will call the callback with a +NULL value to indicate. + +@node Monitoring Zone Information +@subsubsection Monitoring Zone Information + +@c %**end of header + +Clients can also monitor zones to be notified about changes. Here the +clients uses the @code{GNUNET_NAMESTORE_zone_monitor_start} function and +passes the private key of the zone and and a callback function to call +with updates for a zone. +The client can specify to obtain zone information first by iterating over +the zone and specify a synchronization callback to be called when the +client and the namestore are synced. + +On an update, NAMESTORE will call the callback with the private key of the +zone, the label and the records and their number. + +To stop monitoring, the client calls +@code{GNUNET_NAMESTORE_zone_monitor_stop} and passes the handle obtained +from the function to start the monitoring. + +@cindex PEERINFO Subsystem +@node PEERINFO Subsystem +@section PEERINFO Subsystem + +@c %**end of header + +The PEERINFO subsystem is used to store verified (validated) information +about known peers in a persistent way. It obtains these addresses for +example from TRANSPORT service which is in charge of address validation. +Validation means that the information in the HELLO message are checked by +connecting to the addresses and performing a cryptographic handshake to +authenticate the peer instance stating to be reachable with these +addresses. +Peerinfo does not validate the HELLO messages itself but only stores them +and gives them to interested clients. + +As future work, we think about moving from storing just HELLO messages to +providing a generic persistent per-peer information store. +More and more subsystems tend to need to store per-peer information in +persistent way. +To not duplicate this functionality we plan to provide a PEERSTORE +service providing this functionality. + +@menu +* PEERINFO - Features:: +* PEERINFO - Limitations:: +* DeveloperPeer Information:: +* Startup:: +* Managing Information:: +* Obtaining Information:: +* The PEERINFO Client-Service Protocol:: +* libgnunetpeerinfo:: +@end menu + +@node PEERINFO - Features +@subsection PEERINFO - Features + +@c %**end of header + +@itemize @bullet +@item Persistent storage +@item Client notification mechanism on update +@item Periodic clean up for expired information +@item Differentiation between public and friend-only HELLO +@end itemize + +@node PEERINFO - Limitations +@subsection PEERINFO - Limitations + + +@itemize @bullet +@item Does not perform HELLO validation +@end itemize + +@node DeveloperPeer Information +@subsection DeveloperPeer Information + +@c %**end of header + +The PEERINFO subsystem stores these information in the form of HELLO +messages you can think of as business cards. +These HELLO messages contain the public key of a peer and the addresses +a peer can be reached under. +The addresses include an expiration date describing how long they are +valid. This information is updated regularly by the TRANSPORT service by +revalidating the address. +If an address is expired and not renewed, it can be removed from the +HELLO message. + +Some peer do not want to have their HELLO messages distributed to other +peers, especially when GNUnet's friend-to-friend modus is enabled. +To prevent this undesired distribution. PEERINFO distinguishes between +@emph{public} and @emph{friend-only} HELLO messages. +Public HELLO messages can be freely distributed to other (possibly +unknown) peers (for example using the hostlist, gossiping, broadcasting), +whereas friend-only HELLO messages may not be distributed to other peers. +Friend-only HELLO messages have an additional flag @code{friend_only} set +internally. For public HELLO message this flag is not set. +PEERINFO does and cannot not check if a client is allowed to obtain a +specific HELLO type. + +The HELLO messages can be managed using the GNUnet HELLO library. +Other GNUnet systems can obtain these information from PEERINFO and use +it for their purposes. +Clients are for example the HOSTLIST component providing these +information to other peers in form of a hostlist or the TRANSPORT +subsystem using these information to maintain connections to other peers. + +@node Startup +@subsection Startup + +@c %**end of header + +During startup the PEERINFO services loads persistent HELLOs from disk. +First PEERINFO parses the directory configured in the HOSTS value of the +@code{PEERINFO} configuration section to store PEERINFO information. +For all files found in this directory valid HELLO messages are extracted. +In addition it loads HELLO messages shipped with the GNUnet distribution. +These HELLOs are used to simplify network bootstrapping by providing +valid peer information with the distribution. +The use of these HELLOs can be prevented by setting the +@code{USE_INCLUDED_HELLOS} in the @code{PEERINFO} configuration section to +@code{NO}. Files containing invalid information are removed. + +@node Managing Information +@subsection Managing Information + +@c %**end of header + +The PEERINFO services stores information about known PEERS and a single +HELLO message for every peer. +A peer does not need to have a HELLO if no information are available. +HELLO information from different sources, for example a HELLO obtained +from a remote HOSTLIST and a second HELLO stored on disk, are combined +and merged into one single HELLO message per peer which will be given to +clients. During this merge process the HELLO is immediately written to +disk to ensure persistence. + +PEERINFO in addition periodically scans the directory where information +are stored for empty HELLO messages with expired TRANSPORT addresses. +This periodic task scans all files in the directory and recreates the +HELLO messages it finds. +Expired TRANSPORT addresses are removed from the HELLO and if the +HELLO does not contain any valid addresses, it is discarded and removed +from the disk. + +@node Obtaining Information +@subsection Obtaining Information + +@c %**end of header + +When a client requests information from PEERINFO, PEERINFO performs a +lookup for the respective peer or all peers if desired and transmits this +information to the client. +The client can specify if friend-only HELLOs have to be included or not +and PEERINFO filters the respective HELLO messages before transmitting +information. + +To notify clients about changes to PEERINFO information, PEERINFO +maintains a list of clients interested in this notifications. +Such a notification occurs if a HELLO for a peer was updated (due to a +merge for example) or a new peer was added. + +@node The PEERINFO Client-Service Protocol +@subsection The PEERINFO Client-Service Protocol + +@c %**end of header + +To connect and disconnect to and from the PEERINFO Service PEERINFO +utilizes the util client/server infrastructure, so no special messages +types are used here. + +To add information for a peer, the plain HELLO message is transmitted to +the service without any wrapping. All pieces of information required are +stored within the HELLO message. +The PEERINFO service provides a message handler accepting and processing +these HELLO messages. + +When obtaining PEERINFO information using the iterate functionality +specific messages are used. To obtain information for all peers, a +@code{struct ListAllPeersMessage} with message type +@code{GNUNET_MESSAGE_TYPE_PEERINFO_GET_ALL} and a flag +include_friend_only to indicate if friend-only HELLO messages should be +included are transmitted. If information for a specific peer is required +a @code{struct ListAllPeersMessage} with +@code{GNUNET_MESSAGE_TYPE_PEERINFO_GET} containing the peer identity is +used. + +For both variants the PEERINFO service replies for each HELLO message it +wants to transmit with a @code{struct ListAllPeersMessage} with type +@code{GNUNET_MESSAGE_TYPE_PEERINFO_INFO} containing the plain HELLO. +The final message is @code{struct GNUNET_MessageHeader} with type +@code{GNUNET_MESSAGE_TYPE_PEERINFO_INFO}. If the client receives this +message, it can proceed with the next request if any is pending. + +@node libgnunetpeerinfo +@subsection libgnunetpeerinfo + +@c %**end of header + +The PEERINFO API consists mainly of three different functionalities: + +@itemize @bullet +@item maintaining a connection to the service +@item adding new information to the PEERINFO service +@item retrieving information from the PEERINFO service +@end itemize + +@menu +* Connecting to the PEERINFO Service:: +* Adding Information to the PEERINFO Service:: +* Obtaining Information from the PEERINFO Service:: +@end menu + +@node Connecting to the PEERINFO Service +@subsubsection Connecting to the PEERINFO Service + +@c %**end of header + +To connect to the PEERINFO service the function +@code{GNUNET_PEERINFO_connect} is used, taking a configuration handle as +an argument, and to disconnect from PEERINFO the function +@code{GNUNET_PEERINFO_disconnect}, taking the PEERINFO +handle returned from the connect function has to be called. + +@node Adding Information to the PEERINFO Service +@subsubsection Adding Information to the PEERINFO Service + +@c %**end of header + +@code{GNUNET_PEERINFO_add_peer} adds a new peer to the PEERINFO subsystem +storage. This function takes the PEERINFO handle as an argument, the HELLO +message to store and a continuation with a closure to be called with the +result of the operation. +The @code{GNUNET_PEERINFO_add_peer} returns a handle to this operation +allowing to cancel the operation with the respective cancel function +@code{GNUNET_PEERINFO_add_peer_cancel}. To retrieve information from +PEERINFO you can iterate over all information stored with PEERINFO or you +can tell PEERINFO to notify if new peer information are available. + +@node Obtaining Information from the PEERINFO Service +@subsubsection Obtaining Information from the PEERINFO Service + +@c %**end of header + +To iterate over information in PEERINFO you use +@code{GNUNET_PEERINFO_iterate}. +This function expects the PEERINFO handle, a flag if HELLO messages +intended for friend only mode should be included, a timeout how long the +operation should take and a callback with a callback closure to be called +for the results. +If you want to obtain information for a specific peer, you can specify +the peer identity, if this identity is NULL, information for all peers are +returned. The function returns a handle to allow to cancel the operation +using @code{GNUNET_PEERINFO_iterate_cancel}. + +To get notified when peer information changes, you can use +@code{GNUNET_PEERINFO_notify}. +This function expects a configuration handle and a flag if friend-only +HELLO messages should be included. The PEERINFO service will notify you +about every change and the callback function will be called to notify you +about changes. The function returns a handle to cancel notifications +with @code{GNUNET_PEERINFO_notify_cancel}. + +@cindex PEERSTORE Subsystem +@node PEERSTORE Subsystem +@section PEERSTORE Subsystem + +@c %**end of header + +GNUnet's PEERSTORE subsystem offers persistent per-peer storage for other +GNUnet subsystems. GNUnet subsystems can use PEERSTORE to persistently +store and retrieve arbitrary data. +Each data record stored with PEERSTORE contains the following fields: + +@itemize @bullet +@item subsystem: Name of the subsystem responsible for the record. +@item peerid: Identity of the peer this record is related to. +@item key: a key string identifying the record. +@item value: binary record value. +@item expiry: record expiry date. +@end itemize + +@menu +* Functionality:: +* Architecture:: +* libgnunetpeerstore:: +@end menu + +@node Functionality +@subsection Functionality + +@c %**end of header + +Subsystems can store any type of value under a (subsystem, peerid, key) +combination. A "replace" flag set during store operations forces the +PEERSTORE to replace any old values stored under the same +(subsystem, peerid, key) combination with the new value. +Additionally, an expiry date is set after which the record is *possibly* +deleted by PEERSTORE. + +Subsystems can iterate over all values stored under any of the following +combination of fields: + +@itemize @bullet +@item (subsystem) +@item (subsystem, peerid) +@item (subsystem, key) +@item (subsystem, peerid, key) +@end itemize + +Subsystems can also request to be notified about any new values stored +under a (subsystem, peerid, key) combination by sending a "watch" +request to PEERSTORE. + +@node Architecture +@subsection Architecture + +@c %**end of header + +PEERSTORE implements the following components: + +@itemize @bullet +@item PEERSTORE service: Handles store, iterate and watch operations. +@item PEERSTORE API: API to be used by other subsystems to communicate and +issue commands to the PEERSTORE service. +@item PEERSTORE plugins: Handles the persistent storage. At the moment, +only an "sqlite" plugin is implemented. +@end itemize + +@cindex libgnunetpeerstore +@node libgnunetpeerstore +@subsection libgnunetpeerstore + +@c %**end of header + +libgnunetpeerstore is the library containing the PEERSTORE API. Subsystems +wishing to communicate with the PEERSTORE service use this API to open a +connection to PEERSTORE. This is done by calling +@code{GNUNET_PEERSTORE_connect} which returns a handle to the newly +created connection. +This handle has to be used with any further calls to the API. + +To store a new record, the function @code{GNUNET_PEERSTORE_store} is to +be used which requires the record fields and a continuation function that +will be called by the API after the STORE request is sent to the +PEERSTORE service. +Note that calling the continuation function does not mean that the record +is successfully stored, only that the STORE request has been successfully +sent to the PEERSTORE service. +@code{GNUNET_PEERSTORE_store_cancel} can be called to cancel the STORE +request only before the continuation function has been called. + +To iterate over stored records, the function +@code{GNUNET_PEERSTORE_iterate} is +to be used. @emph{peerid} and @emph{key} can be set to NULL. An iterator +callback function will be called with each matching record found and a +NULL record at the end to signal the end of result set. +@code{GNUNET_PEERSTORE_iterate_cancel} can be used to cancel the ITERATE +request before the iterator callback is called with a NULL record. + +To be notified with new values stored under a (subsystem, peerid, key) +combination, the function @code{GNUNET_PEERSTORE_watch} is to be used. +This will register the watcher with the PEERSTORE service, any new +records matching the given combination will trigger the callback +function passed to @code{GNUNET_PEERSTORE_watch}. This continues until +@code{GNUNET_PEERSTORE_watch_cancel} is called or the connection to the +service is destroyed. + +After the connection is no longer needed, the function +@code{GNUNET_PEERSTORE_disconnect} can be called to disconnect from the +PEERSTORE service. +Any pending ITERATE or WATCH requests will be destroyed. +If the @code{sync_first} flag is set to @code{GNUNET_YES}, the API will +delay the disconnection until all pending STORE requests are sent to +the PEERSTORE service, otherwise, the pending STORE requests will be +destroyed as well. + +@cindex SET Subsystem +@node SET Subsystem +@section SET Subsystem + +@c %**end of header + +The SET service implements efficient set operations between two peers +over a mesh tunnel. +Currently, set union and set intersection are the only supported +operations. Elements of a set consist of an @emph{element type} and +arbitrary binary @emph{data}. +The size of an element's data is limited to around 62 KB. + +@menu +* Local Sets:: +* Set Modifications:: +* Set Operations:: +* Result Elements:: +* libgnunetset:: +* The SET Client-Service Protocol:: +* The SET Intersection Peer-to-Peer Protocol:: +* The SET Union Peer-to-Peer Protocol:: +@end menu + +@node Local Sets +@subsection Local Sets + +@c %**end of header + +Sets created by a local client can be modified and reused for multiple +operations. As each set operation requires potentially expensive special +auxilliary data to be computed for each element of a set, a set can only +participate in one type of set operation (i.e. union or intersection). +The type of a set is determined upon its creation. +If a the elements of a set are needed for an operation of a different +type, all of the set's element must be copied to a new set of appropriate +type. + +@node Set Modifications +@subsection Set Modifications + +@c %**end of header + +Even when set operations are active, one can add to and remove elements +from a set. +However, these changes will only be visible to operations that have been +created after the changes have taken place. That is, every set operation +only sees a snapshot of the set from the time the operation was started. +This mechanism is @emph{not} implemented by copying the whole set, but by +attaching @emph{generation information} to each element and operation. + +@node Set Operations +@subsection Set Operations + +@c %**end of header + +Set operations can be started in two ways: Either by accepting an +operation request from a remote peer, or by requesting a set operation +from a remote peer. +Set operations are uniquely identified by the involved @emph{peers}, an +@emph{application id} and the @emph{operation type}. + +The client is notified of incoming set operations by @emph{set listeners}. +A set listener listens for incoming operations of a specific operation +type and application id. +Once notified of an incoming set request, the client can accept the set +request (providing a local set for the operation) or reject it. + +@node Result Elements +@subsection Result Elements + +@c %**end of header + +The SET service has three @emph{result modes} that determine how an +operation's result set is delivered to the client: + +@itemize @bullet +@item @strong{Full Result Set.} All elements of set resulting from the set +operation are returned to the client. +@item @strong{Added Elements.} Only elements that result from the +operation and are not already in the local peer's set are returned. +Note that for some operations (like set intersection) this result mode +will never return any elements. +This can be useful if only the remove peer is actually interested in +the result of the set operation. +@item @strong{Removed Elements.} Only elements that are in the local +peer's initial set but not in the operation's result set are returned. +Note that for some operations (like set union) this result mode will +never return any elements. This can be useful if only the remove peer is +actually interested in the result of the set operation. +@end itemize + +@cindex libgnunetset +@node libgnunetset +@subsection libgnunetset + +@c %**end of header + +@menu +* Sets:: +* Listeners:: +* Operations:: +* Supplying a Set:: +* The Result Callback:: +@end menu + +@node Sets +@subsubsection Sets + +@c %**end of header + +New sets are created with @code{GNUNET_SET_create}. Both the local peer's +configuration (as each set has its own client connection) and the +operation type must be specified. +The set exists until either the client calls @code{GNUNET_SET_destroy} or +the client's connection to the service is disrupted. +In the latter case, the client is notified by the return value of +functions dealing with sets. This return value must always be checked. + +Elements are added and removed with @code{GNUNET_SET_add_element} and +@code{GNUNET_SET_remove_element}. + +@node Listeners +@subsubsection Listeners + +@c %**end of header + +Listeners are created with @code{GNUNET_SET_listen}. Each time time a +remote peer suggests a set operation with an application id and operation +type matching a listener, the listener's callback is invoked. +The client then must synchronously call either @code{GNUNET_SET_accept} +or @code{GNUNET_SET_reject}. Note that the operation will not be started +until the client calls @code{GNUNET_SET_commit} +(see Section "Supplying a Set"). + +@node Operations +@subsubsection Operations + +@c %**end of header + +Operations to be initiated by the local peer are created with +@code{GNUNET_SET_prepare}. Note that the operation will not be started +until the client calls @code{GNUNET_SET_commit} +(see Section "Supplying a Set"). + +@node Supplying a Set +@subsubsection Supplying a Set + +@c %**end of header + +To create symmetry between the two ways of starting a set operation +(accepting and nitiating it), the operation handles returned by +@code{GNUNET_SET_accept} and @code{GNUNET_SET_prepare} do not yet have a +set to operate on, thus they can not do any work yet. + +The client must call @code{GNUNET_SET_commit} to specify a set to use for +an operation. @code{GNUNET_SET_commit} may only be called once per set +operation. + +@node The Result Callback +@subsubsection The Result Callback + +@c %**end of header + +Clients must specify both a result mode and a result callback with +@code{GNUNET_SET_accept} and @code{GNUNET_SET_prepare}. The result +callback with a status indicating either that an element was received, or +the operation failed or succeeded. +The interpretation of the received element depends on the result mode. +The callback needs to know which result mode it is used in, as the +arguments do not indicate if an element is part of the full result set, +or if it is in the difference between the original set and the final set. + +@node The SET Client-Service Protocol +@subsection The SET Client-Service Protocol + +@c %**end of header + +@menu +* Creating Sets:: +* Listeners2:: +* Initiating Operations:: +* Modifying Sets:: +* Results and Operation Status:: +* Iterating Sets:: +@end menu + +@node Creating Sets +@subsubsection Creating Sets + +@c %**end of header + +For each set of a client, there exists a client connection to the service. +Sets are created by sending the @code{GNUNET_SERVICE_SET_CREATE} message +over a new client connection. Multiple operations for one set are +multiplexed over one client connection, using a request id supplied by +the client. + +@node Listeners2 +@subsubsection Listeners2 + +@c %**end of header + +Each listener also requires a seperate client connection. By sending the +@code{GNUNET_SERVICE_SET_LISTEN} message, the client notifies the service +of the application id and operation type it is interested in. A client +rejects an incoming request by sending @code{GNUNET_SERVICE_SET_REJECT} +on the listener's client connection. +In contrast, when accepting an incoming request, a +@code{GNUNET_SERVICE_SET_ACCEPT} message must be sent over the@ set that +is supplied for the set operation. + +@node Initiating Operations +@subsubsection Initiating Operations + +@c %**end of header + +Operations with remote peers are initiated by sending a +@code{GNUNET_SERVICE_SET_EVALUATE} message to the service. The@ client +connection that this message is sent by determines the set to use. + +@node Modifying Sets +@subsubsection Modifying Sets + +@c %**end of header + +Sets are modified with the @code{GNUNET_SERVICE_SET_ADD} and +@code{GNUNET_SERVICE_SET_REMOVE} messages. + + +@c %@menu +@c %* Results and Operation Status:: +@c %* Iterating Sets:: +@c %@end menu + +@node Results and Operation Status +@subsubsection Results and Operation Status +@c %**end of header + +The service notifies the client of result elements and success/failure of +a set operation with the @code{GNUNET_SERVICE_SET_RESULT} message. + +@node Iterating Sets +@subsubsection Iterating Sets + +@c %**end of header + +All elements of a set can be requested by sending +@code{GNUNET_SERVICE_SET_ITER_REQUEST}. The server responds with +@code{GNUNET_SERVICE_SET_ITER_ELEMENT} and eventually terminates the +iteration with @code{GNUNET_SERVICE_SET_ITER_DONE}. +After each received element, the client +must send @code{GNUNET_SERVICE_SET_ITER_ACK}. Note that only one set +iteration may be active for a set at any given time. + +@node The SET Intersection Peer-to-Peer Protocol +@subsection The SET Intersection Peer-to-Peer Protocol + +@c %**end of header + +The intersection protocol operates over CADET and starts with a +GNUNET_MESSAGE_TYPE_SET_P2P_OPERATION_REQUEST being sent by the peer +initiating the operation to the peer listening for inbound requests. +It includes the number of elements of the initiating peer, which is used +to decide which side will send a Bloom filter first. + +The listening peer checks if the operation type and application +identifier are acceptable for its current state. +If not, it responds with a GNUNET_MESSAGE_TYPE_SET_RESULT and a status of +GNUNET_SET_STATUS_FAILURE (and terminates the CADET channel). + +If the application accepts the request, the listener sends back a +@code{GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_ELEMENT_INFO} if it has +more elements in the set than the client. +Otherwise, it immediately starts with the Bloom filter exchange. +If the initiator receives a +@code{GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_ELEMENT_INFO} response, +it beings the Bloom filter exchange, unless the set size is indicated to +be zero, in which case the intersection is considered finished after +just the initial handshake. + + +@menu +* The Bloom filter exchange:: +* Salt:: +@end menu + +@node The Bloom filter exchange +@subsubsection The Bloom filter exchange + +@c %**end of header + +In this phase, each peer transmits a Bloom filter over the remaining +keys of the local set to the other peer using a +@code{GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_BF} message. This +message additionally includes the number of elements left in the sender's +set, as well as the XOR over all of the keys in that set. + +The number of bits 'k' set per element in the Bloom filter is calculated +based on the relative size of the two sets. +Furthermore, the size of the Bloom filter is calculated based on 'k' and +the number of elements in the set to maximize the amount of data filtered +per byte transmitted on the wire (while avoiding an excessively high +number of iterations). + +The receiver of the message removes all elements from its local set that +do not pass the Bloom filter test. +It then checks if the set size of the sender and the XOR over the keys +match what is left of his own set. If they do, he sends a +@code{GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_DONE} back to indicate +that the latest set is the final result. +Otherwise, the receiver starts another Bloom fitler exchange, except +this time as the sender. + +@node Salt +@subsubsection Salt + +@c %**end of header + +Bloomfilter operations are probablistic: With some non-zero probability +the test may incorrectly say an element is in the set, even though it is +not. + +To mitigate this problem, the intersection protocol iterates exchanging +Bloom filters using a different random 32-bit salt in each iteration (the +salt is also included in the message). +With different salts, set operations may fail for different elements. +Merging the results from the executions, the probability of failure drops +to zero. + +The iterations terminate once both peers have established that they have +sets of the same size, and where the XOR over all keys computes the same +512-bit value (leaving a failure probability of 2-511). + +@node The SET Union Peer-to-Peer Protocol +@subsection The SET Union Peer-to-Peer Protocol + +@c %**end of header + +The SET union protocol is based on Eppstein's efficient set reconciliation +without prior context. You should read this paper first if you want to +understand the protocol. + +The union protocol operates over CADET and starts with a +GNUNET_MESSAGE_TYPE_SET_P2P_OPERATION_REQUEST being sent by the peer +initiating the operation to the peer listening for inbound requests. +It includes the number of elements of the initiating peer, which is +currently not used. + +The listening peer checks if the operation type and application +identifier are acceptable for its current state. If not, it responds with +a @code{GNUNET_MESSAGE_TYPE_SET_RESULT} and a status of +@code{GNUNET_SET_STATUS_FAILURE} (and terminates the CADET channel). + +If the application accepts the request, it sends back a strata estimator +using a message of type GNUNET_MESSAGE_TYPE_SET_UNION_P2P_SE. The +initiator evaluates the strata estimator and initiates the exchange of +invertible Bloom filters, sending a GNUNET_MESSAGE_TYPE_SET_UNION_P2P_IBF. + +During the IBF exchange, if the receiver cannot invert the Bloom filter or +detects a cycle, it sends a larger IBF in response (up to a defined +maximum limit; if that limit is reached, the operation fails). +Elements decoded while processing the IBF are transmitted to the other +peer using GNUNET_MESSAGE_TYPE_SET_P2P_ELEMENTS, or requested from the +other peer using GNUNET_MESSAGE_TYPE_SET_P2P_ELEMENT_REQUESTS messages, +depending on the sign observed during decoding of the IBF. +Peers respond to a GNUNET_MESSAGE_TYPE_SET_P2P_ELEMENT_REQUESTS message +with the respective element in a GNUNET_MESSAGE_TYPE_SET_P2P_ELEMENTS +message. If the IBF fully decodes, the peer responds with a +GNUNET_MESSAGE_TYPE_SET_UNION_P2P_DONE message instead of another +GNUNET_MESSAGE_TYPE_SET_UNION_P2P_IBF. + +All Bloom filter operations use a salt to mingle keys before hasing them +into buckets, such that future iterations have a fresh chance of +succeeding if they failed due to collisions before. + +@cindex STATISTICS Subsystem +@node STATISTICS Subsystem +@section STATISTICS Subsystem + +@c %**end of header + +In GNUnet, the STATISTICS subsystem offers a central place for all +subsystems to publish unsigned 64-bit integer run-time statistics. +Keeping this information centrally means that there is a unified way for +the user to obtain data on all subsystems, and individual subsystems do +not have to always include a custom data export method for performance +metrics and other statistics. For example, the TRANSPORT system uses +STATISTICS to update information about the number of directly connected +peers and the bandwidth that has been consumed by the various plugins. +This information is valuable for diagnosing connectivity and performance +issues. + +Following the GNUnet service architecture, the STATISTICS subsystem is +divided into an API which is exposed through the header +@strong{gnunet_statistics_service.h} and the STATISTICS service +@strong{gnunet-service-statistics}. The @strong{gnunet-statistics} +command-line tool can be used to obtain (and change) information about +the values stored by the STATISTICS service. The STATISTICS service does +not communicate with other peers. + +Data is stored in the STATISTICS service in the form of tuples +@strong{(subsystem, name, value, persistence)}. The subsystem determines +to which other GNUnet's subsystem the data belongs. name is the name +through which value is associated. It uniquely identifies the record +from among other records belonging to the same subsystem. +In some parts of the code, the pair @strong{(subsystem, name)} is called +a @strong{statistic} as it identifies the values stored in the STATISTCS +service.The persistence flag determines if the record has to be preserved +across service restarts. A record is said to be persistent if this flag +is set for it; if not, the record is treated as a non-persistent record +and it is lost after service restart. Persistent records are written to +and read from the file @strong{statistics.data} before shutdown +and upon startup. The file is located in the HOME directory of the peer. + +An anomaly of the STATISTICS service is that it does not terminate +immediately upon receiving a shutdown signal if it has any clients +connected to it. It waits for all the clients that are not monitors to +close their connections before terminating itself. +This is to prevent the loss of data during peer shutdown --- delaying the +STATISTICS service shutdown helps other services to store important data +to STATISTICS during shutdown. + +@menu +* libgnunetstatistics:: +* The STATISTICS Client-Service Protocol:: +@end menu + +@cindex libgnunetstatistics +@node libgnunetstatistics +@subsection libgnunetstatistics + +@c %**end of header + +@strong{libgnunetstatistics} is the library containing the API for the +STATISTICS subsystem. Any process requiring to use STATISTICS should use +this API by to open a connection to the STATISTICS service. +This is done by calling the function @code{GNUNET_STATISTICS_create()}. +This function takes the subsystem's name which is trying to use STATISTICS +and a configuration. +All values written to STATISTICS with this connection will be placed in +the section corresponding to the given subsystem's name. +The connection to STATISTICS can be destroyed with the function +@code{GNUNET_STATISTICS_destroy()}. This function allows for the +connection to be destroyed immediately or upon transferring all +pending write requests to the service. + +Note: STATISTICS subsystem can be disabled by setting @code{DISABLE = YES} +under the @code{[STATISTICS]} section in the configuration. With such a +configuration all calls to @code{GNUNET_STATISTICS_create()} return +@code{NULL} as the STATISTICS subsystem is unavailable and no other +functions from the API can be used. + + +@menu +* Statistics retrieval:: +* Setting statistics and updating them:: +* Watches:: +@end menu + +@node Statistics retrieval +@subsubsection Statistics retrieval + +@c %**end of header + +Once a connection to the statistics service is obtained, information +about any other system which uses statistics can be retrieved with the +function GNUNET_STATISTICS_get(). +This function takes the connection handle, the name of the subsystem +whose information we are interested in (a @code{NULL} value will +retrieve information of all available subsystems using STATISTICS), the +name of the statistic we are interested in (a @code{NULL} value will +retrieve all available statistics), a continuation callback which is +called when all of requested information is retrieved, an iterator +callback which is called for each parameter in the retrieved information +and a closure for the aforementioned callbacks. The library then invokes +the iterator callback for each value matching the request. + +Call to @code{GNUNET_STATISTICS_get()} is asynchronous and can be +canceled with the function @code{GNUNET_STATISTICS_get_cancel()}. +This is helpful when retrieving statistics takes too long and especially +when we want to shutdown and cleanup everything. + +@node Setting statistics and updating them +@subsubsection Setting statistics and updating them + +@c %**end of header + +So far we have seen how to retrieve statistics, here we will learn how we +can set statistics and update them so that other subsystems can retrieve +them. + +A new statistic can be set using the function +@code{GNUNET_STATISTICS_set()}. +This function takes the name of the statistic and its value and a flag to +make the statistic persistent. +The value of the statistic should be of the type @code{uint64_t}. +The function does not take the name of the subsystem; it is determined +from the previous @code{GNUNET_STATISTICS_create()} invocation. If +the given statistic is already present, its value is overwritten. + +An existing statistics can be updated, i.e its value can be increased or +decreased by an amount with the function +@code{GNUNET_STATISTICS_update()}. +The parameters to this function are similar to +@code{GNUNET_STATISTICS_set()}, except that it takes the amount to be +changed as a type @code{int64_t} instead of the value. + +The library will combine multiple set or update operations into one +message if the client performs requests at a rate that is faster than the +available IPC with the STATISTICS service. Thus, the client does not have +to worry about sending requests too quickly. + +@node Watches +@subsubsection Watches + +@c %**end of header + +As interesting feature of STATISTICS lies in serving notifications +whenever a statistic of our interest is modified. +This is achieved by registering a watch through the function +@code{GNUNET_STATISTICS_watch()}. +The parameters of this function are similar to those of +@code{GNUNET_STATISTICS_get()}. +Changes to the respective statistic's value will then cause the given +iterator callback to be called. +Note: A watch can only be registered for a specific statistic. Hence +the subsystem name and the parameter name cannot be @code{NULL} in a +call to @code{GNUNET_STATISTICS_watch()}. + +A registered watch will keep notifying any value changes until +@code{GNUNET_STATISTICS_watch_cancel()} is called with the same +parameters that are used for registering the watch. + +@node The STATISTICS Client-Service Protocol +@subsection The STATISTICS Client-Service Protocol +@c %**end of header + + +@menu +* Statistics retrieval2:: +* Setting and updating statistics:: +* Watching for updates:: +@end menu + +@node Statistics retrieval2 +@subsubsection Statistics retrieval2 + +@c %**end of header + +To retrieve statistics, the client transmits a message of type +@code{GNUNET_MESSAGE_TYPE_STATISTICS_GET} containing the given subsystem +name and statistic parameter to the STATISTICS service. +The service responds with a message of type +@code{GNUNET_MESSAGE_TYPE_STATISTICS_VALUE} for each of the statistics +parameters that match the client request for the client. The end of +information retrieved is signaled by the service by sending a message of +type @code{GNUNET_MESSAGE_TYPE_STATISTICS_END}. + +@node Setting and updating statistics +@subsubsection Setting and updating statistics + +@c %**end of header + +The subsystem name, parameter name, its value and the persistence flag are +communicated to the service through the message +@code{GNUNET_MESSAGE_TYPE_STATISTICS_SET}. + +When the service receives a message of type +@code{GNUNET_MESSAGE_TYPE_STATISTICS_SET}, it retrieves the subsystem +name and checks for a statistic parameter with matching the name given in +the message. +If a statistic parameter is found, the value is overwritten by the new +value from the message; if not found then a new statistic parameter is +created with the given name and value. + +In addition to just setting an absolute value, it is possible to perform a +relative update by sending a message of type +@code{GNUNET_MESSAGE_TYPE_STATISTICS_SET} with an update flag +(@code{GNUNET_STATISTICS_SETFLAG_RELATIVE}) signifying that the value in +the message should be treated as an update value. + +@node Watching for updates +@subsubsection Watching for updates + +@c %**end of header + +The function registers the watch at the service by sending a message of +type @code{GNUNET_MESSAGE_TYPE_STATISTICS_WATCH}. The service then sends +notifications through messages of type +@code{GNUNET_MESSAGE_TYPE_STATISTICS_WATCH_VALUE} whenever the statistic +parameter's value is changed. + +@cindex DHT +@cindex Distributed Hash Table +@node Distributed Hash Table (DHT) +@section Distributed Hash Table (DHT) + +@c %**end of header + +GNUnet includes a generic distributed hash table that can be used by +developers building P2P applications in the framework. +This section documents high-level features and how developers are +expected to use the DHT. +We have a research paper detailing how the DHT works. +Also, Nate's thesis includes a detailed description and performance +analysis (in chapter 6). + +Key features of GNUnet's DHT include: + +@itemize @bullet +@item stores key-value pairs with values up to (approximately) 63k in size +@item works with many underlay network topologies (small-world, random +graph), underlay does not need to be a full mesh / clique +@item support for extended queries (more than just a simple 'key'), +filtering duplicate replies within the network (bloomfilter) and content +validation (for details, please read the subsection on the block library) +@item can (optionally) return paths taken by the PUT and GET operations +to the application +@item provides content replication to handle churn +@end itemize + +GNUnet's DHT is randomized and unreliable. Unreliable means that there is +no strict guarantee that a value stored in the DHT is always +found --- values are only found with high probability. +While this is somewhat true in all P2P DHTs, GNUnet developers should be +particularly wary of this fact (this will help you write secure, +fault-tolerant code). Thus, when writing any application using the DHT, +you should always consider the possibility that a value stored in the +DHT by you or some other peer might simply not be returned, or returned +with a significant delay. +Your application logic must be written to tolerate this (naturally, some +loss of performance or quality of service is expected in this case). + +@menu +* Block library and plugins:: +* libgnunetdht:: +* The DHT Client-Service Protocol:: +* The DHT Peer-to-Peer Protocol:: +@end menu + +@node Block library and plugins +@subsection Block library and plugins + +@c %**end of header + +@menu +* What is a Block?:: +* The API of libgnunetblock:: +* Queries:: +* Sample Code:: +* Conclusion2:: +@end menu + +@node What is a Block? +@subsubsection What is a Block? + +@c %**end of header + +Blocks are small (< 63k) pieces of data stored under a key (struct +GNUNET_HashCode). Blocks have a type (enum GNUNET_BlockType) which defines +their data format. Blocks are used in GNUnet as units of static data +exchanged between peers and stored (or cached) locally. +Uses of blocks include file-sharing (the files are broken up into blocks), +the VPN (DNS information is stored in blocks) and the DHT (all +information in the DHT and meta-information for the maintenance of the +DHT are both stored using blocks). +The block subsystem provides a few common functions that must be +available for any type of block. + +@cindex libgnunetblock API +@node The API of libgnunetblock +@subsubsection The API of libgnunetblock + +@c %**end of header + +The block library requires for each (family of) block type(s) a block +plugin (implementing @file{gnunet_block_plugin.h}) that provides basic +functions that are needed by the DHT (and possibly other subsystems) to +manage the block. +These block plugins are typically implemented within their respective +subsystems. +The main block library is then used to locate, load and query the +appropriate block plugin. +Which plugin is appropriate is determined by the block type (which is +just a 32-bit integer). Block plugins contain code that specifies which +block types are supported by a given plugin. The block library loads all +block plugins that are installed at the local peer and forwards the +application request to the respective plugin. + +The central functions of the block APIs (plugin and main library) are to +allow the mapping of blocks to their respective key (if possible) and the +ability to check that a block is well-formed and matches a given +request (again, if possible). +This way, GNUnet can avoid storing invalid blocks, storing blocks under +the wrong key and forwarding blocks in response to a query that they do +not answer. + +One key function of block plugins is that it allows GNUnet to detect +duplicate replies (via the Bloom filter). All plugins MUST support +detecting duplicate replies (by adding the current response to the +Bloom filter and rejecting it if it is encountered again). +If a plugin fails to do this, responses may loop in the network. + +@node Queries +@subsubsection Queries +@c %**end of header + +The query format for any block in GNUnet consists of four main components. +First, the type of the desired block must be specified. Second, the query +must contain a hash code. The hash code is used for lookups in hash +tables and databases and must not be unique for the block (however, if +possible a unique hash should be used as this would be best for +performance). +Third, an optional Bloom filter can be specified to exclude known results; +replies that hash to the bits set in the Bloom filter are considered +invalid. False-positives can be eliminated by sending the same query +again with a different Bloom filter mutator value, which parameterizes +the hash function that is used. +Finally, an optional application-specific "eXtended query" (xquery) can +be specified to further constrain the results. It is entirely up to +the type-specific plugin to determine whether or not a given block +matches a query (type, hash, Bloom filter, and xquery). +Naturally, not all xquery's are valid and some types of blocks may not +support Bloom filters either, so the plugin also needs to check if the +query is valid in the first place. + +Depending on the results from the plugin, the DHT will then discard the +(invalid) query, forward the query, discard the (invalid) reply, cache the +(valid) reply, and/or forward the (valid and non-duplicate) reply. + +@node Sample Code +@subsubsection Sample Code + +@c %**end of header + +The source code in @strong{plugin_block_test.c} is a good starting point +for new block plugins --- it does the minimal work by implementing a +plugin that performs no validation at all. +The respective @strong{Makefile.am} shows how to build and install a +block plugin. + +@node Conclusion2 +@subsubsection Conclusion2 + +@c %**end of header + +In conclusion, GNUnet subsystems that want to use the DHT need to define a +block format and write a plugin to match queries and replies. For testing, +the @code{GNUNET_BLOCK_TYPE_TEST} block type can be used; it accepts +any query as valid and any reply as matching any query. +This type is also used for the DHT command line tools. +However, it should NOT be used for normal applications due to the lack +of error checking that results from this primitive implementation. + +@cindex libgnunetdht +@node libgnunetdht +@subsection libgnunetdht + +@c %**end of header + +The DHT API itself is pretty simple and offers the usual GET and PUT +functions that work as expected. The specified block type refers to the +block library which allows the DHT to run application-specific logic for +data stored in the network. + + +@menu +* GET:: +* PUT:: +* MONITOR:: +* DHT Routing Options:: +@end menu + +@node GET +@subsubsection GET + +@c %**end of header + +When using GET, the main consideration for developers (other than the +block library) should be that after issuing a GET, the DHT will +continuously cause (small amounts of) network traffic until the operation +is explicitly canceled. +So GET does not simply send out a single network request once; instead, +the DHT will continue to search for data. This is needed to achieve good +success rates and also handles the case where the respective PUT +operation happens after the GET operation was started. +Developers should not cancel an existing GET operation and then +explicitly re-start it to trigger a new round of network requests; +this is simply inefficient, especially as the internal automated version +can be more efficient, for example by filtering results in the network +that have already been returned. + +If an application that performs a GET request has a set of replies that it +already knows and would like to filter, it can call@ +@code{GNUNET_DHT_get_filter_known_results} with an array of hashes over +the respective blocks to tell the DHT that these results are not +desired (any more). +This way, the DHT will filter the respective blocks using the block +library in the network, which may result in a significant reduction in +bandwidth consumption. + +@node PUT +@subsubsection PUT + +@c %**end of header + +In contrast to GET operations, developers @strong{must} manually re-run +PUT operations periodically (if they intend the content to continue to be +available). Content stored in the DHT expires or might be lost due to +churn. +Furthermore, GNUnet's DHT typically requires multiple rounds of PUT +operations before a key-value pair is consistently available to all +peers (the DHT randomizes paths and thus storage locations, and only +after multiple rounds of PUTs there will be a sufficient number of +replicas in large DHTs). An explicit PUT operation using the DHT API will +only cause network traffic once, so in order to ensure basic availability +and resistance to churn (and adversaries), PUTs must be repeated. +While the exact frequency depends on the application, a rule of thumb is +that there should be at least a dozen PUT operations within the content +lifetime. Content in the DHT typically expires after one day, so +DHT PUT operations should be repeated at least every 1-2 hours. + +@node MONITOR +@subsubsection MONITOR + +@c %**end of header + +The DHT API also allows applications to monitor messages crossing the +local DHT service. +The types of messages used by the DHT are GET, PUT and RESULT messages. +Using the monitoring API, applications can choose to monitor these +requests, possibly limiting themselves to requests for a particular block +type. + +The monitoring API is not only usefu only for diagnostics, it can also be +used to trigger application operations based on PUT operations. +For example, an application may use PUTs to distribute work requests to +other peers. +The workers would then monitor for PUTs that give them work, instead of +looking for work using GET operations. +This can be beneficial, especially if the workers have no good way to +guess the keys under which work would be stored. +Naturally, additional protocols might be needed to ensure that the desired +number of workers will process the distributed workload. + +@node DHT Routing Options +@subsubsection DHT Routing Options + +@c %**end of header + +There are two important options for GET and PUT requests: + +@table @asis +@item GNUNET_DHT_RO_DEMULITPLEX_EVERYWHERE This option means that all +peers should process the request, even if their peer ID is not closest to +the key. For a PUT request, this means that all peers that a request +traverses may make a copy of the data. +Similarly for a GET request, all peers will check their local database +for a result. Setting this option can thus significantly improve caching +and reduce bandwidth consumption --- at the expense of a larger DHT +database. If in doubt, we recommend that this option should be used. +@item GNUNET_DHT_RO_RECORD_ROUTE This option instructs the DHT to record +the path that a GET or a PUT request is taking through the overlay +network. The resulting paths are then returned to the application with +the respective result. This allows the receiver of a result to construct +a path to the originator of the data, which might then be used for +routing. Naturally, setting this option requires additional bandwidth +and disk space, so applications should only set this if the paths are +needed by the application logic. +@item GNUNET_DHT_RO_FIND_PEER This option is an internal option used by +the DHT's peer discovery mechanism and should not be used by applications. +@item GNUNET_DHT_RO_BART This option is currently not implemented. It may +in the future offer performance improvements for clique topologies. +@end table + +@node The DHT Client-Service Protocol +@subsection The DHT Client-Service Protocol + +@c %**end of header + +@menu +* PUTting data into the DHT:: +* GETting data from the DHT:: +* Monitoring the DHT:: +@end menu + +@node PUTting data into the DHT +@subsubsection PUTting data into the DHT + +@c %**end of header + +To store (PUT) data into the DHT, the client sends a +@code{struct GNUNET_DHT_ClientPutMessage} to the service. +This message specifies the block type, routing options, the desired +replication level, the expiration time, key, +value and a 64-bit unique ID for the operation. The service responds with +a @code{struct GNUNET_DHT_ClientPutConfirmationMessage} with the same +64-bit unique ID. Note that the service sends the confirmation as soon as +it has locally processed the PUT request. The PUT may still be +propagating through the network at this time. + +In the future, we may want to change this to provide (limited) feedback +to the client, for example if we detect that the PUT operation had no +effect because the same key-value pair was already stored in the DHT. +However, changing this would also require additional state and messages +in the P2P interaction. + +@node GETting data from the DHT +@subsubsection GETting data from the DHT + +@c %**end of header + +To retrieve (GET) data from the DHT, the client sends a +@code{struct GNUNET_DHT_ClientGetMessage} to the service. The message +specifies routing options, a replication level (for replicating the GET, +not the content), the desired block type, the key, the (optional) +extended query and unique 64-bit request ID. + +Additionally, the client may send any number of +@code{struct GNUNET_DHT_ClientGetResultSeenMessage}s to notify the +service about results that the client is already aware of. +These messages consist of the key, the unique 64-bit ID of the request, +and an arbitrary number of hash codes over the blocks that the client is +already aware of. As messages are restricted to 64k, a client that +already knows more than about a thousand blocks may need to send +several of these messages. Naturally, the client should transmit these +messages as quickly as possible after the original GET request such that +the DHT can filter those results in the network early on. Naturally, as +these messages are send after the original request, it is conceivalbe +that the DHT service may return blocks that match those already known +to the client anyway. + +In response to a GET request, the service will send @code{struct +GNUNET_DHT_ClientResultMessage}s to the client. These messages contain the +block type, expiration, key, unique ID of the request and of course the +value (a block). Depending on the options set for the respective +operations, the replies may also contain the path the GET and/or the PUT +took through the network. + +A client can stop receiving replies either by disconnecting or by sending +a @code{struct GNUNET_DHT_ClientGetStopMessage} which must contain the +key and the 64-bit unique ID of the original request. Using an +explicit "stop" message is more common as this allows a client to run +many concurrent GET operations over the same connection with the DHT +service --- and to stop them individually. + +@node Monitoring the DHT +@subsubsection Monitoring the DHT + +@c %**end of header + +To begin monitoring, the client sends a +@code{struct GNUNET_DHT_MonitorStartStop} message to the DHT service. +In this message, flags can be set to enable (or disable) monitoring of +GET, PUT and RESULT messages that pass through a peer. The message can +also restrict monitoring to a particular block type or a particular key. +Once monitoring is enabled, the DHT service will notify the client about +any matching event using @code{struct GNUNET_DHT_MonitorGetMessage}s for +GET events, @code{struct GNUNET_DHT_MonitorPutMessage} for PUT events +and @code{struct GNUNET_DHT_MonitorGetRespMessage} for RESULTs. Each of +these messages contains all of the information about the event. + +@node The DHT Peer-to-Peer Protocol +@subsection The DHT Peer-to-Peer Protocol +@c %**end of header + + +@menu +* Routing GETs or PUTs:: +* PUTting data into the DHT2:: +* GETting data from the DHT2:: +@end menu + +@node Routing GETs or PUTs +@subsubsection Routing GETs or PUTs + +@c %**end of header + +When routing GETs or PUTs, the DHT service selects a suitable subset of +neighbours for forwarding. The exact number of neighbours can be zero or +more and depends on the hop counter of the query (initially zero) in +relation to the (log of) the network size estimate, the desired +replication level and the peer's connectivity. +Depending on the hop counter and our network size estimate, the selection +of the peers maybe randomized or by proximity to the key. +Furthermore, requests include a set of peers that a request has already +traversed; those peers are also excluded from the selection. + +@node PUTting data into the DHT2 +@subsubsection PUTting data into the DHT2 + +@c %**end of header + +To PUT data into the DHT, the service sends a @code{struct PeerPutMessage} +of type @code{GNUNET_MESSAGE_TYPE_DHT_P2P_PUT} to the respective +neighbour. +In addition to the usual information about the content (type, routing +options, desired replication level for the content, expiration time, key +and value), the message contains a fixed-size Bloom filter with +information about which peers (may) have already seen this request. +This Bloom filter is used to ensure that DHT messages never loop back to +a peer that has already processed the request. +Additionally, the message includes the current hop counter and, depending +on the routing options, the message may include the full path that the +message has taken so far. +The Bloom filter should already contain the identity of the previous hop; +however, the path should not include the identity of the previous hop and +the receiver should append the identity of the sender to the path, not +its own identity (this is done to reduce bandwidth). + +@node GETting data from the DHT2 +@subsubsection GETting data from the DHT2 + +@c %**end of header + +A peer can search the DHT by sending @code{struct PeerGetMessage}s of type +@code{GNUNET_MESSAGE_TYPE_DHT_P2P_GET} to other peers. In addition to the +usual information about the request (type, routing options, desired +replication level for the request, the key and the extended query), a GET +request also again contains a hop counter, a Bloom filter over the peers +that have processed the request already and depending on the routing +options the full path traversed by the GET. +Finally, a GET request includes a variable-size second Bloom filter and a +so-called Bloom filter mutator value which together indicate which +replies the sender has already seen. During the lookup, each block that +matches they block type, key and extended query is additionally subjected +to a test against this Bloom filter. +The block plugin is expected to take the hash of the block and combine it +with the mutator value and check if the result is not yet in the Bloom +filter. The originator of the query will from time to time modify the +mutator to (eventually) allow false-positives filtered by the Bloom filter +to be returned. + +Peers that receive a GET request perform a local lookup (depending on +their proximity to the key and the query options) and forward the request +to other peers. +They then remember the request (including the Bloom filter for blocking +duplicate results) and when they obtain a matching, non-filtered response +a @code{struct PeerResultMessage} of type +@code{GNUNET_MESSAGE_TYPE_DHT_P2P_RESULT} is forwarded to the previous +hop. +Whenver a result is forwarded, the block plugin is used to update the +Bloom filter accordingly, to ensure that the same result is never +forwarded more than once. +The DHT service may also cache forwarded results locally if the +"CACHE_RESULTS" option is set to "YES" in the configuration. + +@cindex GNS +@cindex GNU Name System +@node GNU Name System (GNS) +@section GNU Name System (GNS) + +@c %**end of header + +The GNU Name System (GNS) is a decentralized database that enables users +to securely resolve names to values. +Names can be used to identify other users (for example, in social +networking), or network services (for example, VPN services running at a +peer in GNUnet, or purely IP-based services on the Internet). +Users interact with GNS by typing in a hostname that ends in ".gnu" +or ".zkey". + +Videos giving an overview of most of the GNS and the motivations behind +it is available here and here. +The remainder of this chapter targets developers that are familiar with +high level concepts of GNS as presented in these talks. +@c TODO: Add links to here and here and to these. + +GNS-aware applications should use the GNS resolver to obtain the +respective records that are stored under that name in GNS. +Each record consists of a type, value, expiration time and flags. + +The type specifies the format of the value. Types below 65536 correspond +to DNS record types, larger values are used for GNS-specific records. +Applications can define new GNS record types by reserving a number and +implementing a plugin (which mostly needs to convert the binary value +representation to a human-readable text format and vice-versa). +The expiration time specifies how long the record is to be valid. +The GNS API ensures that applications are only given non-expired values. +The flags are typically irrelevant for applications, as GNS uses them +internally to control visibility and validity of records. + +Records are stored along with a signature. +The signature is generated using the private key of the authoritative +zone. This allows any GNS resolver to verify the correctness of a +name-value mapping. + +Internally, GNS uses the NAMECACHE to cache information obtained from +other users, the NAMESTORE to store information specific to the local +users, and the DHT to exchange data between users. +A plugin API is used to enable applications to define new GNS +record types. + +@menu +* libgnunetgns:: +* libgnunetgnsrecord:: +* GNS plugins:: +* The GNS Client-Service Protocol:: +* Hijacking the DNS-Traffic using gnunet-service-dns:: +* Serving DNS lookups via GNS on W32:: +@end menu + +@node libgnunetgns +@subsection libgnunetgns + +@c %**end of header + +The GNS API itself is extremely simple. Clients first connec to the +GNS service using @code{GNUNET_GNS_connect}. +They can then perform lookups using @code{GNUNET_GNS_lookup} or cancel +pending lookups using @code{GNUNET_GNS_lookup_cancel}. +Once finished, clients disconnect using @code{GNUNET_GNS_disconnect}. + +@menu +* Looking up records:: +* Accessing the records:: +* Creating records:: +* Future work:: +@end menu + +@node Looking up records +@subsubsection Looking up records + +@c %**end of header + +@code{GNUNET_GNS_lookup} takes a number of arguments: + +@table @asis +@item handle This is simply the GNS connection handle from +@code{GNUNET_GNS_connect}. +@item name The client needs to specify the name to +be resolved. This can be any valid DNS or GNS hostname. +@item zone The client +needs to specify the public key of the GNS zone against which the +resolution should be done (the ".gnu" zone). +Note that a key must be provided, even if the name ends in ".zkey". +This should typically be the public key of the master-zone of the user. +@item type This is the desired GNS or DNS record type +to look for. While all records for the given name will be returned, this +can be important if the client wants to resolve record types that +themselves delegate resolution, such as CNAME, PKEY or GNS2DNS. +Resolving a record of any of these types will only work if the respective +record type is specified in the request, as the GNS resolver will +otherwise follow the delegation and return the records from the +respective destination, instead of the delegating record. +@item only_cached This argument should typically be set to +@code{GNUNET_NO}. Setting it to @code{GNUNET_YES} disables resolution via +the overlay network. +@item shorten_zone_key If GNS encounters new names during resolution, +their respective zones can automatically be learned and added to the +"shorten zone". If this is desired, clients must pass the private key of +the shorten zone. If NULL is passed, shortening is disabled. +@item proc This argument identifies +the function to call with the result. It is given proc_cls, the number of +records found (possilby zero) and the array of the records as arguments. +proc will only be called once. After proc,> has been called, the lookup +must no longer be cancelled. +@item proc_cls The closure for proc. +@end table + +@node Accessing the records +@subsubsection Accessing the records + +@c %**end of header + +The @code{libgnunetgnsrecord} library provides an API to manipulate the +GNS record array that is given to proc. In particular, it offers +functions such as converting record values to human-readable +strings (and back). However, most @code{libgnunetgnsrecord} functions are +not interesting to GNS client applications. + +For DNS records, the @code{libgnunetdnsparser} library provides +functions for parsing (and serializing) common types of DNS records. + +@node Creating records +@subsubsection Creating records + +@c %**end of header + +Creating GNS records is typically done by building the respective record +information (possibly with the help of @code{libgnunetgnsrecord} and +@code{libgnunetdnsparser}) and then using the @code{libgnunetnamestore} to +publish the information. The GNS API is not involved in this +operation. + +@node Future work +@subsubsection Future work + +@c %**end of header + +In the future, we want to expand @code{libgnunetgns} to allow +applications to observe shortening operations performed during GNS +resolution, for example so that users can receive visual feedback when +this happens. + +@node libgnunetgnsrecord +@subsection libgnunetgnsrecord + +@c %**end of header + +The @code{libgnunetgnsrecord} library is used to manipulate GNS +records (in plaintext or in their encrypted format). +Applications mostly interact with @code{libgnunetgnsrecord} by using the +functions to convert GNS record values to strings or vice-versa, or to +lookup a GNS record type number by name (or vice-versa). +The library also provides various other functions that are mostly +used internally within GNS, such as converting keys to names, checking for +expiration, encrypting GNS records to GNS blocks, verifying GNS block +signatures and decrypting GNS records from GNS blocks. + +We will now discuss the four commonly used functions of the API.@ +@code{libgnunetgnsrecord} does not perform these operations itself, +but instead uses plugins to perform the operation. +GNUnet includes plugins to support common DNS record types as well as +standard GNS record types. + +@menu +* Value handling:: +* Type handling:: +@end menu + +@node Value handling +@subsubsection Value handling + +@c %**end of header + +@code{GNUNET_GNSRECORD_value_to_string} can be used to convert +the (binary) representation of a GNS record value to a human readable, +0-terminated UTF-8 string. +NULL is returned if the specified record type is not supported by any +available plugin. + +@code{GNUNET_GNSRECORD_string_to_value} can be used to try to convert a +human readable string to the respective (binary) representation of +a GNS record value. + +@node Type handling +@subsubsection Type handling + +@c %**end of header + +@code{GNUNET_GNSRECORD_typename_to_number} can be used to obtain the +numeric value associated with a given typename. For example, given the +typename "A" (for DNS A reocrds), the function will return the number 1. +A list of common DNS record types is +@uref{http://en.wikipedia.org/wiki/List_of_DNS_record_types, here}. +Note that not all DNS record types are supported by GNUnet GNSRECORD +plugins at this time. + +@code{GNUNET_GNSRECORD_number_to_typename} can be used to obtain the +typename associated with a given numeric value. +For example, given the type number 1, the function will return the +typename "A". + +@node GNS plugins +@subsection GNS plugins + +@c %**end of header + +Adding a new GNS record type typically involves writing (or extending) a +GNSRECORD plugin. The plugin needs to implement the +@code{gnunet_gnsrecord_plugin.h} API which provides basic functions that +are needed by GNSRECORD to convert typenames and values of the respective +record type to strings (and back). +These gnsrecord plugins are typically implemented within their respective +subsystems. +Examples for such plugins can be found in the GNSRECORD, GNS and +CONVERSATION subsystems. + +The @code{libgnunetgnsrecord} library is then used to locate, load and +query the appropriate gnsrecord plugin. +Which plugin is appropriate is determined by the record type (which is +just a 32-bit integer). The @code{libgnunetgnsrecord} library loads all +block plugins that are installed at the local peer and forwards the +application request to the plugins. If the record type is not +supported by the plugin, it should simply return an error code. + +The central functions of the block APIs (plugin and main library) are the +same four functions for converting between values and strings, and +typenames and numbers documented in the previous subsection. + +@node The GNS Client-Service Protocol +@subsection The GNS Client-Service Protocol +@c %**end of header + +The GNS client-service protocol consists of two simple messages, the +@code{LOOKUP} message and the @code{LOOKUP_RESULT}. Each @code{LOOKUP} +message contains a unique 32-bit identifier, which will be included in the +corresponding response. Thus, clients can send many lookup requests in +parallel and receive responses out-of-order. +A @code{LOOKUP} request also includes the public key of the GNS zone, +the desired record type and fields specifying whether shortening is +enabled or networking is disabled. Finally, the @code{LOOKUP} message +includes the name to be resolved. + +The response includes the number of records and the records themselves +in the format created by @code{GNUNET_GNSRECORD_records_serialize}. +They can thus be deserialized using +@code{GNUNET_GNSRECORD_records_deserialize}. + +@node Hijacking the DNS-Traffic using gnunet-service-dns +@subsection Hijacking the DNS-Traffic using gnunet-service-dns + +@c %**end of header + +This section documents how the gnunet-service-dns (and the +gnunet-helper-dns) intercepts DNS queries from the local system. +This is merely one method for how we can obtain GNS queries. +It is also possible to change @code{resolv.conf} to point to a machine +running @code{gnunet-dns2gns} or to modify libc's name system switch +(NSS) configuration to include a GNS resolution plugin. +The method described in this chaper is more of a last-ditch catch-all +approach. + +@code{gnunet-service-dns} enables intercepting DNS traffic using policy +based routing. +We MARK every outgoing DNS-packet if it was not sent by our application. +Using a second routing table in the Linux kernel these marked packets are +then routed through our virtual network interface and can thus be +captured unchanged. + +Our application then reads the query and decides how to handle it: A +query to an address ending in ".gnu" or ".zkey" is hijacked by +@code{gnunet-service-gns} and resolved internally using GNS. +In the future, a reverse query for an address of the configured virtual +network could be answered with records kept about previous forward +queries. +Queries that are not hijacked by some application using the DNS service +will be sent to the original recipient. +The answer to the query will always be sent back through the virtual +interface with the original nameserver as source address. + + +@menu +* Network Setup Details:: +@end menu + +@node Network Setup Details +@subsubsection Network Setup Details + +@c %**end of header + +The DNS interceptor adds the following rules to the Linux kernel: +@example +iptables -t mangle -I OUTPUT 1 -p udp --sport $LOCALPORT --dport 53 \ +-j ACCEPT iptables -t mangle -I OUTPUT 2 -p udp --dport 53 -j MARK \ +--set-mark 3 ip rule add fwmark 3 table2 ip route add default via \ +$VIRTUALDNS table2 +@end example + +@c FIXME: Rewrite to reflect display which is no longer content by line +@c FIXME: due to the < 74 characters limit. +Line 1 makes sure that all packets coming from a port our application +opened beforehand (@code{$LOCALPORT}) will be routed normally. +Line 2 marks every other packet to a DNS-Server with mark 3 (chosen +arbitrarily). The third line adds a routing policy based on this mark +3 via the routing table. + +@node Serving DNS lookups via GNS on W32 +@subsection Serving DNS lookups via GNS on W32 + +@c %**end of header + +This section documents how the libw32nsp (and +gnunet-gns-helper-service-w32) do DNS resolutions of DNS queries on the +local system. This only applies to GNUnet running on W32. + +W32 has a concept of "Namespaces" and "Namespace providers". +These are used to present various name systems to applications in a +generic way. +Namespaces include DNS, mDNS, NLA and others. For each namespace any +number of providers could be registered, and they are queried in an order +of priority (which is adjustable). + +Applications can resolve names by using WSALookupService*() family of +functions. + +However, these are WSA-only facilities. Common BSD socket functions for +namespace resolutions are gethostbyname and getaddrinfo (among others). +These functions are implemented internally (by default - by mswsock, +which also implements the default DNS provider) as wrappers around +WSALookupService*() functions (see "Sample Code for a Service Provider" +on MSDN). + +On W32 GNUnet builds a libw32nsp - a namespace provider, which can then be +installed into the system by using w32nsp-install (and uninstalled by +w32nsp-uninstall), as described in "Installation Handbook". + +libw32nsp is very simple and has almost no dependencies. As a response to +NSPLookupServiceBegin(), it only checks that the provider GUID passed to +it by the caller matches GNUnet DNS Provider GUID, checks that name being +resolved ends in ".gnu" or ".zkey", then connects to +gnunet-gns-helper-service-w32 at 127.0.0.1:5353 (hardcoded) and sends the +name resolution request there, returning the connected socket to the +caller. + +When the caller invokes NSPLookupServiceNext(), libw32nsp reads a +completely formed reply from that socket, unmarshalls it, then gives +it back to the caller. + +At the moment gnunet-gns-helper-service-w32 is implemented to ever give +only one reply, and subsequent calls to NSPLookupServiceNext() will fail +with WSA_NODATA (first call to NSPLookupServiceNext() might also fail if +GNS failed to find the name, or there was an error connecting to it). + +gnunet-gns-helper-service-w32 does most of the processing: + +@itemize @bullet +@item Maintains a connection to GNS. +@item Reads GNS config and loads appropriate keys. +@item Checks service GUID and decides on the type of record to look up, +refusing to make a lookup outright when unsupported service GUID is +passed. +@item Launches the lookup +@end itemize + +When lookup result arrives, gnunet-gns-helper-service-w32 forms a complete +reply (including filling a WSAQUERYSETW structure and, possibly, a binary +blob with a hostent structure for gethostbyname() client), marshalls it, +and sends it back to libw32nsp. If no records were found, it sends an +empty header. + +This works for most normal applications that use gethostbyname() or +getaddrinfo() to resolve names, but fails to do anything with +applications that use alternative means of resolving names (such as +sending queries to a DNS server directly by themselves). +This includes some of well known utilities, like "ping" and "nslookup". + +@cindex GNS Namecache +@node GNS Namecache +@section GNS Namecache + +@c %**end of header + +The NAMECACHE subsystem is responsible for caching (encrypted) resolution +results of the GNU Name System (GNS). GNS makes zone information available +to other users via the DHT. However, as accessing the DHT for every +lookup is expensive (and as the DHT's local cache is lost whenever the +peer is restarted), GNS uses the NAMECACHE as a more persistent cache for +DHT lookups. +Thus, instead of always looking up every name in the DHT, GNS first +checks if the result is already available locally in the NAMECACHE. +Only if there is no result in the NAMECACHE, GNS queries the DHT. +The NAMECACHE stores data in the same (encrypted) format as the DHT. +It thus makes no sense to iterate over all items in the +NAMECACHE --- the NAMECACHE does not have a way to provide the keys +required to decrypt the entries. + +Blocks in the NAMECACHE share the same expiration mechanism as blocks in +the DHT --- the block expires wheneever any of the records in +the (encrypted) block expires. +The expiration time of the block is the only information stored in +plaintext. The NAMECACHE service internally performs all of the required +work to expire blocks, clients do not have to worry about this. +Also, given that NAMECACHE stores only GNS blocks that local users +requested, there is no configuration option to limit the size of the +NAMECACHE. It is assumed to be always small enough (a few MB) to fit on +the drive. + +The NAMECACHE supports the use of different database backends via a +plugin API. + +@menu +* libgnunetnamecache:: +* The NAMECACHE Client-Service Protocol:: +* The NAMECACHE Plugin API:: +@end menu + +@node libgnunetnamecache +@subsection libgnunetnamecache + +@c %**end of header + +The NAMECACHE API consists of five simple functions. First, there is +@code{GNUNET_NAMECACHE_connect} to connect to the NAMECACHE service. +This returns the handle required for all other operations on the +NAMECACHE. Using @code{GNUNET_NAMECACHE_block_cache} clients can insert a +block into the cache. +@code{GNUNET_NAMECACHE_lookup_block} can be used to lookup blocks that +were stored in the NAMECACHE. Both operations can be cancelled using +@code{GNUNET_NAMECACHE_cancel}. Note that cancelling a +@code{GNUNET_NAMECACHE_block_cache} operation can result in the block +being stored in the NAMECACHE --- or not. Cancellation primarily ensures +that the continuation function with the result of the operation will no +longer be invoked. +Finally, @code{GNUNET_NAMECACHE_disconnect} closes the connection to the +NAMECACHE. + +The maximum size of a block that can be stored in the NAMECACHE is +@code{GNUNET_NAMECACHE_MAX_VALUE_SIZE}, which is defined to be 63 kB. + +@node The NAMECACHE Client-Service Protocol +@subsection The NAMECACHE Client-Service Protocol + +@c %**end of header + +All messages in the NAMECACHE IPC protocol start with the +@code{struct GNUNET_NAMECACHE_Header} which adds a request +ID (32-bit integer) to the standard message header. +The request ID is used to match requests with the +respective responses from the NAMECACHE, as they are allowed to happen +out-of-order. + + +@menu +* Lookup:: +* Store:: +@end menu + +@node Lookup +@subsubsection Lookup + +@c %**end of header + +The @code{struct LookupBlockMessage} is used to lookup a block stored in +the cache. +It contains the query hash. The NAMECACHE always responds with a +@code{struct LookupBlockResponseMessage}. If the NAMECACHE has no +response, it sets the expiration time in the response to zero. +Otherwise, the response is expected to contain the expiration time, the +ECDSA signature, the derived key and the (variable-size) encrypted data +of the block. + +@node Store +@subsubsection Store + +@c %**end of header + +The @code{struct BlockCacheMessage} is used to cache a block in the +NAMECACHE. +It has the same structure as the @code{struct LookupBlockResponseMessage}. +The service responds with a @code{struct BlockCacheResponseMessage} which +contains the result of the operation (success or failure). +In the future, we might want to make it possible to provide an error +message as well. + +@node The NAMECACHE Plugin API +@subsection The NAMECACHE Plugin API +@c %**end of header + +The NAMECACHE plugin API consists of two functions, @code{cache_block} to +store a block in the database, and @code{lookup_block} to lookup a block +in the database. + + +@menu +* Lookup2:: +* Store2:: +@end menu + +@node Lookup2 +@subsubsection Lookup2 + +@c %**end of header + +The @code{lookup_block} function is expected to return at most one block +to the iterator, and return @code{GNUNET_NO} if there were no non-expired +results. +If there are multiple non-expired results in the cache, the lookup is +supposed to return the result with the largest expiration time. + +@node Store2 +@subsubsection Store2 + +@c %**end of header + +The @code{cache_block} function is expected to try to store the block in +the database, and return @code{GNUNET_SYSERR} if this was not possible +for any reason. +Furthermore, @code{cache_block} is expected to implicitly perform cache +maintenance and purge blocks from the cache that have expired. Note that +@code{cache_block} might encounter the case where the database already has +another block stored under the same key. In this case, the plugin must +ensure that the block with the larger expiration time is preserved. +Obviously, this can done either by simply adding new blocks and selecting +for the most recent expiration time during lookup, or by checking which +block is more recent during the store operation. + +@cindex REVOCATION Subsystem +@node REVOCATION Subsystem +@section REVOCATION Subsystem +@c %**end of header + +The REVOCATION subsystem is responsible for key revocation of Egos. +If a user learns that theis private key has been compromised or has lost +it, they can use the REVOCATION system to inform all of the other users +that their private key is no longer valid. +The subsystem thus includes ways to query for the validity of keys and to +propagate revocation messages. + +@menu +* Dissemination:: +* Revocation Message Design Requirements:: +* libgnunetrevocation:: +* The REVOCATION Client-Service Protocol:: +* The REVOCATION Peer-to-Peer Protocol:: +@end menu + +@node Dissemination +@subsection Dissemination + +@c %**end of header + +When a revocation is performed, the revocation is first of all +disseminated by flooding the overlay network. +The goal is to reach every peer, so that when a peer needs to check if a +key has been revoked, this will be purely a local operation where the +peer looks at his local revocation list. Flooding the network is also the +most robust form of key revocation --- an adversary would have to control +a separator of the overlay graph to restrict the propagation of the +revocation message. Flooding is also very easy to implement --- peers that +receive a revocation message for a key that they have never seen before +simply pass the message to all of their neighbours. + +Flooding can only distribute the revocation message to peers that are +online. +In order to notify peers that join the network later, the revocation +service performs efficient set reconciliation over the sets of known +revocation messages whenever two peers (that both support REVOCATION +dissemination) connect. +The SET service is used to perform this operation efficiently. + +@node Revocation Message Design Requirements +@subsection Revocation Message Design Requirements + +@c %**end of header + +However, flooding is also quite costly, creating O(|E|) messages on a +network with |E| edges. +Thus, revocation messages are required to contain a proof-of-work, the +result of an expensive computation (which, however, is cheap to verify). +Only peers that have expended the CPU time necessary to provide +this proof will be able to flood the network with the revocation message. +This ensures that an attacker cannot simply flood the network with +millions of revocation messages. The proof-of-work required by GNUnet is +set to take days on a typical PC to compute; if the ability to quickly +revoke a key is needed, users have the option to pre-compute revocation +messages to store off-line and use instantly after their key has expired. + +Revocation messages must also be signed by the private key that is being +revoked. Thus, they can only be created while the private key is in the +possession of the respective user. This is another reason to create a +revocation message ahead of time and store it in a secure location. + +@node libgnunetrevocation +@subsection libgnunetrevocation + +@c %**end of header + +The REVOCATION API consists of two parts, to query and to issue +revocations. + + +@menu +* Querying for revoked keys:: +* Preparing revocations:: +* Issuing revocations:: +@end menu + +@node Querying for revoked keys +@subsubsection Querying for revoked keys + +@c %**end of header + +@code{GNUNET_REVOCATION_query} is used to check if a given ECDSA public +key has been revoked. +The given callback will be invoked with the result of the check. +The query can be cancelled using @code{GNUNET_REVOCATION_query_cancel} on +the return value. + +@node Preparing revocations +@subsubsection Preparing revocations + +@c %**end of header + +It is often desirable to create a revocation record ahead-of-time and +store it in an off-line location to be used later in an emergency. +This is particularly true for GNUnet revocations, where performing the +revocation operation itself is computationally expensive and thus is +likely to take some time. +Thus, if users want the ability to perform revocations quickly in an +emergency, they must pre-compute the revocation message. +The revocation API enables this with two functions that are used to +compute the revocation message, but not trigger the actual revocation +operation. + +@code{GNUNET_REVOCATION_check_pow} should be used to calculate the +proof-of-work required in the revocation message. This function takes the +public key, the required number of bits for the proof of work (which in +GNUnet is a network-wide constant) and finally a proof-of-work number as +arguments. +The function then checks if the given proof-of-work number is a valid +proof of work for the given public key. Clients preparing a revocation +are expected to call this function repeatedly (typically with a +monotonically increasing sequence of numbers of the proof-of-work number) +until a given number satisfies the check. +That number should then be saved for later use in the revocation +operation. + +@code{GNUNET_REVOCATION_sign_revocation} is used to generate the +signature that is required in a revocation message. +It takes the private key that (possibly in the future) is to be revoked +and returns the signature. +The signature can again be saved to disk for later use, which will then +allow performing a revocation even without access to the private key. + +@node Issuing revocations +@subsubsection Issuing revocations + + +Given a ECDSA public key, the signature from @code{GNUNET_REVOCATION_sign} +and the proof-of-work, +@code{GNUNET_REVOCATION_revoke} can be used to perform the +actual revocation. The given callback is called upon completion of the +operation. @code{GNUNET_REVOCATION_revoke_cancel} can be used to stop the +library from calling the continuation; however, in that case it is +undefined whether or not the revocation operation will be executed. + +@node The REVOCATION Client-Service Protocol +@subsection The REVOCATION Client-Service Protocol + + +The REVOCATION protocol consists of four simple messages. + +A @code{QueryMessage} containing a public ECDSA key is used to check if a +particular key has been revoked. The service responds with a +@code{QueryResponseMessage} which simply contains a bit that says if the +given public key is still valid, or if it has been revoked. + +The second possible interaction is for a client to revoke a key by +passing a @code{RevokeMessage} to the service. The @code{RevokeMessage} +contains the ECDSA public key to be revoked, a signature by the +corresponding private key and the proof-of-work, The service responds +with a @code{RevocationResponseMessage} which can be used to indicate +that the @code{RevokeMessage} was invalid (i.e. proof of work incorrect), +or otherwise indicates that the revocation has been processed +successfully. + +@node The REVOCATION Peer-to-Peer Protocol +@subsection The REVOCATION Peer-to-Peer Protocol + +@c %**end of header + +Revocation uses two disjoint ways to spread revocation information among +peers. +First of all, P2P gossip exchanged via CORE-level neighbours is used to +quickly spread revocations to all connected peers. +Second, whenever two peers (that both support revocations) connect, +the SET service is used to compute the union of the respective revocation +sets. + +In both cases, the exchanged messages are @code{RevokeMessage}s which +contain the public key that is being revoked, a matching ECDSA signature, +and a proof-of-work. +Whenever a peer learns about a new revocation this way, it first +validates the signature and the proof-of-work, then stores it to disk +(typically to a file $GNUNET_DATA_HOME/revocation.dat) and finally +spreads the information to all directly connected neighbours. + +For computing the union using the SET service, the peer with the smaller +hashed peer identity will connect (as a "client" in the two-party set +protocol) to the other peer after one second (to reduce traffic spikes +on connect) and initiate the computation of the set union. +All revocation services use a common hash to identify the SET operation +over revocation sets. + +The current implementation accepts revocation set union operations from +all peers at any time; however, well-behaved peers should only initiate +this operation once after establishing a connection to a peer with a +larger hashed peer identity. + +@cindex FS +@cindex FS Subsystem +@node File-sharing (FS) Subsystem +@section File-sharing (FS) Subsystem + +@c %**end of header + +This chapter describes the details of how the file-sharing service works. +As with all services, it is split into an API (libgnunetfs), the service +process (gnunet-service-fs) and user interface(s). +The file-sharing service uses the datastore service to store blocks and +the DHT (and indirectly datacache) for lookups for non-anonymous +file-sharing. +Furthermore, the file-sharing service uses the block library (and the +block fs plugin) for validation of DHT operations. + +In contrast to many other services, libgnunetfs is rather complex since +the client library includes a large number of high-level abstractions; +this is necessary since the Fs service itself largely only operates on +the block level. +The FS library is responsible for providing a file-based abstraction to +applications, including directories, meta data, keyword search, +verification, and so on. + +The method used by GNUnet to break large files into blocks and to use +keyword search is called the +"Encoding for Censorship Resistant Sharing" (ECRS). +ECRS is largely implemented in the fs library; block validation is also +reflected in the block FS plugin and the FS service. +ECRS on-demand encoding is implemented in the FS service. + +NOTE: The documentation in this chapter is quite incomplete. + +@menu +* Encoding for Censorship-Resistant Sharing (ECRS):: +* File-sharing persistence directory structure:: +@end menu + +@cindex ECRS +@cindex Encoding for Censorship-Resistant Sharing +@node Encoding for Censorship-Resistant Sharing (ECRS) +@subsection Encoding for Censorship-Resistant Sharing (ECRS) + +@c %**end of header + +When GNUnet shares files, it uses a content encoding that is called ECRS, +the Encoding for Censorship-Resistant Sharing. +Most of ECRS is described in the (so far unpublished) research paper +attached to this page. ECRS obsoletes the previous ESED and ESED II +encodings which were used in GNUnet before version 0.7.0. +The rest of this page assumes that the reader is familiar with the +attached paper. What follows is a description of some minor extensions +that GNUnet makes over what is described in the paper. +The reason why these extensions are not in the paper is that we felt +that they were obvious or trivial extensions to the original scheme and +thus did not warrant space in the research report. + +@menu +* Namespace Advertisements:: +* KSBlocks:: +@end menu + +@node Namespace Advertisements +@subsubsection Namespace Advertisements + +@c %**end of header +@c %**FIXME: all zeroses -> ? + +An @code{SBlock} with identifier all zeros is a signed +advertisement for a namespace. This special @code{SBlock} contains +metadata describing the content of the namespace. +Instead of the name of the identifier for a potential update, it contains +the identifier for the root of the namespace. +The URI should always be empty. The @code{SBlock} is signed with the +content provder's RSA private key (just like any other SBlock). Peers +can search for @code{SBlock}s in order to find out more about a namespace. + +@node KSBlocks +@subsubsection KSBlocks + +@c %**end of header + +GNUnet implements @code{KSBlocks} which are @code{KBlocks} that, instead +of encrypting a CHK and metadata, encrypt an @code{SBlock} instead. +In other words, @code{KSBlocks} enable GNUnet to find @code{SBlocks} +using the global keyword search. +Usually the encrypted @code{SBlock} is a namespace advertisement. +The rationale behind @code{KSBlock}s and @code{SBlock}s is to enable +peers to discover namespaces via keyword searches, and, to associate +useful information with namespaces. When GNUnet finds @code{KSBlocks} +during a normal keyword search, it adds the information to an internal +list of discovered namespaces. Users looking for interesting namespaces +can then inspect this list, reducing the need for out-of-band discovery +of namespaces. +Naturally, namespaces (or more specifically, namespace advertisements) can +also be referenced from directories, but @code{KSBlock}s should make it +easier to advertise namespaces for the owner of the pseudonym since they +eliminate the need to first create a directory. + +Collections are also advertised using @code{KSBlock}s. + +@table @asis +@item Attachment Size +@item ecrs.pdf 270.68 KB +@item https://gnunet.org/sites/default/files/ecrs.pdf +@end table + +@node File-sharing persistence directory structure +@subsection File-sharing persistence directory structure + +@c %**end of header + +This section documents how the file-sharing library implements +persistence of file-sharing operations and specifically the resulting +directory structure. +This code is only active if the @code{GNUNET_FS_FLAGS_PERSISTENCE} flag +was set when calling @code{GNUNET_FS_start}. +In this case, the file-sharing library will try hard to ensure that all +major operations (searching, downloading, publishing, unindexing) are +persistent, that is, can live longer than the process itself. +More specifically, an operation is supposed to live until it is +explicitly stopped. + +If @code{GNUNET_FS_stop} is called before an operation has been stopped, a +@code{SUSPEND} event is generated and then when the process calls +@code{GNUNET_FS_start} next time, a @code{RESUME} event is generated. +Additionally, even if an application crashes (segfault, SIGKILL, system +crash) and hence @code{GNUNET_FS_stop} is never called and no +@code{SUSPEND} events are generated, operations are still resumed (with +@code{RESUME} events). +This is implemented by constantly writing the current state of the +file-sharing operations to disk. +Specifically, the current state is always written to disk whenever +anything significant changes (the exception are block-wise progress in +publishing and unindexing, since those operations would be slowed down +significantly and can be resumed cheaply even without detailed +accounting). +Note that if the process crashes (or is killed) during a serialization +operation, FS does not guarantee that this specific operation is +recoverable (no strict transactional semantics, again for performance +reasons). However, all other unrelated operations should resume nicely. + +Since we need to serialize the state continuously and want to recover as +much as possible even after crashing during a serialization operation, +we do not use one large file for serialization. +Instead, several directories are used for the various operations. +When @code{GNUNET_FS_start} executes, the master directories are scanned +for files describing operations to resume. +Sometimes, these operations can refer to related operations in child +directories which may also be resumed at this point. +Note that corrupted files are cleaned up automatically. +However, dangling files in child directories (those that are not +referenced by files from the master directories) are not automatically +removed. + +Persistence data is kept in a directory that begins with the "STATE_DIR" +prefix from the configuration file +(by default, "$SERVICEHOME/persistence/") followed by the name of the +client as given to @code{GNUNET_FS_start} (for example, "gnunet-gtk") +followed by the actual name of the master or child directory. + +The names for the master directories follow the names of the operations: + +@itemize @bullet +@item "search" +@item "download" +@item "publish" +@item "unindex" +@end itemize + +Each of the master directories contains names (chosen at random) for each +active top-level (master) operation. +Note that a download that is associated with a search result is not a +top-level operation. + +In contrast to the master directories, the child directories are only +consulted when another operation refers to them. +For each search, a subdirectory (named after the master search +synchronization file) contains the search results. +Search results can have an associated download, which is then stored in +the general "download-child" directory. +Downloads can be recursive, in which case children are stored in +subdirectories mirroring the structure of the recursive download +(either starting in the master "download" directory or in the +"download-child" directory depending on how the download was initiated). +For publishing operations, the "publish-file" directory contains +information about the individual files and directories that are part of +the publication. +However, this directory structure is flat and does not mirror the +structure of the publishing operation. +Note that unindex operations cannot have associated child operations. + +@cindex REGEX subsystem +@node REGEX Subsystem +@section REGEX Subsystem + +@c %**end of header + +Using the REGEX subsystem, you can discover peers that offer a particular +service using regular expressions. +The peers that offer a service specify it using a regular expressions. +Peers that want to patronize a service search using a string. +The REGEX subsystem will then use the DHT to return a set of matching +offerers to the patrons. + +For the technical details, we have Max's defense talk and Max's Master's +thesis. + +@c An additional publication is under preparation and available to +@c team members (in Git). +@c FIXME: Where is the file? Point to it. Assuming that it's szengel2012ms + +@menu +* How to run the regex profiler:: +@end menu + +@node How to run the regex profiler +@subsection How to run the regex profiler + +@c %**end of header + +The gnunet-regex-profiler can be used to profile the usage of mesh/regex +for a given set of regular expressions and strings. +Mesh/regex allows you to announce your peer ID under a certain regex and +search for peers matching a particular regex using a string. +See @uref{https://gnunet.org/szengel2012ms, szengel2012ms} for a full +introduction. + +First of all, the regex profiler uses GNUnet testbed, thus all the +implications for testbed also apply to the regex profiler +(for example you need password-less ssh login to the machines listed in +your hosts file). + +@strong{Configuration} + +Moreover, an appropriate configuration file is needed. +Generally you can refer to the +@file{contrib/regex_profiler_infiniband.conf} file in the sourcecode +of GNUnet for an example configuration. +In the following paragraph the important details are highlighted. + +Announcing of the regular expressions is done by the +gnunet-daemon-regexprofiler, therefore you have to make sure it is +started, by adding it to the AUTOSTART set of ARM: + +@example +[regexprofiler] +AUTOSTART = YES +@end example + +@noindent +Furthermore you have to specify the location of the binary: + +@example +[regexprofiler] +# Location of the gnunet-daemon-regexprofiler binary. +BINARY = /home/szengel/gnunet/src/mesh/.libs/gnunet-daemon-regexprofiler +# Regex prefix that will be applied to all regular expressions and +# search string. +REGEX_PREFIX = "GNVPN-0001-PAD" +@end example + +@noindent +When running the profiler with a large scale deployment, you probably +want to reduce the workload of each peer. +Use the following options to do this. + +@example +[dht] +# Force network size estimation +FORCE_NSE = 1 + +[dhtcache] +DATABASE = heap +# Disable RC-file for Bloom filter? (for benchmarking with limited IO +# availability) +DISABLE_BF_RC = YES +# Disable Bloom filter entirely +DISABLE_BF = YES + +[nse] +# Minimize proof-of-work CPU consumption by NSE +WORKBITS = 1 +@end example + +@noindent +@strong{Options} + +To finally run the profiler some options and the input data need to be +specified on the command line. + +@example +gnunet-regex-profiler -c config-file -d log-file -n num-links \ +-p path-compression-length -s search-delay -t matching-timeout \ +-a num-search-strings hosts-file policy-dir search-strings-file +@end example + +@noindent +Where... + +@itemize @bullet +@item ... @code{config-file} means the configuration file created earlier. +@item ... @code{log-file} is the file where to write statistics output. +@item ... @code{num-links} indicates the number of random links between +started peers. +@item ... @code{path-compression-length} is the maximum path compression +length in the DFA. +@item ... @code{search-delay} time to wait between peers finished linking +and starting to match strings. +@item ... @code{matching-timeout} timeout after which to cancel the +searching. +@item ... @code{num-search-strings} number of strings in the +search-strings-file. +@item ... the @code{hosts-file} should contain a list of hosts for the +testbed, one per line in the following format: + +@itemize @bullet +@item @code{user@@host_ip:port} +@end itemize +@item ... the @code{policy-dir} is a folder containing text files +containing one or more regular expressions. A peer is started for each +file in that folder and the regular expressions in the corresponding file +are announced by this peer. +@item ... the @code{search-strings-file} is a text file containing search +strings, one in each line. +@end itemize + +@noindent +You can create regular expressions and search strings for every AS in the +Internet using the attached scripts. You need one of the +@uref{http://data.caida.org/datasets/routing/routeviews-prefix2as/, CAIDA routeviews prefix2as} +data files for this. Run + +@example +create_regex.py <filename> <output path> +@end example + +@noindent +to create the regular expressions and + +@example +create_strings.py <input path> <outfile> +@end example + +@noindent +to create a search strings file from the previously created +regular expressions. diff --git a/doc/documentation/chapters/installation.texi b/doc/documentation/chapters/installation.texi new file mode 100644 index 0000000000..7be1e9833c --- /dev/null +++ b/doc/documentation/chapters/installation.texi @@ -0,0 +1,4086 @@ +@node GNUnet Installation Handbook +@chapter GNUnet Installation Handbook + +This handbook describes how to install (build, setup, compile) and +setup (configure, start) GNUnet @value{VERSION}. After following these +instructions you should be able to install and then start user-interfaces +to interact with the network. + +Note: This manual is far from complete, and we welcome informed +contributions, be it in the form of new chapters or insightful comments. + +@menu +* Dependencies:: +* Pre-installation notes:: +* Generic installation instructions:: +* Build instructions for Ubuntu 12.04 using Git:: +* Build instructions for software builds from source:: +* Build Instructions for Microsoft Windows Platforms:: +* Build instructions for Debian 7.5:: +* Installing GNUnet from Git on Ubuntu 14.4:: +* Build instructions for Debian 8:: +* Outdated build instructions for previous revisions:: +@c * Portable GNUnet:: +* The graphical configuration interface:: +* How to start and stop a GNUnet peer:: +@end menu + +@node Dependencies +@section Dependencies +@c %**end of header + +This section lists the various known dependencies for +GNUnet @value{EDITION}. +Suggestions for missing dependencies or wrong version numbers are welcome. + +@menu +* External dependencies:: +* Optional dependencies:: +* Internal dependencies:: +@end menu + +@node External dependencies +@subsection External dependencies +@c %**end of header + +These packages must be installed before a typical GNUnet installation +can be performed: + +@itemize @bullet +@item autoconf +@item automake +@item pkg-config +@item libltdl +@item gstreamer +@item gst-plugins-base +@item perl +@item python (only 2.7 supported)@footnote{tests and gnunet-qr} +@item jansson +@item nss +@item glib +@item gmp +@item bluez +@item miniupnpc +@item gettext +@item which +@item texinfo @geq{} 5.2 +@item GNU libmicrohttpd @geq{} 0.9.30 @footnote{We recommend to build it +with a GnuTLS version that was configured with libunbound} +@item GNU libextractor @geq{} 1.0 +@item GNU libtool @geq{} 2.2 +@item GNU libunistring @geq{} 0.9.1.1 +@item GNU libidn @geq{} 1.0.0 +@item @uref{https://gnupg.org/software/libgcrypt/, GNU libgcrypt} @geq{} +@uref{https://gnupg.org/ftp/gcrypt/libgcrypt/, 1.6.0} +@item @uref{https://gnutls.org/, GnuTLS} @geq{} 3.2.7 +@footnote{We recommend to compile with libunbound for DANE support; +GnuTLS also requires GNU nettle 2.7 (update: GnuTLS 3.2.7 appears NOT +to work against GNU nettle > 2.7, due to some API updatings done by +nettle. Thus it should be compiled against nettle 2.7 +and, in case you get some error on the reference to `rpl_strerror' being +undefined, follow the instructions on +@uref{http://lists.gnupg.org/pipermail/gnutls-devel/2013-November/006588.html, this} +post (and the link inside it)).} +@item @uref{https://gnunet.org/gnurl, gnURL} libgnurl @geq{} 7.34.0 +@footnote{must be compiled after @code{GnuTLS}} +@item libglpk @geq{} 4.45 +@item @uref{http://www.openssl.org/, OpenSSL} @geq{} 1.0 +@item TeX Live @geq{} 2012, optional (for gnunet-bcd) +@item Texinfo @geq{} 5.2 (for documentation) +@item libsqlite @geq{} 3.8.0 @footnote{(note that the code will +compile and often work with lower version numbers, but you may get subtle +bugs with respect to quota management in certain rare cases); +alternatively, MySQL or Postgres can also be installed, but those +databases will require more complex configurations (not +recommended for first-time users)} +@item zlib +@end itemize + +@node Optional dependencies +@subsection Optional dependencies + +These applications must be installed for various experimental or otherwise +optional features such as @command{gnunet-conversation}, +and @command{gnunet-gtk} (most of these features are only build if you +configure GNUnet with @command{--enable-experimental}): + +@itemize @bullet +@item libpulse @geq{} 2.0, +optional (for @command{gnunet-conversation}) +@item libopus @geq{} 1.0.1, +optional (for @command{gnunet-conversation}) +@item libogg @geq{} 1.3.0, +optional (for @command{gnunet-conversation}) +@item libnss contained @command{certool} binary, +optional for convenient installation of +the GNS proxy. +@item python-zbar @geq{} 0.10, +optional (for @command{gnunet-qr}) +@item Gtk+ @geq{} 3.0, +optional (for @command{gnunet-gtk}) +@item libgladeui (must match Gtk+ version), +optional (for @command{gnunet-gtk}) +@item libqrencode @geq{} 3.0, +optional (for @command{gnunet-namestore-gtk}) +@end itemize + +@node Internal dependencies +@subsection Internal dependencies + +This section tries to give an overview of what processes a typical GNUnet +peer running a particular application would consist of. All of the +processes listed here should be automatically started by +@command{gnunet-arm -s}. +The list is given as a rough first guide to users for failure diagnostics. +Ideally, end-users should never have to worry about these internal +dependencies. + +In terms of internal dependencies, a minimum file-sharing system consists +of the following GNUnet processes (in order of dependency): + +@itemize @bullet +@item gnunet-service-arm +@item gnunet-service-resolver (required by all) +@item gnunet-service-statistics (required by all) +@item gnunet-service-peerinfo +@item gnunet-service-transport (requires peerinfo) +@item gnunet-service-core (requires transport) +@item gnunet-daemon-hostlist (requires core) +@item gnunet-daemon-topology (requires hostlist, peerinfo) +@item gnunet-service-datastore +@item gnunet-service-dht (requires core) +@item gnunet-service-identity +@item gnunet-service-fs (requires identity, mesh, dht, datastore, core) +@end itemize + +@noindent +A minimum VPN system consists of the following GNUnet processes (in +order of dependency): + +@itemize @bullet +@item gnunet-service-arm +@item gnunet-service-resolver (required by all) +@item gnunet-service-statistics (required by all) +@item gnunet-service-peerinfo +@item gnunet-service-transport (requires peerinfo) +@item gnunet-service-core (requires transport) +@item gnunet-daemon-hostlist (requires core) +@item gnunet-service-dht (requires core) +@item gnunet-service-mesh (requires dht, core) +@item gnunet-service-dns (requires dht) +@item gnunet-service-regex (requires dht) +@item gnunet-service-vpn (requires regex, dns, mesh, dht) +@end itemize + +@noindent +A minimum GNS system consists of the following GNUnet processes (in +order of dependency): + +@itemize @bullet +@item gnunet-service-arm +@item gnunet-service-resolver (required by all) +@item gnunet-service-statistics (required by all) +@item gnunet-service-peerinfo +@item gnunet-service-transport (requires peerinfo) +@item gnunet-service-core (requires transport) +@item gnunet-daemon-hostlist (requires core) +@item gnunet-service-dht (requires core) +@item gnunet-service-mesh (requires dht, core) +@item gnunet-service-dns (requires dht) +@item gnunet-service-regex (requires dht) +@item gnunet-service-vpn (requires regex, dns, mesh, dht) +@item gnunet-service-identity +@item gnunet-service-namestore (requires identity) +@item gnunet-service-gns (requires vpn, dns, dht, namestore, identity) +@end itemize + +@node Pre-installation notes +@section Pre-installation notes + +Please note that in the code instructions for the installation, +@emph{#} indicates commands run as privileged root user and +@emph{$} shows commands run as unprivileged ("normal") system user. + + +@node Generic installation instructions +@section Generic installation instructions + +First, in addition to the GNUnet sources you might require downloading the +latest version of various dependencies, depending on how recent the +software versions in your distribution of GNU/Linux are. +Most distributions do not include sufficiently recent versions of these +dependencies. +Thus, a typically installation on a "modern" GNU/Linux distribution +requires you to install the following dependencies (ideally in this +order): + +@itemize @bullet +@item libgpgerror and libgcrypt +@item libnettle and libunbound (possibly from distribution), GnuTLS +@item libgnurl (read the README) +@item GNU libmicrohttpd +@item GNU libextractor +@end itemize + +Make sure to first install the various mandatory and optional +dependencies including development headers from your distribution. + +Other dependencies that you should strongly consider to install is a +database (MySQL, sqlite or Postgres). +The following instructions will assume that you installed at least sqlite. +For most distributions you should be able to find pre-build packages for +the database. Again, make sure to install the client libraries @b{and} the +respective development headers (if they are packaged separately) as well. + +You can find specific, detailed instructions for installing of the +dependencies (and possibly the rest of the GNUnet installation) in the +platform-specific descriptions, which can be found in the Index. +Please consult them now. +If your distribution is not listed, please study +@ref{Build instructions for Debian 8}, the build instructions for +Debian stable, carefully as you try to install the dependencies for your +own distribution. +Contributing additional instructions for further platforms is always +appreciated. +Please take in mind that operating system development tends to move at +a rather fast speed. Due to this you should be aware that some of +the instructions could be outdated by the time you are reading this. +If you find a mistake, please tell us about it (or even better: send +a patch to the documentation to fix it!). + +Before proceeding further, please double-check the dependency list. +Note that in addition to satisfying the dependencies, you might have to +make sure that development headers for the various libraries are also +installed. +There maybe files for other distributions, or you might be able to find +equivalent packages for your distribution. + +While it is possible to build and install GNUnet without having root +access, we will assume that you have full control over your system in +these instructions. +First, you should create a system user @emph{gnunet} and an additional +group @emph{gnunetdns}. On the GNU/Linux distributions Debian and Ubuntu, +type: + +@example +# adduser --system --home /var/lib/gnunet --group \ +--disabled-password gnunet +# addgroup --system gnunetdns +@end example + +@noindent +On other Unixes and GNU systems, this should have the same effect: + +@example +# useradd --system --groups gnunet --home-dir /var/lib/gnunet +# addgroup --system gnunetdns +@end example + +Now compile and install GNUnet using: + +@example +$ tar xvf gnunet-@value{VERSION}.tar.gz +$ cd gnunet-@value{VERSION} +$ ./configure --with-sudo=sudo --with-nssdir=/lib +$ make +$ sudo make install +@end example + +If you want to be able to enable DEBUG-level log messages, add +@code{--enable-logging=verbose} to the end of the +@command{./configure} command. +@code{DEBUG}-level log messages are in English only and +should only be useful for developers (or for filing +really detailed bug reports). + +Finally, you probably want to compile @command{gnunet-gtk}, which +includes @command{gnunet-setup} (a graphical tool for +GNUnet configuration) and @command{gnunet-fs-gtk} (a graphical tool for +GNUnet file-sharing): + +@example +$ tar xvf gnunet-gtk-@value{VERSION}.tar.gz +$ cd gnunet-gtk-@value{VERSION} +$ ./configure --with-gnunet=/usr/local/ +$ make +$ sudo make install +$ cd .. +# just to be safe run this: +$ sudo ldconfig +@end example + +@noindent +Next, edit the file @file{/etc/gnunet.conf} to contain the following: + +@example +[arm] +SYSTEM_ONLY = YES +USER_ONLY = NO +@end example + +@noindent +You may need to update your @code{ld.so} cache to include +files installed in @file{/usr/local/lib}: + +@example +# ldconfig +@end example + +@noindent +Then, switch from user @code{root} to user @code{gnunet} to start +the peer: + +@example +# su -s /bin/sh - gnunet +$ gnunet-arm -c /etc/gnunet.conf -s +@end example + +You may also want to add the last line in the gnunet user's @file{crontab} +prefixed with @code{@@reboot} so that it is executed whenever the system +is booted: + +@example +@@reboot /usr/local/bin/gnunet-arm -c /etc/gnunet.conf -s +@end example + +@noindent +This will only start the system-wide GNUnet services. +Type exit to get back your root shell. +Now, you need to configure the per-user part. For each +$USER that should get access to GNUnet on the system, run: + +@example +# adduser $USER gnunet +@end example + +@noindent +to allow them to access the system-wide GNUnet services. Then, each +user should create a configuration file @file{~/.config/gnunet.conf} +with the lines: + +@example +[arm] +SYSTEM_ONLY = NO +USER_ONLY = YES +DEFAULTSERVICES = gns +@end example + +@noindent +and start the per-user services using + +@example +$ gnunet-arm -c ~/.config/gnunet.conf -s +@end example + +@noindent +Again, adding a @code{crontab} entry to autostart the peer is advised: + +@example +@@reboot /usr/local/bin/gnunet-arm -c $HOME/.config/gnunet.conf -s +@end example + +@noindent +Note that some GNUnet services (such as SOCKS5 proxies) may need a +system-wide TCP port for each user. +For those services, systems with more than one user may require each user +to specify a different port number in their personal configuration file. + +Finally, the user should perform the basic initial setup for the GNU Name +System (GNS). This is done by running two commands: + +@example +$ gnunet-gns-import.sh +$ gnunet-gns-proxy-setup-ca +@end example + +@noindent +The first generates the default zones, wheras the second setups the GNS +Certificate Authority with the user's browser. Now, to activate GNS in the +normal DNS resolution process, you need to edit your +@file{/etc/nsswitch.conf} where you should find a line like this: + +@example +hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4 +@end example + +@noindent +The exact details may differ a bit, which is fine. Add the text +@emph{"gns [NOTFOUND=return]"} after @emph{"files"}. +Keep in mind that we included a backslash ("\") here just for +markup reasons. You should write the text below on @b{one line} +and @b{without} the "\": + +@example +hosts: files gns [NOTFOUND=return] mdns4_minimal \ +[NOTFOUND=return] dns mdns4 +@end example + +@c FIXME: Document new behavior. +You might want to make sure that @file{/lib/libnss_gns.so.2} exists on +your system, it should have been created during the installation. + +@node Build instructions for Ubuntu 12.04 using Git +@section Build instructions for Ubuntu 12.04 using Git + +@menu +* Install the required build tools:: +* Install libgcrypt 1.6 and libgpg-error:: +* Install gnutls with DANE support:: +* Install libgnurl:: +* Install libmicrohttpd from Git:: +* Install libextractor from Git:: +* Install GNUnet dependencies:: +* Build GNUnet:: +* Install the GNUnet-gtk user interface from Git:: +@end menu + +@node Install the required build tools +@subsection Install the required build tools + +First, make sure Git is installed on your system: + +@example +$ sudo apt-get install git +@end example + +Install the essential buildtools: + +@example +$ sudo apt-get install automake autopoint autoconf libtool +@end example + +@node Install libgcrypt 1.6 and libgpg-error +@subsection Install libgcrypt 1.6 and libgpg-error + +@ref{generic source installation - libgpg-error} + +@node Install gnutls with DANE support +@subsection Install gnutls with DANE support + +@itemize @bullet +@item @ref{generic source installation - nettle} +@item @ref{generic source installation - ldns} +@item @ref{generic source installation - libunbound/unbound} +@item @ref{generic source installation - gnutls} +@item @ref{generic source installation - libgcrypt} +@end itemize + +@node Install libgnurl +@subsection Install libgnurl + +Follow the @ref{generic source installation - libgnurl}. + +@node Install libmicrohttpd from Git +@subsection Install libmicrohttpd from Git + +@example +$ git clone https://gnunet.org/git/libmicrohttpd +$ cd libmicrohttpd/ +$ ./bootstrap +$ ./configure +$ sudo make install ; cd .. +@end example + +@node Install libextractor from Git +@subsection Install libextractor from Git + +Install libextractor dependencies: + +@example +$ sudo apt-get install zlib1g-dev libgsf-1-dev libmpeg2-4-dev \ + libpoppler-dev libvorbis-dev libexiv2-dev libjpeg-dev \ + libtiff-dev libgif-dev libvorbis-dev libflac-dev libsmf-dev \ + g++ +@end example + +Build libextractor: + +@example +$ git clone https://gnunet.org/git/libextractor +$ cd libextractor +$ ./bootstrap +$ ./configure +$ sudo make install ; cd .. +@end example + +@node Install GNUnet dependencies +@subsection Install GNUnet dependencies + +@example +$ sudo apt-get install libidn11-dev libunistring-dev libglpk-dev \ + libpulse-dev libbluetooth-dev libsqlite-dev +@end example + +Install libopus: + +@example +$ wget http://downloads.xiph.org/releases/opus/opus-1.1.tar.gz +$ tar xf opus-1.1.tar.gz +$ cd opus-1.1/ +$ ./configure +$ sudo make install ; cd .. +@end example + +Choose one or more database backends: + +SQLite3: +@example +$ sudo apt-get install libsqlite3-dev +@end example +MySQL: +@example +$ sudo apt-get install libmysqlclient-dev +@end example +PostgreSQL: +@example +$ sudo apt-get install libpq-dev postgresql +@end example + + + +@node Build GNUnet +@subsection Build GNUnet + + + +@menu +* Configuring the installation path:: +* Configuring the system:: +* Installing components requiring sudo permission:: +* Build:: +@end menu + +@node Configuring the installation path +@subsubsection Configuring the installation path + +You can specify the location of the GNUnet installation by setting the +prefix when calling the configure script with @code{--prefix=DIRECTORY} + +@example +$ export PATH=$PATH:DIRECTORY/bin +@end example + +@node Configuring the system +@subsubsection Configuring the system + +Please make sure NOW that you have created a user and group 'gnunet' +and additionally a group 'gnunetdns': + +@example +$ sudo addgroup gnunet +$ sudo addgroup gnunetdns +$ sudo adduser gnunet +@end example + +Each GNUnet user should be added to the 'gnunet' group (may +require fresh login to come into effect): + +@example +$ sudo useradd -G gnunet +@end example + +@node Installing components requiring sudo permission +@subsubsection Installing components requiring sudo permission + +Some components, like the nss plugin required for GNS, may require root +permissions. To allow these few components to be installed use: + +@example +$ ./configure --with-sudo +@end example + +@node Build +@subsubsection Build + +@example +$ git clone https://gnunet.org/git/gnunet/ +$ cd gnunet/ +$ ./bootstrap +@end example + +Use the required configure call including the optional installation prefix +@code{PREFIX} or the sudo permissions: + +@example +$ ./configure [ --with-sudo | --with-prefix=PREFIX ] +@end example + +@example +$ make; sudo make install +@end example + +After installing it, you need to create an empty configuration file: + +@example +mkdir ~/.gnunet; touch ~/.gnunet/gnunet.conf +@end example + +And finally you can start GNUnet with: + +@example +$ gnunet-arm -s +@end example + +@node Install the GNUnet-gtk user interface from Git +@subsection Install the GNUnet-gtk user interface from Git + + +Install depencies: + +@example +$ sudo apt-get install libgtk-3-dev libunique-3.0-dev libgladeui-dev \ + libqrencode-dev +@end example + +Build GNUnet (with an optional prefix) and execute: + +@example +$ git clone https://gnunet.org/git/gnunet-gtk/ +$ cd gnunet-gtk/ +$ ./bootstrap +$ ./configure [--prefix=PREFIX] --with-gnunet=DIRECTORY +$ make; sudo make install +@end example + +@node Build instructions for software builds from source +@section Build instructions for software builds from source + +This section describes software builds in case your operating +system lacks binary substitutes / binary builds for some dependencies +of GNUnet. +It is assumed that you have installed common build dependencies +and that these instructions are treated as generic without any +debugging help. +It is furthermore assumed that you use the release tarballs of +the software, installation from the respective version control +sources might differ in ways that are only minimal different +(for example a dependency on autotools etc). + +@menu +* generic source installation - nettle:: +* generic source installation - ldns:: +* generic source installation - libunbound/unbound:: +* generic source installation - libav:: +* generic source installation - libextractor:: +* generic source installation - libgpg-error:: +* generic source installation - libgcrypt:: +* generic source installation - gnutls:: +* generic source installation - libmicrohttpd:: +* generic source installation - libgnurl:: +@end menu + +@node generic source installation - nettle +@subsection generic source installation - nettle +@example +$ wget http://www.lysator.liu.se/~nisse/archive/nettle-2.7.1.tar.gz +$ tar xf nettle-2.7.1.tar.gz +$ cd nettle-2.7.1 +$ ./configure +$ sudo make install ; cd .. +@end example + +@node generic source installation - ldns +@subsection generic source installation - ldns +@example +$ wget https://www.nlnetlabs.nl/downloads/ldns/ldns-1.6.16.tar.gz +$ tar xf ldns-1.6.16.tar.gz +$ cd ldns-1.6.16 +$ ./configure +$ sudo make install ; cd .. +@end example + +@node generic source installation - libunbound/unbound +@subsection generic source installation - libunbound/unbound +@example +$ wget https://unbound.net/downloads/unbound-1.4.21.tar.gz +$ tar xf unbound-1.4.21.tar.gz +$ cd unbound-1.4.21 +$ ./configure +$ sudo make install ; cd .. +@end example + +@node generic source installation - libav +@subsection generic source installation - libav +@example +$ wget https://libav.org/releases/libav-9.10.tar.xz +$ cd libav-0.9 ; ./configure --enable-shared; +$ make; sudo make install; cd .. +@end example + +@node generic source installation - libextractor +@subsection generic source installation - libextractor +@example +$ wget https://ftp.gnu.org/gnu/libextractor/libextractor-1.3.tar.gz +$ tar xvf libextractor-1.3.tar.gz +$ cd libextractor-1.3 ; ./configure; +$ make ; sudo make install; cd .. +@end example + +@node generic source installation - libgpg-error +@subsection generic source installation - libgpg-error +@example +$ wget https://ftp.gnupg.org/gcrypt/libgpg-error/libgpg-error-1.12.tar.bz2 +$ tar xvf libgpg-error-1.12.tar.bz2 +$ cd libgpg-error-1.12; ./configure; +$ make ; sudo make install; cd .. +@end example + +@node generic source installation - libgcrypt +@subsection generic source installation - libgcrypt +@example +$ wget https://ftp.gnupg.org/gcrypt/libgcrypt/libgcrypt-1.6.0.tar.bz2 +$ tar xvf libgcrypt-1.6.0.tar.bz2 +$ cd libgcrypt-1.6.0; ./configure --with-gpg-error-prefix=/usr/local; +$ make ; sudo make install ; cd .. +@end example + +@node generic source installation - gnutls +@subsection generic source installation - gnutls +@example +$ wget ftp://ftp.gnutls.org/gcrypt/gnutls/v3.2/gnutls-3.2.7.tar.xz +$ tar xvf gnutls-3.2.7.tar.xz +$ cd gnutls-3.2.7 ; ./configure; +$ make ; sudo make install ; cd .. +@end example + +@noindent +If you want a GnuTLS with DANE functionality, you have to compile +it against libunbound. + +@node generic source installation - libmicrohttpd +@subsection generic source installation - libmicrohttpd +@example +$ wget https://ftp.gnu.org/gnu/libmicrohttpd/libmicrohttpd-0.9.33.tar.gz +$ tar xvf libmicrohttpd-0.9.33.tar.gz +$ cd libmicrohttpd-0.9.33; ./configure; +$ make ; sudo make install ; cd .. +@end example + +@node generic source installation - libgnurl +@subsection generic source installation - libgnurl + +@example +$ wget https://gnunet.org/sites/default/files/gnurl-7.34.0.tar.bz2 +$ tar xvf gnurl-7.34.0.tar.bz2 +$ cd gnurl-7.34.0 +$ ./configure --enable-ipv6 --with-gnutls=/usr/local --without-libssh2 \ + --without-libmetalink --without-winidn --without-librtmp \ + --without-nghttp2 --without-nss --without-cyassl --without-polarssl \ + --without-ssl --without-winssl --without-darwinssl --disable-sspi \ + --disable-ntlm-wb --disable-ldap --disable-rtsp --disable-dict \ + --disable-telnet --disable-tftp --disable-pop3 --disable-imap \ + --disable-smtp --disable-gopher --disable-file --disable-ftp +$ make ; sudo make install; cd .. +@end example + +@menu +* Fixing libgnurl build issues:: +@end menu + +@node Fixing libgnurl build issues +@subsubsection Fixing libgnurl build issues + +@c FIXME: Obviously this subsection should be evaluated and +@c if still necessary moved into gnURL itself (README) or +@c into a separate section which deals with gnURL. +If you have to compile libgnurl from source (for example if the version +included in your distribution is too old or it's not included at all) +you perhaps might get an error message while running the +@command{configure} script: + +@example +$ configure +... +checking for 64-bit curl_off_t data type... unknown +checking for 32-bit curl_off_t data type... unknown +checking for 16-bit curl_off_t data type... unknown +configure: error: cannot find data type for curl_off_t. +@end example + +@noindent +Solution: + +Before running the @command{configure} script, set: + +@example +CFLAGS="-I. -I$BUILD_ROOT/include" +@end example + +@node Build Instructions for Microsoft Windows Platforms +@section Build Instructions for Microsoft Windows Platforms + +@menu +* Introduction to building on MS Windows:: +* Requirements:: +* Dependencies & Initial Setup:: +* GNUnet Installation:: +* Adjusting Windows for running and testing GNUnet:: +* Building the GNUnet Installer:: +* Using GNUnet with Netbeans on Windows:: +@end menu + +@node Introduction to building on MS Windows +@subsection Introduction to building on MS Windows + + +This document is a guide to building GNUnet and its dependencies on +Windows platforms. GNUnet development is mostly done under GNU/Linux and +especially git checkouts may not build out of the box. +We regret any inconvenience, and if you have problems, please report +them. + +@node Requirements +@subsection Requirements + +The Howto is based upon a @strong{Windows Server 2008 32bit} +@strong{Installation}, @strong{sbuild} and thus a +@uref{http://www.mingw.org/wiki/MSYS, MSYS+MinGW} +(W32-GCC-Compiler-Suite + Unix-like Userland) installation. sbuild +is a convenient set of scripts which creates a working msys/mingw +installation and installs most dependencies required for GNUnet. + +As of the point of the creation of these instructions, +GNUnet @strong{requires} a Windows @strong{Server} 2003 or +newer for full feature support. +Windows Vista and later will also work, but +@strong{non-server version can not run a VPN-Exit-Node} as the NAT +features have been removed as of Windows Vista. + +@c TODO: We should document Windows 10! +@c It seems like the situation hasn't changed with W10 + +@node Dependencies & Initial Setup +@subsection Dependencies & Initial Setup + + +@itemize @bullet + +@item +Install a fresh version of @strong{Python 2.x}, even if you are using a +x64-OS, install a 32-bit version for use with sbuild. +Python 3.0 is currently incompatible. + +@item +Install your favorite @uref{http://code.google.com/p/tortoisegit/, git} & +@uref{http://tortoisesvn.net/, subversion}-clients. + +@item +You will also need some archive-manager like +@uref{http://www.7-zip.org/, 7zip}. + +@item +Pull a copy of sbuild to a directory of your choice, which will be used +in the remainder of this guide. For now, we will use +@file{c:\gnunet\sbuild\} + +@item +in @file{sbuild\src\mingw\mingw32-buildall.sh}, comment out the packages +@strong{gnunet-svn} and @strong{gnunet-gtk-svn}, as we don't want sbuild +to compile/install those for us. + +@item +Follow LRN's sbuild installation instructions.- +@end itemize + +Please note that sbuild may (or will most likely) fail during +installation, thus you really HAVE to @strong{check the logfiles} created +during the installation process. +Certain packages may fail to build initially due to missing dependencies, +thus you may have to +@strong{substitute those with binary-versions initially}. Later on once +dependencies are satisfied you can re-build the newer package versions. + +@strong{It is normal that you may have to repeat this step multiple times +and there is no uniform way to fix all compile-time issues, as the +build-process of many of the dependencies installed are rather unstable +on win32 and certain releases may not even compile at all.} + +Most dependencies for GNUnet have been set up by sbuild, thus we now +should add the @file{bin/} directories in your new msys and mingw +installations to PATH. You will want to create a backup of your finished +msys-environment by now. + +@node GNUnet Installation +@subsection GNUnet Installation + +First, we need to launch our msys-shell, you can do this via + +@file{C:\gnunet\sbuild\msys\msys.bat} + +You might wish to take a look at this file and adjust some +login-parameters to your msys environment. + +Also, sbuild added two pointpoints to your msys-environment, though those +might remain invisible: + +@itemize @bullet + +@item +/mingw, which will mount your mingw-directory from sbuild/mingw and the +other one is + +@item +/src which contains all the installation sources sbuild just compiled. +@end itemize + +Check out the current GNUnet sources (git HEAD) from the +GNUnet repository "gnunet.git", we will do this in your home directory: + +@code{git clone https://gnunet.org/git/gnunet/ ~/gnunet} + +Now, we will first need to bootstrap the checked out installation and then +configure it accordingly. + +@example +cd ~/gnunet +./bootstrap +STRIP=true CPPFLAGS="-DUSE_IPV6=1 -DW32_VEH" CFLAGS="$CFLAGS -g -O2" \ +./configure --prefix=/ --docdir=/share/doc/gnunet \ +--with-libiconv-prefix=/mingw --with-libintl-prefix=/mingw \ +--with-libcurl=/mingw --with-extractor=/mingw --with-sqlite=/mingw \ +--with-microhttpd=/mingw --with-plibc=/mingw --enable-benchmarks \ +--enable-expensivetests --enable-experimental --with-qrencode=/mingw \ +--enable-silent-rules --enable-experimental 2>&1 | tee -a ./configure.log +@end example + +The parameters above will configure for a reasonable GNUnet installation +to the your msys-root directory. +Depending on which features your would like to build or you may need to +specify additional dependencies. Sbuild installed most libs into +the /mingw subdirectory, so remember to prefix library locations with +this path. + +Like on a unixoid system, you might want to use your home directory as +prefix for your own GNUnet installation for development, without tainting +the buildenvironment. Just change the "prefix" parameter to point towards +~/ in this case. + +Now it's time to compile GNUnet as usual. Though this will take some time, +so you may fetch yourself a coffee or some Mate now... + +@example +make ; make install +@end example + +@node Adjusting Windows for running and testing GNUnet +@subsection Adjusting Windows for running and testing GNUnet + +Assuming the build succeeded and you +@strong{added the bin directory of your GNUnet to PATH}, you can now use +your gnunet-installation as usual. +Remember that UAC or the windows firewall may popup initially, blocking +further execution of gnunet until you acknowledge them. + +You will also have to take the usual steps to get peer-to-peer (p2p) +software running properly (port forwarding, ...), +and GNUnet will require administrative permissions as it may even +install a device-driver (in case you are using gnunet-vpn and/or +gnunet-exit). + +@node Building the GNUnet Installer +@subsection Building the GNUnet Installer + +The GNUnet installer is made with +@uref{http://nsis.sourceforge.net/, NSIS}. +The installer script is located in @file{contrib\win} in the +GNUnet source tree. + +@node Using GNUnet with Netbeans on Windows +@subsection Using GNUnet with Netbeans on Windows + +TODO + +@node Build instructions for Debian 7.5 +@section Build instructions for Debian 7.5 + + +These are the installation instructions for Debian 7.5. They were tested +using a minimal, fresh Debian 7.5 AMD64 installation without non-free +software (no contrib or non-free). +By "minimal", we mean that during installation, we did not select any +desktop environment, servers or system utilities during the "tasksel" +step. Note that the packages and the dependencies that we will install +during this chapter take about 1.5 GB of disk space. +Combined with GNUnet and space for objects during compilation, you should +not even attempt this unless you have about 2.5 GB free after the minimal +Debian installation. +Using these instructions to build a VM image is likely to require a +minimum of 4-5 GB for the VM (as you will likely also want a desktop +manager). + +GNUnet's security model assumes that your @file{/home} directory is +encrypted. Thus, if possible, you should encrypt your home partition +(or per-user home directory). + +Naturally, the exact details of the starting state for your installation +should not matter much. For example, if you selected any of those +installation groups you might simply already have some of the necessary +packages installed. +We did this for testing, as this way we are less likely to forget to +mention a required package. +Note that we will not install a desktop environment, but of course you +will need to install one to use GNUnet's graphical user interfaces. +Thus, it is suggested that you simply install the desktop environment of +your choice before beginning with the instructions. + + + +@menu +* Update:: +* Stable? Hah!:: +* Update again:: +* Installing packages:: +* Installing dependencies from source:: +* Installing GNUnet from source:: +* But wait there is more!:: +@end menu + +@node Update +@subsection Update + +After any installation, you should begin by running + +@example +# apt-get update ; apt-get upgrade +@end example + +to ensure that all of your packages are up-to-date. Note that the "#" is +used to indicate that you need to type in this command as "root" +(or prefix with "sudo"), whereas "$" is used to indicate typing in a +command as a normal user. + +@node Stable? Hah! +@subsection Stable? Hah! + +Yes, we said we start with a Debian 7.5 "stable" system. However, to +reduce the amount of compilation by hand, we will begin by allowing the +installation of packages from the testing and unstable distributions as +well. +We will stick to "stable" packages where possible, but some packages will +be taken from the other distributions. +Start by modifying @file{/etc/apt/sources.list} to contain the +following (possibly adjusted to point to your mirror of choice): + +@example +# These were there before: +deb http://ftp.de.debian.org/debian/ wheezy main +deb-src http://ftp.de.debian.org/debian/ wheezy main +deb http://security.debian.org/ wheezy/updates main +deb-src http://security.debian.org/ wheezy/updates main +deb http://ftp.de.debian.org/debian/ wheezy-updates main +deb-src http://ftp.de.debian.org/debian/ wheezy-updates main + +# Add these lines (feel free to adjust the mirror): +deb http://ftp.de.debian.org/debian/ testing main +deb http://ftp.de.debian.org/debian/ unstable main +@end example + +The next step is to create/edit your @file{/etc/apt/preferences} +file to look like this: + +@example +Package: * +Pin: release a=stable,n=wheezy +Pin-Priority: 700 + +Package: * +Pin: release o=Debian,a=testing +Pin-Priority: 650 + +Package: * +Pin: release o=Debian,a=unstable +Pin-Priority: 600 +@end example + +You can read more about Apt Preferences here and here. +Note that other pinnings are likely to also work for GNUnet, the key +thing is that you need some packages from unstable (as shown below). +However, as unstable is unlikely to be comprehensive (missing packages) +or might be problematic (crashing packages), you probably want others +from stable and/or testing. + +@node Update again +@subsection Update again + +Now, run again@ + +@example +# apt-get update@ +# apt-get upgrade@ +@end example + +to ensure that all your new distribution indices are downloaded, and +that your pinning is correct: the upgrade step should cause no changes +at all. + +@node Installing packages +@subsection Installing packages + +We begin by installing a few Debian packages from stable:@ + +@example +# apt-get install gcc make python-zbar libltdl-dev libsqlite3-dev \ + libunistring-dev libopus-dev libpulse-dev openssl libglpk-dev \ + texlive libidn11-dev libmysqlclient-dev libpq-dev libarchive-dev \ + libbz2-dev libexiv2-dev libflac-dev libgif-dev libglib2.0-dev \ + libgtk-3-dev libmagic-dev libjpeg8-dev libmpeg2-4-dev libmp4v2-dev \ + librpm-dev libsmf-dev libtidy-dev libtiff5-dev libvorbis-dev \ + libogg-dev zlib1g-dev g++ gettext libgsf-1-dev libunbound-dev \ + libqrencode-dev libgladeui-dev nasm texlive-latex-extra \ + libunique-3.0-dev gawk miniupnpc libfuse-dev libbluetooth-dev +@end example + +After that, we install a few more packages from unstable:@ + +@example +# apt-get install -t unstable nettle-dev libgstreamer1.0-dev \ + gstreamer1.0-plugins-base gstreamer1.0-plugins-good \ + libgstreamer-plugins-base1.0-dev +@end example + +@node Installing dependencies from source +@subsection Installing dependencies from source + +Next, we need to install a few dependencies from source. +You might want to do this as a "normal" user and only run the +@code{make install} steps as root (hence the @code{sudo} in the +commands below). Also, you do this from any +directory. We begin by downloading all dependencies, then extracting the +sources, and finally compiling and installing the libraries. + +For these steps, follow the instructions given in the +installation from source instruction in this order: + +@itemize @bullet +@item @ref{generic source installation - libav} +@item @ref{generic source installation - libextractor} +@item @ref{generic source installation - libgpg-error} +@item @ref{generic source installation - libgcrypt} +@item @ref{generic source installation - gnutls} +@item @ref{generic source installation - libmicrohttpd} +@item @ref{generic source installation - libgnurl} +@end itemize + +@node Installing GNUnet from source +@subsection Installing GNUnet from source + + +For this, simply follow the generic installation instructions from +here. + +@node But wait there is more! +@subsection But wait there is more! + +So far, we installed all of the packages and dependencies required to +ensure that all of GNUnet would be built. +However, while for example the plugins to interact with the MySQL or +Postgres databases have been created, we did not actually install or +configure those databases. Thus, you will need to install +and configure those databases or stick with the default Sqlite database. +Sqlite is usually fine for most applications, but MySQL can offer better +performance and Postgres better resillience. + + +@node Installing GNUnet from Git on Ubuntu 14.4 +@section Installing GNUnet from Git on Ubuntu 14.4 + +@strong{Install the required build tools:} + +@example +$ sudo apt-get install git automake autopoint autoconf +@end example + +@strong{Install the required dependencies} + +@example +$ sudo apt-get install libltdl-dev libgpg-error-dev libidn11-dev \ + libunistring-dev libglpk-dev libbluetooth-dev libextractor-dev \ + libmicrohttpd-dev libgnutls28-dev +@end example + +@strong{Choose one or more database backends} + +@itemize @bullet + +@item SQLite3: + +@example +$ sudo apt-get install libsqlite3-dev +@end example + +@item MySQL: + +@example +$ sudo apt-get install libmysqlclient-dev +@end example + +@item PostgreSQL: + +@example +$ sudo apt-get install libpq-dev postgresql +@end example + +@end itemize + +@strong{Install the optional dependencies for gnunet-conversation:} + +@example +$ sudo apt-get install gstreamer1.0 libpulse-dev libopus-dev +@end example + +@strong{Install the libgrypt 1.6.1:} + +@itemize @bullet + +@item For Ubuntu 14.04: + +@example +$ sudo apt-get install libgcrypt20-dev +@end example + +@item For Ubuntu older 14.04: + +@example +$ wget ftp://ftp.gnupg.org/gcrypt/libgcrypt/libgcrypt-1.6.1.tar.bz2 +$ tar xf libgcrypt-1.6.1.tar.bz2 +$ cd libgcrypt-1.6.1 +$ ./configure +$ sudo make install +$ cd .. +@end example + +@end itemize + +@strong{Install libgnurl} + +@example +$ wget https://gnunet.org/sites/default/files/gnurl-7.35.0.tar.bz2 +$ tar xf gnurl-7.35.0.tar.bz2 +$ cd gnurl-7.35.0 +$ ./configure --enable-ipv6 --with-gnutls --without-libssh2 \ + --without-libmetalink --without-winidn --without-librtmp \ + --without-nghttp2 --without-nss --without-cyassl --without-polarssl \ + --without-ssl --without-winssl --without-darwinssl --disable-sspi \ + --disable-ntlm-wb --disable-ldap --disable-rtsp --disable-dict \ + --disable-telnet --disable-tftp --disable-pop3 --disable-imap \ + --disable-smtp --disable-gopher --disable-file --disable-ftp +$ sudo make install +$ cd .. +@end example + +@strong{Install GNUnet} + +@example +$ git clone https://gnunet.org/git/gnunet/ +$ cd gnunet/ +$ ./bootstrap +@end example + +If you want to: + +@itemize @bullet + +@item Install to a different directory: + +@example +--prefix=PREFIX +@end example + +@item +Have sudo permission, but do not want to compile as root: + +@example +--with-sudo +@end example + +@item +Want debug message enabled: + +@example +--enable-logging=verbose +@end example + +@end itemize + + +@example +$ ./configure [ --with-sudo | --prefix=PREFIX | --enable-logging=verbose] +$ make; sudo make install +@end example + +After installing it, you need to create an empty configuration file: + +@example +touch ~/.config/gnunet.conf +@end example + +And finally you can start GNUnet with + +@example +$ gnunet-arm -s +@end example + +@node Build instructions for Debian 8 +@section Build instructions for Debian 8 +@c FIXME: I -> we + +These are the installation instructions for Debian 8. They were tested +sing a fresh Debian 8 AMD64 installation without non-free software (no +contrib or non-free). During installation, I only selected "lxde" for the +desktop environment. +Note that the packages and the dependencies that we will install during +this chapter take about 1.5 GB of disk space. Combined with GNUnet and +space for objects during compilation, you should not even attempt this +unless you have about 2.5 GB free after the Debian installation. +Using these instructions to build a VM image is likely to require a +minimum of 4-5 GB for the VM (as you will likely also want a desktop +manager). + +GNUnet's security model assumes that your @code{/home} directory is +encrypted. +Thus, if possible, you should encrypt your entire disk, or at least just +your home partition (or per-user home directory). + +Naturally, the exact details of the starting state for your installation +should not matter much. +For example, if you selected any of those installation groups you might +simply already have some of the necessary packages installed. Thus, it is +suggested that you simply install the desktop environment of your choice +before beginning with the instructions. + + +@menu +* Update Debian:: +* Installing Debian Packages:: +* Installing Dependencies from Source2:: +* Installing GNUnet from Source2:: +* But wait (again) there is more!:: +@end menu + +@node Update Debian +@subsection Update Debian + +After any installation, you should begin by running + +@example +# apt-get update +# apt-get upgrade +@end example + +to ensure that all of your packages are up-to-date. Note that the "#" is +used to indicate that you need to type in this command as "root" (or +prefix with "sudo"), whereas "$" is used to indicate typing in a command +as a normal user. + +@node Installing Debian Packages +@subsection Installing Debian Packages + +We begin by installing a few Debian packages from stable: + +@example +# apt-get install gcc make python-zbar libltdl-dev libsqlite3-dev \ +libunistring-dev libopus-dev libpulse-dev openssl libglpk-dev texlive \ +libidn11-dev libmysqlclient-dev libpq-dev libarchive-dev libbz2-dev \ +libflac-dev libgif-dev libglib2.0-dev libgtk-3-dev libmpeg2-4-dev \ +libtidy-dev libvorbis-dev libogg-dev zlib1g-dev g++ gettext \ +libgsf-1-dev libunbound-dev libqrencode-dev libgladeui-dev nasm \ +texlive-latex-extra libunique-3.0-dev gawk miniupnpc libfuse-dev \ +libbluetooth-dev gstreamer1.0-plugins-base gstreamer1.0-plugins-good \ +libgstreamer-plugins-base1.0-dev nettle-dev libextractor-dev \ +libgcrypt20-dev libmicrohttpd-dev +@end example + +@node Installing Dependencies from Source2 +@subsection Installing Dependencies from Source2 + +Yes, we said we start with a Debian 8 "stable" system, but because Debian +linked GnuTLS without support for DANE, we need to compile a few things, +in addition to GNUnet, still by hand. Yes, you can run GNUnet using the +respective Debian packages, but then you will not get DANE support. + +Next, we need to install a few dependencies from source. You might want +to do this as a "normal" user and only run the @code{make install} steps +as root (hence the @code{sudo} in the commands below). Also, you do this +from any directory. We begin by downloading all dependencies, then +extracting the sources, and finally compiling and installing the +libraries: + +@example +$ wget ftp://ftp.gnutls.org/gcrypt/gnutls/v3.3/gnutls-3.3.12.tar.xz +$ tar xvf gnutls-3.3.12.tar.xz +$ cd gnutls-3.3.12 ; ./configure ; make ; sudo make install ; cd .. +@end example + +For the installation and compilation of libgnurl/gnURL refer to +the generic installation section, +@xref{generic source installation - libgnurl}. + +@node Installing GNUnet from Source2 +@subsection Installing GNUnet from Source2 + +For this, simply follow the generic installation instructions from@ +here. + +@node But wait (again) there is more! +@subsection But wait (again) there is more! + +So far, we installed all of the packages and dependencies required to +ensure that all of GNUnet would be built. However, while for example the +plugins to interact with the MySQL or Postgres databases have been +created, we did not actually install or configure those databases. +Thus, you will need to install and configure those databases or stick +with the default Sqlite database. Sqlite is usually fine for most +applications, but MySQL can offer better performance and Postgres better +resillience. + +@node Outdated build instructions for previous revisions +@section Outdated build instructions for previous revisions + +This chapter contains a collection of outdated, older installation guides. +They are mostly intended to serve as a starting point for writing +up-to-date instructions and should not be expected to work for +GNUnet 0.10.x. +A set of older installation instructions can also be found in the +file @file{doc/outdated-and-old-installation-instructions.txt} in the +source tree of GNUnet. + +This file covers old instructions which no longer receive security +updates or any kind of support. + +@menu +* Installing GNUnet 0.10.1 on Ubuntu 14.04:: +* Building GLPK for MinGW:: +* GUI build instructions for Ubuntu 12.04 using Subversion:: +@c * Installation with gnunet-update:: +* Instructions for Microsoft Windows Platforms (Old):: +@end menu + + +@node Installing GNUnet 0.10.1 on Ubuntu 14.04 +@subsection Installing GNUnet 0.10.1 on Ubuntu 14.04 + +Install the required dependencies: + +@example +$ sudo apt-get install libltdl-dev libgpg-error-dev libidn11-dev \ + libunistring-dev libglpk-dev libbluetooth-dev libextractor-dev \ + libmicrohttpd-dev libgnutls28-dev +@end example + +Choose one or more database backends: + +@itemize @bullet + +@item SQLite3 + +@example + $ sudo apt-get install libsqlite3-dev@ +@end example + +@item MySQL + +@example +$ sudo apt-get install libmysqlclient-dev@ +@end example + +@item PostgreSQL + +@example + $ sudo apt-get install libpq-dev postgresql@ +@end example + +@end itemize + +Install the optional dependencies for gnunet-conversation: + +@example + $ sudo apt-get install gstreamer1.0 libpulse-dev libopus-dev +@end example + +Install libgcrypt 1.6: + +@itemize @bullet + +@item For Ubuntu 14.04: + +@example +$ sudo apt-get install libgcrypt20-dev +@end example + +@item For Ubuntu older than 14.04: + +@example +wget ftp://ftp.gnupg.org/gcrypt/libgcrypt/libgcrypt-1.6.1.tar.bz2 +$ tar xf libgcrypt-1.6.1.tar.bz2 +$ cd libgcrypt-1.6.1 +$ ./configure +$ sudo make install +$ cd .. +@end example +@end itemize + +Install libgnurl: + +@pxref{generic source installation - libgnurl}. + +Install GNUnet: + +@example +$ wget http://ftpmirror.gnu.org/gnunet/gnunet-0.10.1.tar.gz +$ tar xf gnunet-0.10.1.tar.gz +$ cd gnunet-0.10.1 +@end example + +If you want to: + +@itemize @bullet + +@item +Install to a different directory: + +@example +--prefix=PREFIX +@end example + +@item +Have sudo permission, but do not want to compile as root: + +@example +--with-sudo +@end example + +@item +Want debug message enabled: + +@example +--enable-logging=verbose +@end example + +@end itemize + +@example +$ ./configure [ --with-sudo | --prefix=PREFIX | --enable-logging=verbose] +$ make; sudo make install +@end example + +After installing it, you need to create an empty configuration file: + +@example +touch ~/.config/gnunet.conf +@end example + +And finally you can start GNUnet with + +@example +$ gnunet-arm -s +@end example + +@node Building GLPK for MinGW +@subsection Building GLPK for MinGW + +GNUnet now requires the GNU Linear Programming Kit (GLPK). +Since there's is no package you can install with @code{mingw-get} you +have to compile it from source: + +@itemize @bullet + +@item Download the latest version from +@uref{http://ftp.gnu.org/gnu/glpk/} + +@item Unzip the downloaded source tarball using your favourite +unzipper application In the MSYS shell + +@item change to the respective directory + +@item Configure glpk for "i686-pc-mingw32": + +@example +./configure '--build=i686-pc-mingw32' +@end example + +@item run + +@example +make install check +@end example + +@end itemize + +MinGW does not automatically detect the correct buildtype so you have to +specify it manually. + + +@node GUI build instructions for Ubuntu 12.04 using Subversion +@subsection GUI build instructions for Ubuntu 12.04 using Subversion + +After installing GNUnet you can continue installing the GNUnet GUI tools: + +First, install the required dependencies: + +@example +$ sudo apt-get install libgladeui-dev libqrencode-dev +@end example + +Please ensure that the GNUnet shared libraries can be found by the linker. +If you installed GNUnet libraries in a non standard path +(say GNUNET_PREFIX=/usr/local/lib/), you can + +@itemize @bullet + +@item set the environmental variable permanently to: + +@example +LD_LIBRARY_PATH=$GNUNET_PREFIX +@end example + +@item or add @code{$GNUNET_PREFIX} to @file{/etc/ld.so.conf} + +@end itemize + +Now you can checkout and compile the GNUnet GUI tools: + +@example +$ git clone https://gnunet.org/git/gnunet-gtk +$ cd gnunet-gtk +$ ./bootstrap +$ ./configure --prefix=$GNUNET_PREFIX/.. --with-gnunet=$GNUNET_PREFIX/.. +$ make install +@end example + +@c @node Installation with gnunet-update +@c @subsection Installation with gnunet-update + +@c gnunet-update project is an effort to introduce updates to GNUnet +@c installations. An interesting to-be-implemented-feature of gnunet-update +@c is that these updates are propagated through GNUnet's peer-to-peer +@c network. More information about gnunet-update can be found at +@c @c FIXME: Use correct cgit URL +@c @uref{https://gnunet.org/git/gnunet-update.git/tree/plain/README}. + +@c While the project is still under development, we have implemented the +@c following features which we believe may be helpful for users and we +@c would like them to be tested: + +@c @itemize @bullet + +@c @item +@c Packaging GNUnet installation along with its run-time dependencies into +@c update packages + +@c @item +@c Installing update packages into compatible hosts + +@c @item +@c Updating an existing installation (which had been installed by +@c gnunet-update) to a newer one + +@c @end itemize + +@c The above said features of gnunet-update are currently available for +@c testing on GNU/Linux systems. + +@c The following is a guide to help you get started with gnunet-update. +@c It shows you how to install the testing binary packages of GNUnet +@c 0.9.1 we have at @uref{https://gnunet.org/install/}. + +@c gnunet-update needs the following dependencies: + +@c @itemize @bullet +@c @item +@c python @geq{} 2.6 + +@c @item +@c gnupg + +@c @item +@c python-gpgme +@c @end itemize + + +@c Checkout gnunet-update: + +@c @c FIXME: git! +@c @example +@c $ svn checkout -r24905 https://gnunet.org/svn/gnunet-update@ +@c @end example + +@c For security reasons, all packages released for gnunet-update from us are +@c signed with the key at @uref{https://gnunet.org/install/key.txt}. +@c You would need to import this key into your gpg key ring. +@c gnunet-update uses this key to verify the integrity of the packages it +@c installs: + +@c @example +@c $ gpg --recv-keys 7C613D78@ +@c @end example + +@c Download the packages relevant to your architecture (currently I have +@c access to GNU/Linux machines on x86_64 and i686, so only two for now, +@c hopefully more later) from https://gnunet.org/install/. + +@c To install the downloaded package into the directory /foo: + +@c @example +@c gnunet-update/bin/gnunet-update install downloaded/package /foo +@c @end example + +@c The installer reports the directories into which shared libraries and +@c dependencies have been installed. You may need to add the reported shared +@c library installation paths to LD_LIBRARY_PATH before you start running any +@c installed binaries. + +@c Please report bugs at https://gnunet.org/bugs/ under the project +@c 'gnunet-update'. + +@node Instructions for Microsoft Windows Platforms (Old) +@subsection Instructions for Microsoft Windows Platforms (Old) + +This document is a @b{DEPRECATED} installation guide for GNUnet on +Windows. +It will not work for recent GNUnet versions, but maybe it will be of +some use if problems arise. + +The Windows build uses a UNIX emulator for Windows, +@uref{http://www.mingw.org/, MinGW}, to build the executable modules. +These modules run natively on Windows and do not require additional +emulation software besides the usual dependencies. + +GNUnet development is mostly done under GNU/Linux and especially git +checkouts may not build out of the box. +We regret any inconvenience, and if you have problems, please report them. + +@menu +* Hardware and OS requirements:: +* Software installation:: +* Building libextractor and GNUnet:: +* Installer:: +* Source:: +@end menu + +@node Hardware and OS requirements +@subsubsection Hardware and OS requirements + +@itemize @bullet + +@item Pentium II or equivalent processor, @geq{} 350 MHz + +@item 128 MB RAM + +@item 600 MB free disk space + +@item Windows 2000 or Windows XP are recommended + +@end itemize + +@node Software installation +@subsubsection Software installation + +@itemize @bullet + +@item +@strong{Compression software}@ + +The software packages GNUnet depends on are usually compressed using UNIX +tools like @command{tar}, @command{gzip}, @command{xzip} and +@command{bzip2}. +If you do not already have an utility that is able to extract such +archives, get @uref{http://www.7-zip.org/, 7-Zip}. + +@item +@strong{UNIX environment}@ + +The MinGW project provides the compiler toolchain that is used to build +GNUnet. +Get the following packages from the +@uref{http://sourceforge.net/projects/mingw/files/, MinGW} project: + +@itemize @bullet + +@item GCC core +@item GCC g++ +@item MSYS +@item MSYS Developer Tool Kit (msysDTK) +@item MSYS Developer Tool Kit - msys-autoconf (bin) +@item MSYS Developer Tool Kit - msys-automake (bin) +@item MinGW Runtime +@item MinGW Utilities +@item Windows API +@item Binutils +@item make +@item pdcurses +@item GDB (snapshot) +@end itemize + +@itemize @bullet + + +@item Install MSYS (to c:\mingw, for example.)@ +Do @strong{not} use spaces in the pathname. +For example, avoid a location such as @file{c:\program files\mingw}. + +@item Install MinGW runtime, utilities and GCC to a subdirectory +(to @file{c:\mingw\mingw}, for example) + +@item Install the Development Kit to the MSYS directory +(@file{c:\mingw}) + +@item Create a batch file bash.bat in your MSYS directory with +the files: + +@example +bin\sh.exe --login +@end example + +This batch file opens a shell which is used to invoke the build +processes. +MinGW's standard shell (@command{msys.bat}) is not suitable +because it opens a separate console window. +On Vista, @command{bash.bat} needs to be run as Administrator. + +@item +Start @command{bash.sh} and rename +@file{c:\mingw\mingw\lib\libstdc++.la} to avoid problems: + +@example +mv /usr/mingw/lib/libstdc++.la /usr/mingw/lib/libstdc++.la.broken +@end example + +@item +Unpack the Windows API to the MinGW directory (@file{c:\mingw\mingw\}) and +remove the declaration of DATADIR from +(@file{c:\mingw\mingw\include\objidl.h} (lines 55-58) + +@item +Unpack autoconf, automake to the MSYS directory (@file{c:\mingw}) + +@item +Install all other packages to the MinGW directory (@file{c:\mingw\mingw\}) +@end itemize + + +@item @strong{GNU Libtool}@ +GNU Libtool is required to use shared libraries. +Get the prebuilt package from here and unpack it to the +MinGW directory (@file{c:\mingw}) + +@item @strong{Pthreads}@ +GNUnet uses the portable POSIX thread library for multi-threading: + +@itemize @bullet + +@item Save +@uref{ftp://sources.redhat.com/pub/pthreads-win32/dll-latest/lib/x86/libpthreadGC2.a, libpthreadGC2.a} +(x86) or +@uref{ftp://sources.redhat.com/pub/pthreads-win32/dll-latest/lib/x64/libpthreadGC2.a, libpthreadGC2.a} +(x64) as libpthread.a into the @file{lib} +directory (@file{c:\mingw\mingw\lib\libpthread.a}). + +@item Save +@uref{ftp://sources.redhat.com/pub/pthreads-win32/dll-latest/lib/x86/pthreadGC2.dll, pthreadGC2.dll} +(x86) or +@uref{ftp://sources.redhat.com/pub/pthreads-win32/dll-latest/lib/x64/pthreadGC2.dll, libpthreadGC2.a} +(x64) into the MinGW @file{bin} directory (@file{c:\mingw\mingw\bin}). + +@item Download all header files from +@uref{ftp://sources.redhat.com/pub/pthreads-win32/dll-latest/include/, include/} +to the @file{include} directory (@file{c:\mingw\mingw\include}). +@end itemize + + +@item @strong{GNU MP}@ +GNUnet uses the GNU Multiple Precision library for special cryptographic +operations. Get the GMP binary package from the +@uref{http://sourceforge.net/projects/mingwrep/, MinGW repository} and +unpack it to the MinGW directory (@file{c:\mingw\mingw}) + +@item @strong{GNU Gettext}@ +GNU gettext is used to provide national language support. +Get the prebuilt package from hereand unpack it to the MinGW +directory (@file{c:\mingw\mingw}) + +@item @strong{GNU iconv}@ +GNU Libiconv is used for character encoding conversion. +Get the prebuilt package from here and unpack it to the MinGW +directory (@file{c:\mingw\mingw}). + +@item @strong{SQLite}@ +GNUnet uses the SQLite database to store data. +Get the prebuilt binary from here and unpack it to your MinGW directory. + +@item @strong{MySQL}@ +As an alternative to SQLite, GNUnet also supports MySQL. + +@itemize @bullet + +@item Get the binary installer from the +@uref{http://dev.mysql.com/downloads/mysql/4.1.html#Windows, MySQL project} +(version 4.1), install it and follow the instructions in +@file{README.mysql}. + +@item Create a temporary build directory (@file{c:\mysql}) + +@item Copy the directories @file{include\} and @file{lib\} from the +MySQL directory to the new directory + +@item Get the patches from +@uref{http://bugs.mysql.com/bug.php?id=8906&files=1, Bug #8906} and +@uref{http://bugs.mysql.com/bug.php?id=8872&files=1, Bug #8872} (the +latter is only required for MySQL + +@example +patch -p 0 +@end example + +@item Move @file{lib\opt\libmysql.dll} to @file{lib\libmysql.dll} + +@item Change to @file{lib\} and create an import library: + +@example +dlltool --input-def ../include/libmySQL.def \ +--dllname libmysql.dll \ +--output-lib libmysqlclient.a -k +@end example + +@item Copy include\* to include\mysql\ + +@item Pass @code{--with-mysql=/c/mysql} to +@command{./configure} and copy @file{libmysql.dll} +to your PATH or GNUnet's @file{bin} directory +@end itemize + + +@item @strong{GTK+}@ +@command{gnunet-gtk} and @command{libextractor} depend on GTK. +Get the the binary and developer packages of @command{atk}, +@command{glib}, @command{gtk}, @command{iconv}, +@command{gettext-runtime}, @command{pango} from +@uref{ftp://ftp.gtk.org/pub/gtk/v2.6/win32, gtk.org} and unpack them +to the MinGW directory (@file{c:\mingw\mingw}). +@c FIXME: The URL below for pkg-config seems wrong. +Get @uref{http://www.gtk.org/download/win32.php, pkg-config} and +@command{libpng} and unpack them to the MinGW directory +(@file{c:\mingw\mingw}). +Here is an all-in-one package for the +@uref{http://ftp.gnome.org/pub/gnome/binaries/win32/gtk+/2.24/gtk+-bundle_2.24.10-20120208_win32.zip, gtk+dependencies} +. Do not overwrite any existing files! + +@item @strong{Glade}@ +@command{gnunet-gtk} and @command{gnunet-setup} were created using +this interface builder + +@itemize @bullet + +@item Get the Glade and libglade (-bin and -devel) packages +(without GTK!) from +@uref{http://gladewin32.sourceforge.net/, GladeWin32} and unpack them to +the MinGW directory (@file{c:\mingw\mingw}). + +@item Get @command{libxml} from here and unpack it to the MinGW +directory (@file{c:\mingw\mingw}). +@end itemize + +@c FIXME: URLs +@item @strong{zLib}@ +@command{libextractor} requires @command{zLib} to decompress some file +formats. GNUnet uses it to (de)compress meta-data. +Get zLib from here (Signature) and unpack it to the MinGW directory +(@file{c:\mingw\mingw}). + +@item @strong{Bzip2}@ +@command{libextractor} also requires @command{Bzip2} to +decompress some file formats. +Get the Bzip2 (binary and developer package) from +@uref{http://gnuwin32.sourceforge.net/packages/bzip2.htm, GnuWin32} and +unpack it to the MinGW directory (@file{c:\mingw\mingw}). + +@item @strong{Libgcrypt}@ +@command{Libgcrypt} provides the cryptographic functions used by GNUnet. +Get Libgcrypt from @uref{ftp://ftp.gnupg.org/gcrypt/libgcrypt/, here}, +compile and place it in the MinGW directory +(@file{c:\mingw\mingw}). Currently libgcrypt @geq{} 1.4.2 is required to +compile GNUnet. + +@item @strong{PlibC}@ +PlibC emulates Unix functions under Windows. Get PlibC from here and +unpack it to the MinGW directory (c:\mingw\mingw) + +@item @strong{OGG Vorbis}@ +@command{OGG Vorbis} is used to extract meta-data from @file{.ogg} files. +Get the packages +@uref{http://www.gnunet.org/libextractor/download/win/libogg-1.1.4.zip, libogg} +and +@uref{http://www.gnunet.org/libextractor/download/win/libvorbis-1.2.3.zip, libvorbis} +from the +@uref{http://ftp.gnu.org/gnu/libextractor/libextractor-w32-1.0.0.zip, libextractor win32 build} +and unpack them to the MinGW directory (c:\mingw\mingw). + +@item @strong{Exiv2}@ +(lib)Exiv2 is used to extract meta-data from files with Exiv2 meta-data. +Download +@uref{http://www.gnunet.org/libextractor/download/win/exiv2-0.18.2.zip, Exiv2} +and unpack it to the MSYS directory (c:\mingw). +@end itemize + +@node Building libextractor and GNUnet +@subsubsection Building libextractor and GNUnet + +Before you compile @command{libextractor} or @command{GNUnet}, +be sure to set @code{PKG_CONFIG_PATH}: + +@example +export PKG_CONFIG_PATH=/mingw/lib/pkgconfig +@end example + +@noindent +@xref{GNUnet Installation Handbook}, for basic instructions on building +@command{libextractor} and @command{GNUnet}. +By default, all modules that are created in this way contain +debug information and are quite large. To compile release versions +(small and fast) set the variable @code{CFLAGS}: + +@example +export CFLAGS='-O2 -march=pentium -fomit-frame-pointer' +./configure --prefix=$HOME --with-extractor=$HOME +@end example + +@node Installer +@subsubsection Installer + +The GNUnet installer is made with +@uref{http://nsis.sourceforge.net/, NSIS}. The installer script is +located in @file{contrib\win} in the GNUnet source tree. + +@node Source +@subsubsection Source + +@c FIXME: URL +The sources of all dependencies are available here. + +@c @node Portable GNUnet +@c @section Portable GNUnet + +@c Quick instructions on how to use the most recent GNUnet on most GNU/Linux +@c distributions + +@c Currently this has only been tested on Ubuntu 12.04, 12.10, 13.04, Debian +@c and CentOS 6, but it should work on almost any GNU/Linux distribution. +@c More in-detail information can be found in the handbook. + +@c Note 2017-10: Currently this section assumes the old SVN repo of GNUnet +@c which no longer exists. + +@c @menu +@c * Prerequisites:: +@c * Download & set up gnunet-update:: +@c * Install GNUnet:: +@c @end menu + +@c @node Prerequisites +@c @subsection Prerequisites + +@c Open a terminal and paste this line into it to install all required tools +@c needed: + +@c @example +@c sudo apt-get install python-gpgme subversion +@c @end example + +@c @node Download & set up gnunet-update +@c @subsection Download & set up gnunet-update + +@c The following command will download a working version of gnunet-update +@c with the subversion tool and import the public key which is needed for +@c authentication: + +@c @example +@c svn checkout -r24905 https://gnunet.org/svn/gnunet-update ~/gnunet-update +@c cd ~/gnunet-update +@c gpg --keyserver "hkp://keys.gnupg.net" --recv-keys 7C613D78 +@c @end example + +@c @node Install GNUnet +@c @subsection Install GNUnet + +@c Download and install GNUnet binaries which can be found here and set +@c library paths: + +@c @example +@c wget -P /tmp https://gnunet.org/install/packs/gnunet-0.9.4-`uname -m`.tgz +@c ./bin/gnunet-update install /tmp/gnunet-0.9*.tgz ~ +@c echo "PATH DEFAULT=$@{PATH@}:$HOME/bin" >> ~/.pam_environment +@c echo -e "$@{HOME@}/lib\n$@{HOME@}/lib/gnunet-deps" | sudo tee \ +@c /etc/ld.so.conf.d/gnunet.conf > /dev/null +@c sudo ldconfig +@c @end example + +@c You may need to re-login once after executing these last commands + +@c That's it, GNUnet is installed in your home directory now. GNUnet can be +@c configured and afterwards started by executing: + +@c @example +@c gnunet-arm -s +@c @end example + +@node The graphical configuration interface +@section The graphical configuration interface + +If you also would like to use @command{gnunet-gtk} and +@command{gnunet-setup} (highly recommended for beginners), do: + +@example +wget -P /tmp \ +https://gnunet.org/install/packs/gnunet-0.9.4-gtk-0.9.4-`uname -m`.tgz +sh ~/gnunet-update/bin/gnunet-update install /tmp/gnunet-*gtk*.tgz ~ +sudo ldconfig +@end example + +Now you can run @command{gnunet-setup} for easy configuration of your +GNUnet peer. + +@menu +* Configuring your peer:: +* Configuring the Friend-to-Friend (F2F) mode:: +* Configuring the hostlist to bootstrap:: +* Configuration of the HOSTLIST proxy settings:: +* Configuring your peer to provide a hostlist :: +* Configuring the datastore:: +* Configuring the MySQL database:: +* Reasons for using MySQL:: +* Reasons for not using MySQL:: +* Setup Instructions:: +* Testing:: +* Performance Tuning:: +* Setup for running Testcases:: +* Configuring the Postgres database:: +* Reasons to use Postgres:: +* Reasons not to use Postgres:: +* Manual setup instructions:: +* Testing the setup manually:: +* Configuring the datacache:: +* Configuring the file-sharing service:: +* Configuring logging:: +* Configuring the transport service and plugins:: +* Configuring the wlan transport plugin:: +* Configuring HTTP(S) reverse proxy functionality using Apache or nginx:: +* Blacklisting peers:: +* Configuration of the HTTP and HTTPS transport plugins:: +* Configuring the GNU Name System:: +* Configuring the GNUnet VPN:: +* Bandwidth Configuration:: +* Configuring NAT:: +* Peer configuration for distributions:: +@end menu + +@node Configuring your peer +@subsection Configuring your peer + +This chapter will describe the various configuration options in GNUnet. + +The easiest way to configure your peer is to use the +@command{gnunet-setup} tool. +@command{gnunet-setup} is part of the @command{gnunet-gtk} +application. You might have to install it separately. + +Many of the specific sections from this chapter actually are linked from +within @command{gnunet-setup} to help you while using the setup tool. + +While you can also configure your peer by editing the configuration +file by hand, this is not recommended for anyone except for developers +as it requires a more in-depth understanding of the configuration files +and internal dependencies of GNUnet. + +@node Configuring the Friend-to-Friend (F2F) mode +@subsection Configuring the Friend-to-Friend (F2F) mode + +GNUnet knows three basic modes of operation: +@itemize @bullet +@item In standard "peer-to-peer" mode, +your peer will connect to any peer. +@item In the pure "friend-to-friend" +mode, your peer will ONLY connect to peers from a list of friends +specified in the configuration. +@item Finally, in mixed mode, +GNUnet will only connect to arbitrary peers if it +has at least a specified number of connections to friends. +@end itemize + +When configuring any of the F2F ("friend-to-friend") modes, +you first need to create a file with the peer identities +of your friends. Ask your friends to run + +@example +$ gnunet-peerinfo -sq +@end example + +@noindent +The resulting output of this command needs to be added to your +@file{friends} file, which is simply a plain text file with one line +per friend with the output from the above command. + +You then specify the location of your @file{friends} file in the +@code{FRIENDS} option of the "topology" section. + +Once you have created the @file{friends} file, you can tell GNUnet to only +connect to your friends by setting the @code{FRIENDS-ONLY} option +(again in the "topology" section) to YES. + +If you want to run in mixed-mode, set "FRIENDS-ONLY" to NO and configure a +minimum number of friends to have (before connecting to arbitrary peers) +under the "MINIMUM-FRIENDS" option. + +If you want to operate in normal P2P-only mode, simply set +@code{MINIMUM-FRIENDS} to zero and @code{FRIENDS_ONLY} to NO. +This is the default. + +@node Configuring the hostlist to bootstrap +@subsection Configuring the hostlist to bootstrap + +After installing the software you need to get connected to the GNUnet +network. The configuration file included in your download is already +configured to connect you to the GNUnet network. +In this section the relevant configuration settings are explained. + +To get an initial connection to the GNUnet network and to get to know +peers already connected to the network you can use the so called +"bootstrap servers". +These servers can give you a list of peers connected to the network. +To use these bootstrap servers you have to configure the hostlist daemon +to activate bootstrapping. + +To activate bootstrapping, edit the @code{[hostlist]}-section in your +configuration file. You have to set the argument @command{-b} in the +options line: + +@example +[hostlist] +OPTIONS = -b +@end example + +Additionally you have to specify which server you want to use. +The default bootstrapping server is +"@uref{http://v10.gnunet.org/hostlist, http://v10.gnunet.org/hostlist}". +[^] To set the server you have to edit the line "SERVERS" in the hostlist +section. To use the default server you should set the lines to + +@example +SERVERS = http://v10.gnunet.org/hostlist [^] +@end example + +@noindent +To use bootstrapping your configuration file should include these lines: + +@example +[hostlist] +OPTIONS = -b +SERVERS = http://v10.gnunet.org/hostlist [^] +@end example + +@noindent +Besides using bootstrap servers you can configure your GNUnet peer to +recieve hostlist advertisements. +Peers offering hostlists to other peers can send advertisement messages +to peers that connect to them. If you configure your peer to receive these +messages, your peer can download these lists and connect to the peers +included. These lists are persistent, which means that they are saved to +your hard disk regularly and are loaded during startup. + +To activate hostlist learning you have to add the @command{-e} +switch to the @code{OPTIONS} line in the hostlist section: + +@example +[hostlist] +OPTIONS = -b -e +@end example + +@noindent +Furthermore you can specify in which file the lists are saved. +To save the lists in the file @file{hostlists.file} just add the line: + +@example +HOSTLISTFILE = hostlists.file +@end example + +@noindent +Best practice is to activate both bootstrapping and hostlist learning. +So your configuration file should include these lines: + +@example +[hostlist] +OPTIONS = -b -e +HTTPPORT = 8080 +SERVERS = http://v10.gnunet.org/hostlist [^] +HOSTLISTFILE = $SERVICEHOME/hostlists.file +@end example + +@node Configuration of the HOSTLIST proxy settings +@subsection Configuration of the HOSTLIST proxy settings + +The hostlist client can be configured to use a proxy to connect to the +hostlist server. +This functionality can be configured in the configuration file directly +or using the @command{gnunet-setup} tool. + +The hostlist client supports the following proxy types at the moment: + +@itemize @bullet +@item HTTP and HTTP 1.0 only proxy +@item SOCKS 4/4a/5/5 with hostname +@end itemize + +In addition authentication at the proxy with username and password can be +configured. + +To configure proxy support for the hostlist client in the +@command{gnunet-setup} tool, select the "hostlist" tab and select +the appropriate proxy type. +The hostname or IP address (including port if required) has to be entered +in the "Proxy hostname" textbox. If required, enter username and password +in the "Proxy username" and "Proxy password" boxes. +Be aware that this information will be stored in the configuration in +plain text (TODO: Add explanation and generalize the part in Chapter 3.6 +about the encrypted home). + +To provide these options directly in the configuration, you can +enter the following settings in the @code{[hostlist]} section of +the configuration: + +@example +# Type of proxy server, +# Valid values: HTTP, HTTP_1_0, SOCKS4, SOCKS5, SOCKS4A, SOCKS5_HOSTNAME +# Default: HTTP +# PROXY_TYPE = HTTP + +# Hostname or IP of proxy server +# PROXY = +# User name for proxy server +# PROXY_USERNAME = +# User password for proxy server +# PROXY_PASSWORD = +@end example + +@node Configuring your peer to provide a hostlist +@subsection Configuring your peer to provide a hostlist + +If you operate a peer permanently connected to GNUnet you can configure +your peer to act as a hostlist server, providing other peers the list of +peers known to him. + +Your server can act as a bootstrap server and peers needing to obtain a +list of peers can contact it to download this list. +To download this hostlist the peer uses HTTP. +For this reason you have to build your peer with libgnurl (or libcurl) +and microhttpd support. +How you build your peer with these options can be found here: +@xref{Generic installation instructions}. + +To configure your peer to act as a bootstrap server you have to add the +@command{-p} option to @code{OPTIONS} in the @code{[hostlist]} section +of your configuration file. +Besides that you have to specify a port number for the http server. +In conclusion you have to add the following lines: + +@example +[hostlist] +HTTPPORT = 12980 +OPTIONS = -p +@end example + +@noindent +If your peer acts as a bootstrap server other peers should know about +that. You can advertise the hostlist your are providing to other peers. +Peers connecting to your peer will get a message containing an +advertisement for your hostlist and the URL where it can be downloaded. +If this peer is in learning mode, it will test the hostlist and, in the +case it can obtain the list successfully, it will save it for +bootstrapping. + +To activate hostlist advertisement on your peer, you have to set the +following lines in your configuration file: + +@example +[hostlist] +EXTERNAL_DNS_NAME = example.org +HTTPPORT = 12981 +OPTIONS = -p -a +@end example + +@noindent +With this configuration your peer will a act as a bootstrap server and +advertise this hostlist to other peers connecting to it. +The URL used to download the list will be +@code{@uref{http://example.org:12981/, http://example.org:12981/}}. + +Please notice: + +@itemize @bullet +@item The hostlist is @b{not} human readable, so you should not try to +download it using your webbrowser. Just point your GNUnet peer to the +address! +@item Advertising without providing a hostlist does not make sense and +will not work. +@end itemize + +@node Configuring the datastore +@subsection Configuring the datastore + +The datastore is what GNUnet uses for long-term storage of file-sharing +data. Note that long-term does not mean 'forever' since content does have +an expiration date, and of course storage space is finite (and hence +sometimes content may have to be discarded). + +Use the @code{QUOTA} option to specify how many bytes of storage space +you are willing to dedicate to GNUnet. + +In addition to specifying the maximum space GNUnet is allowed to use for +the datastore, you need to specify which database GNUnet should use to do +so. Currently, you have the choice between sqLite, MySQL and Postgres. + +@node Configuring the MySQL database +@subsection Configuring the MySQL database + +This section describes how to setup the MySQL database for GNUnet. + +Note that the mysql plugin does NOT work with mysql before 4.1 since we +need prepared statements. +We are generally testing the code against MySQL 5.1 at this point. + +@node Reasons for using MySQL +@subsection Reasons for using MySQL + +@itemize @bullet + +@item On up-to-date hardware wher +mysql can be used comfortably, this module +will have better performance than the other database choices (according +to our tests). + +@item Its often possible to recover the mysql database from internal +inconsistencies. Some of the other databases do not support repair. +@end itemize + +@node Reasons for not using MySQL +@subsection Reasons for not using MySQL + +@itemize @bullet +@item Memory usage (likely not an issue if you have more than 1 GB) +@item Complex manual setup +@end itemize + +@node Setup Instructions +@subsection Setup Instructions + +@itemize @bullet + +@item In @file{gnunet.conf} set in section @code{DATASTORE} the value for +@code{DATABASE} to @code{mysql}. + +@item Access mysql as root: + +@example +$ mysql -u root -p +@end example + +@noindent +and issue the following commands, replacing $USER with the username +that will be running @command{gnunet-arm} (so typically "gnunet"): + +@example +CREATE DATABASE gnunet; +GRANT select,insert,update,delete,create,alter,drop,create \ +temporary tables ON gnunet.* TO $USER@@localhost; +SET PASSWORD FOR $USER@@localhost=PASSWORD('$the_password_you_like'); +FLUSH PRIVILEGES; +@end example + +@item +In the $HOME directory of $USER, create a @file{.my.cnf} file with the +following lines + +@example +[client] +user=$USER +password=$the_password_you_like +@end example + +@end itemize + +Thats it. Note that @file{.my.cnf} file is a slight security risk unless +its on a safe partition. The @file{$HOME/.my.cnf} can of course be +a symbolic link. +Luckily $USER has only priviledges to mess up GNUnet's tables, +which should be pretty harmless. + +@node Testing +@subsection Testing + +You should briefly try if the database connection works. First, login +as $USER. Then use: + +@example +$ mysql -u $USER +mysql> use gnunet; +@end example + +@noindent +If you get the message + +@example +Database changed +@end example + +@noindent +it probably works. + +If you get + +@example +ERROR 2002: Can't connect to local MySQL server +through socket '/tmp/mysql.sock' (2) +@end example + +@noindent +it may be resolvable by + +@example +ln -s /var/run/mysqld/mysqld.sock /tmp/mysql.sock +@end example + +@noindent +so there may be some additional trouble depending on your mysql setup. + +@node Performance Tuning +@subsection Performance Tuning + +For GNUnet, you probably want to set the option + +@example +innodb_flush_log_at_trx_commit = 0 +@end example + +@noindent +for a rather dramatic boost in MySQL performance. However, this reduces +the "safety" of your database as with this options you may loose +transactions during a power outage. +While this is totally harmless for GNUnet, the option applies to all +applications using MySQL. So you should set it if (and only if) GNUnet is +the only application on your system using MySQL. + +@node Setup for running Testcases +@subsection Setup for running Testcases + +If you want to run the testcases, you must create a second database +"gnunetcheck" with the same username and password. This database will +then be used for testing (@command{make check}). + +@node Configuring the Postgres database +@subsection Configuring the Postgres database + +This text describes how to setup the Postgres database for GNUnet. + +This Postgres plugin was developed for Postgres 8.3 but might work for +earlier versions as well. + +@node Reasons to use Postgres +@subsection Reasons to use Postgres + +@itemize @bullet +@item Easier to setup than MySQL +@item Real database +@end itemize + +@node Reasons not to use Postgres +@subsection Reasons not to use Postgres + +@itemize @bullet +@item Quite slow +@item Still some manual setup required +@end itemize + +@node Manual setup instructions +@subsection Manual setup instructions + +@itemize @bullet +@item In @file{gnunet.conf} set in section @code{DATASTORE} the value for +@code{DATABASE} to @code{postgres}. +@item Access Postgres to create a user: + +@table @asis +@item with Postgres 8.x, use: + +@example +# su - postgres +$ createuser +@end example + +@noindent +and enter the name of the user running GNUnet for the role interactively. +Then, when prompted, do not set it to superuser, allow the creation of +databases, and do not allow the creation of new roles. + +@item with Postgres 9.x, use: + +@example +# su - postgres +$ createuser -d $GNUNET_USER +@end example + +@noindent +where $GNUNET_USER is the name of the user running GNUnet. + +@end table + + +@item +As that user (so typically as user "gnunet"), create a database (or two): + +@example +$ createdb gnunet +# this way you can run "make check" +$ createdb gnunetcheck +@end example + +@end itemize + +Now you should be able to start @code{gnunet-arm}. + +@node Testing the setup manually +@subsection Testing the setup manually + +You may want to try if the database connection works. First, again login +as the user who will run @command{gnunet-arm}. Then use: + +@example +$ psql gnunet # or gnunetcheck +gnunet=> \dt +@end example + +@noindent +If, after you have started @command{gnunet-arm} at least once, you get +a @code{gn090} table here, it probably works. + +@node Configuring the datacache +@subsection Configuring the datacache +@c %**end of header + +The datacache is what GNUnet uses for storing temporary data. This data is +expected to be wiped completely each time GNUnet is restarted (or the +system is rebooted). + +You need to specify how many bytes GNUnet is allowed to use for the +datacache using the @code{QUOTA} option in the section @code{[dhtcache]}. +Furthermore, you need to specify which database backend should be used to +store the data. Currently, you have the choice between +sqLite, MySQL and Postgres. + +@node Configuring the file-sharing service +@subsection Configuring the file-sharing service + +In order to use GNUnet for file-sharing, you first need to make sure +that the file-sharing service is loaded. +This is done by setting the @code{AUTOSTART} option in +section @code{[fs]} to "YES". Alternatively, you can run + +@example +$ gnunet-arm -i fs +@end example + +@noindent +to start the file-sharing service by hand. + +Except for configuring the database and the datacache the only important +option for file-sharing is content migration. + +Content migration allows your peer to cache content from other peers as +well as send out content stored on your system without explicit requests. +This content replication has positive and negative impacts on both system +performance and privacy. + +FIXME: discuss the trade-offs. Here is some older text about it... + +Setting this option to YES allows gnunetd to migrate data to the local +machine. Setting this option to YES is highly recommended for efficiency. +Its also the default. If you set this value to YES, GNUnet will store +content on your machine that you cannot decrypt. +While this may protect you from liability if the judge is sane, it may +not (IANAL). If you put illegal content on your machine yourself, setting +this option to YES will probably increase your chances to get away with it +since you can plausibly deny that you inserted the content. +Note that in either case, your anonymity would have to be broken first +(which may be possible depending on the size of the GNUnet network and the +strength of the adversary). + +@node Configuring logging +@subsection Configuring logging + +Logging in GNUnet 0.9.0 is controlled via the "-L" and "-l" options. +Using @code{-L}, a log level can be specified. With log level +@code{ERROR} only serious errors are logged. +The default log level is @code{WARNING} which causes anything of +concern to be logged. +Log level @code{INFO} can be used to log anything that might be +interesting information whereas +@code{DEBUG} can be used by developers to log debugging messages +(but you need to run @code{./configure} with +@code{--enable-logging=verbose} to get them compiled). +The @code{-l} option is used to specify the log file. + +Since most GNUnet services are managed by @code{gnunet-arm}, using the +@code{-l} or @code{-L} options directly is not possible. +Instead, they can be specified using the @code{OPTIONS} configuration +value in the respective section for the respective service. +In order to enable logging globally without editing the @code{OPTIONS} +values for each service, @command{gnunet-arm} supports a +@code{GLOBAL_POSTFIX} option. +The value specified here is given as an extra option to all services for +which the configuration does contain a service-specific @code{OPTIONS} +field. + +@code{GLOBAL_POSTFIX} can contain the special sequence "@{@}" which +is replaced by the name of the service that is being started. +Furthermore, @code{GLOBAL_POSTFIX} is special in that sequences +starting with "$" anywhere in the string are expanded (according +to options in @code{PATHS}); this expansion otherwise is +only happening for filenames and then the "$" must be the +first character in the option. Both of these restrictions do +not apply to @code{GLOBAL_POSTFIX}. +Note that specifying @code{%} anywhere in the @code{GLOBAL_POSTFIX} +disables both of these features. + +In summary, in order to get all services to log at level +@code{INFO} to log-files called @code{SERVICENAME-logs}, the +following global prefix should be used: + +@example +GLOBAL_POSTFIX = -l $SERVICEHOME/@{@}-logs -L INFO +@end example + +@node Configuring the transport service and plugins +@subsection Configuring the transport service and plugins + +The transport service in GNUnet is responsible to maintain basic +connectivity to other peers. +Besides initiating and keeping connections alive it is also responsible +for address validation. + +The GNUnet transport supports more than one transport protocol. +These protocols are configured together with the transport service. + +The configuration section for the transport service itself is quite +similar to all the other services + +@example +AUTOSTART = YES +@@UNIXONLY@@ PORT = 2091 +HOSTNAME = localhost +HOME = $SERVICEHOME +CONFIG = $DEFAULTCONFIG +BINARY = gnunet-service-transport +#PREFIX = valgrind +NEIGHBOUR_LIMIT = 50 +ACCEPT_FROM = 127.0.0.1; +ACCEPT_FROM6 = ::1; +PLUGINS = tcp udp +UNIXPATH = /tmp/gnunet-service-transport.sock +@end example + +Different are the settings for the plugins to load @code{PLUGINS}. +The first setting specifies which transport plugins to load. + +@itemize @bullet +@item transport-unix +A plugin for local only communication with UNIX domain sockets. Used for +testing and available on unix systems only. Just set the port + +@example +[transport-unix] +PORT = 22086 +TESTING_IGNORE_KEYS = ACCEPT_FROM; +@end example + +@item transport-tcp +A plugin for communication with TCP. Set port to 0 for client mode with +outbound only connections + +@example +[transport-tcp] +# Use 0 to ONLY advertise as a peer behind NAT (no port binding) +PORT = 2086 +ADVERTISED_PORT = 2086 +TESTING_IGNORE_KEYS = ACCEPT_FROM; +# Maximum number of open TCP connections allowed +MAX_CONNECTIONS = 128 +@end example + +@item transport-udp +A plugin for communication with UDP. Supports peer discovery using +broadcasts. + +@example +[transport-udp] +PORT = 2086 +BROADCAST = YES +BROADCAST_INTERVAL = 30 s +MAX_BPS = 1000000 +TESTING_IGNORE_KEYS = ACCEPT_FROM; +@end example + +@item transport-http +HTTP and HTTPS support is split in two part: a client plugin initiating +outbound connections and a server part accepting connections from the +client. The client plugin just takes the maximum number of connections as +an argument. + +@example +[transport-http_client] +MAX_CONNECTIONS = 128 +TESTING_IGNORE_KEYS = ACCEPT_FROM; +@end example + +@example +[transport-https_client] +MAX_CONNECTIONS = 128 +TESTING_IGNORE_KEYS = ACCEPT_FROM; +@end example + +@noindent +The server has a port configured and the maximum nunber of connections. +The HTTPS part has two files with the certificate key and the certificate +file. + +The server plugin supports reverse proxies, so a external hostname can be +set using the @code{EXTERNAL_HOSTNAME} setting. +The webserver under this address should forward the request to the peer +and the configure port. + +@example +[transport-http_server] +EXTERNAL_HOSTNAME = fulcrum.net.in.tum.de/gnunet +PORT = 1080 +MAX_CONNECTIONS = 128 +TESTING_IGNORE_KEYS = ACCEPT_FROM; +@end example + +@example +[transport-https_server] +PORT = 4433 +CRYPTO_INIT = NORMAL +KEY_FILE = https.key +CERT_FILE = https.cert +MAX_CONNECTIONS = 128 +TESTING_IGNORE_KEYS = ACCEPT_FROM; +@end example + +@item transport-wlan + +The next section describes how to setup the WLAN plugin, +so here only the settings. Just specify the interface to use: + +@example +[transport-wlan] +# Name of the interface in monitor mode (typically monX) +INTERFACE = mon0 +# Real hardware, no testing +TESTMODE = 0 +TESTING_IGNORE_KEYS = ACCEPT_FROM; +@end example +@end itemize + +@node Configuring the wlan transport plugin +@subsection Configuring the wlan transport plugin + +The wlan transport plugin enables GNUnet to send and to receive data on a +wlan interface. +It has not to be connected to a wlan network as long as sender and +receiver are on the same channel. This enables you to get connection to +GNUnet where no internet access is possible, for example during +catastrophes or when censorship cuts you off from the internet. + + +@menu +* Requirements for the WLAN plugin:: +* Configuration:: +* Before starting GNUnet:: +* Limitations and known bugs:: +@end menu + + +@node Requirements for the WLAN plugin +@subsubsection Requirements for the WLAN plugin + +@itemize @bullet + +@item wlan network card with monitor support and packet injection +(see @uref{http://www.aircrack-ng.org/, aircrack-ng.org}) + +@item Linux kernel with mac80211 stack, introduced in 2.6.22, tested with +2.6.35 and 2.6.38 + +@item Wlantools to create the a monitor interface, tested with airmon-ng +of the aircrack-ng package +@end itemize + +@node Configuration +@subsubsection Configuration + +There are the following options for the wlan plugin (they should be like +this in your default config file, you only need to adjust them if the +values are incorrect for your system) + +@example +# section for the wlan transport plugin +[transport-wlan] +# interface to use, more information in the +# "Before starting GNUnet" section of the handbook. +INTERFACE = mon0 +# testmode for developers: +# 0 use wlan interface, +#1 or 2 use loopback driver for tests 1 = server, 2 = client +TESTMODE = 0 +@end example + +@node Before starting GNUnet +@subsubsection Before starting GNUnet + +Before starting GNUnet, you have to make sure that your wlan interface is +in monitor mode. +One way to put the wlan interface into monitor mode (if your interface +name is wlan0) is by executing: + +@example +sudo airmon-ng start wlan0 +@end example + +@noindent +Here is an example what the result should look like: + +@example +Interface Chipset Driver +wlan0 Intel 4965 a/b/g/n iwl4965 - [phy0] +(monitor mode enabled on mon0) +@end example + +@noindent +The monitor interface is mon0 is the one that you have to put into the +configuration file. + +@node Limitations and known bugs +@subsubsection Limitations and known bugs + +Wlan speed is at the maximum of 1 Mbit/s because support for choosing the +wlan speed with packet injection was removed in newer kernels. +Please pester the kernel developers about fixing this. + +The interface channel depends on the wlan network that the card is +connected to. If no connection has been made since the start of the +computer, it is usually the first channel of the card. +Peers will only find each other and communicate if they are on the same +channel. Channels must be set manually, i.e. using: + +@example +iwconfig wlan0 channel 1 +@end example + +@node Configuring HTTP(S) reverse proxy functionality using Apache or nginx +@subsection Configuring HTTP(S) reverse proxy functionality using Apache or nginx + +The HTTP plugin supports data transfer using reverse proxies. A reverse +proxy forwards the HTTP request he receives with a certain URL to another +webserver, here a GNUnet peer. + +So if you have a running Apache or nginx webserver you can configure it to +be a GNUnet reverse proxy. Especially if you have a well-known webiste +this improves censorship resistance since it looks as normal surfing +behaviour. + +To do so, you have to do two things: + +@itemize @bullet +@item Configure your webserver to forward the GNUnet HTTP traffic +@item Configure your GNUnet peer to announce the respective address +@end itemize + +As an example we want to use GNUnet peer running: + +@itemize @bullet + +@item HTTP server plugin on @code{gnunet.foo.org:1080} + +@item HTTPS server plugin on @code{gnunet.foo.org:4433} + +@item A apache or nginx webserver on +@uref{http://www.foo.org/, http://www.foo.org:80/} + +@item A apache or nginx webserver on https://www.foo.org:443/ +@end itemize + +And we want the webserver to accept GNUnet traffic under +@code{http://www.foo.org/bar/}. The required steps are described here: + +@menu +* Reverse Proxy - Configure your Apache2 HTTP webserver:: +* Reverse Proxy - Configure your Apache2 HTTPS webserver:: +* Reverse Proxy - Configure your nginx HTTPS webserver:: +* Reverse Proxy - Configure your nginx HTTP webserver:: +* Reverse Proxy - Configure your GNUnet peer:: +@end menu + +@node Reverse Proxy - Configure your Apache2 HTTP webserver +@subsubsection Reverse Proxy - Configure your Apache2 HTTP webserver + +First of all you need mod_proxy installed. + +Edit your webserver configuration. Edit +@code{/etc/apache2/apache2.conf} or the site-specific configuration file. + +In the respective @code{server config},@code{virtual host} or +@code{directory} section add the following lines: + +@example +ProxyTimeout 300 +ProxyRequests Off +<Location /bar/ > +ProxyPass http://gnunet.foo.org:1080/ +ProxyPassReverse http://gnunet.foo.org:1080/ +</Location> +@end example + +@node Reverse Proxy - Configure your Apache2 HTTPS webserver +@subsubsection Reverse Proxy - Configure your Apache2 HTTPS webserver + +We assume that you already have an HTTPS server running, if not please +check how to configure a HTTPS host. An easy to use example is the +@file{apache2/sites-available/default-ssl} example configuration file. + +In the respective HTTPS @code{server config},@code{virtual host} or +@code{directory} section add the following lines: + +@example +SSLProxyEngine On +ProxyTimeout 300 +ProxyRequests Off +<Location /bar/ > +ProxyPass https://gnunet.foo.org:4433/ +ProxyPassReverse https://gnunet.foo.org:4433/ +</Location> +@end example + +@noindent +More information about the apache mod_proxy configuration can be found +here: @uref{http://httpd.apache.org/docs/2.2/mod/mod_proxy.html#proxypass} +. + +@node Reverse Proxy - Configure your nginx HTTPS webserver +@subsubsection Reverse Proxy - Configure your nginx HTTPS webserver + +Since nginx does not support chunked encoding, you first of all have to +install @code{chunkin}: @uref{http://wiki.nginx.org/HttpChunkinModule}. + +To enable chunkin add: + +@example +chunkin on; +error_page 411 = @@my_411_error; +location @@my_411_error @{ +chunkin_resume; +@} +@end example + +@noindent +Edit your webserver configuration. Edit @file{/etc/nginx/nginx.conf} or +the site-specific configuration file. + +In the @code{server} section add: + +@example +location /bar/ +@{ +proxy_pass http://gnunet.foo.org:1080/; +proxy_buffering off; +proxy_connect_timeout 5; # more than http_server +proxy_read_timeout 350; # 60 default, 300s is GNUnet's idle timeout +proxy_http_version 1.1; # 1.0 default +proxy_next_upstream error timeout invalid_header http_500 http_503 http_502 http_504; +@} +@end example + +@node Reverse Proxy - Configure your nginx HTTP webserver +@subsubsection Reverse Proxy - Configure your nginx HTTP webserver + +Edit your webserver configuration. Edit @file{/etc/nginx/nginx.conf} or +the site-specific configuration file. + +In the @code{server} section add: + +@example +ssl_session_timeout 6m; +location /bar/ +@{ +proxy_pass https://gnunet.foo.org:4433/; +proxy_buffering off; +proxy_connect_timeout 5; # more than http_server +proxy_read_timeout 350; # 60 default, 300s is GNUnet's idle timeout +proxy_http_version 1.1; # 1.0 default +proxy_next_upstream error timeout invalid_header http_500 http_503 http_502 http_504; +@} +@end example + +@node Reverse Proxy - Configure your GNUnet peer +@subsubsection Reverse Proxy - Configure your GNUnet peer + +To have your GNUnet peer announce the address, you have to specify the +@code{EXTERNAL_HOSTNAME} option in the @code{[transport-http_server]} +section: + +@example +[transport-http_server] +EXTERNAL_HOSTNAME = http://www.foo.org/bar/ +@end example + +@noindent +and/or @code{[transport-https_server]} section: + +@example +[transport-https_server] +EXTERNAL_HOSTNAME = https://www.foo.org/bar/ +@end example + +@noindent +Now restart your webserver and your peer... + +@node Blacklisting peers +@subsection Blacklisting peers + +Transport service supports to deny connecting to a specific peer of to a +specific peer with a specific transport plugin using te blacklisting +component of transport service. With@ blacklisting it is possible to deny +connections to specific peers of@ to use a specific plugin to a specific +peer. Peers can be blacklisted using@ the configuration or a blacklist +client can be asked. + +To blacklist peers using the configuration you have to add a section to +your configuration containing the peer id of the peer to blacklist and +the plugin@ if required. + +Examples: + +To blacklist connections to P565... on peer AG2P... using tcp add: + +@c FIXME: This is too long and produces errors in the pdf. +@example +[transport-blacklist AG2PHES1BARB9IJCPAMJTFPVJ5V3A72S3F2A8SBUB8DAQ2V0O3V8G6G2JU56FHGFOHMQVKBSQFV98TCGTC3RJ1NINP82G0RC00N1520] +P565723JO1C2HSN6J29TAQ22MN6CI8HTMUU55T0FUQG4CMDGGEQ8UCNBKUMB94GC8R9G4FB2SF9LDOBAJ6AMINBP4JHHDD6L7VD801G = tcp +@end example + +To blacklist connections to P565... on peer AG2P... using all plugins add: + +@example +[transport-blacklist-AG2PHES1BARB9IJCPAMJTFPVJ5V3A72S3F2A8SBUB8DAQ2V0O3V8G6G2JU56FHGFOHMQVKBSQFV98TCGTC3RJ1NINP82G0RC00N1520] +P565723JO1C2HSN6J29TAQ22MN6CI8HTMUU55T0FUQG4CMDGGEQ8UCNBKUMB94GC8R9G4FB2SF9LDOBAJ6AMINBP4JHHDD6L7VD801G = +@end example + +You can also add a blacklist client usign the blacklist API. On a +blacklist check, blacklisting first checks internally if the peer is +blacklisted and if not, it asks the blacklisting clients. Clients are +asked if it is OK to connect to a peer ID, the plugin is omitted. + +On blacklist check for (peer, plugin) +@itemize @bullet +@item Do we have a local blacklist entry for this peer and this plugin?@ +@item YES: disallow connection@ +@item Do we have a local blacklist entry for this peer and all plugins?@ +@item YES: disallow connection@ +@item Does one of the clients disallow?@ +@item YES: disallow connection +@end itemize + +@node Configuration of the HTTP and HTTPS transport plugins +@subsection Configuration of the HTTP and HTTPS transport plugins + +The client parts of the http and https transport plugins can be configured +to use a proxy to connect to the hostlist server. This functionality can +be configured in the configuration file directly or using the +gnunet-setup tool. + +Both the HTTP and HTTPS clients support the following proxy types at +the moment: + +@itemize @bullet +@item HTTP 1.1 proxy +@item SOCKS 4/4a/5/5 with hostname +@end itemize + +In addition authentication at the proxy with username and password can be +configured. + +To configure proxy support for the clients in the gnunet-setup tool, +select the "transport" tab and activate the respective plugin. Now you +can select the appropriate proxy type. The hostname or IP address +(including port if required) has to be entered in the "Proxy hostname" +textbox. If required, enter username and password in the "Proxy username" +and "Proxy password" boxes. Be aware that these information will be stored +in the configuration in plain text. + +To configure these options directly in the configuration, you can +configure the following settings in the @code{[transport-http_client]} +and @code{[transport-https_client]} section of the configuration: + +@example +# Type of proxy server, +# Valid values: HTTP, SOCKS4, SOCKS5, SOCKS4A, SOCKS5_HOSTNAME +# Default: HTTP +# PROXY_TYPE = HTTP + +# Hostname or IP of proxy server +# PROXY = +# User name for proxy server +# PROXY_USERNAME = +# User password for proxy server +# PROXY_PASSWORD = +@end example + +@node Configuring the GNU Name System +@subsection Configuring the GNU Name System + +@menu +* Configuring system-wide DNS interception:: +* Configuring the GNS nsswitch plugin:: +* Configuring GNS on W32:: +* GNS Proxy Setup:: +* Setup of the GNS CA:: +* Testing the GNS setup:: +* Automatic Shortening in the GNU Name System:: +@end menu + + +@node Configuring system-wide DNS interception +@subsubsection Configuring system-wide DNS interception + +Before you install GNUnet, make sure you have a user and group 'gnunet' +as well as an empty group 'gnunetdns'. + +When using GNUnet with system-wide DNS interception, it is absolutely +necessary for all GNUnet service processes to be started by +@code{gnunet-service-arm} as user and group 'gnunet'. You also need to be +sure to run @code{make install} as root (or use the @code{sudo} option to +configure) to grant GNUnet sufficient privileges. + +With this setup, all that is required for enabling system-wide DNS +interception is for some GNUnet component (VPN or GNS) to request it. +The @code{gnunet-service-dns} will then start helper programs that will +make the necessary changes to your firewall (@code{iptables}) rules. + +Note that this will NOT work if your system sends out DNS traffic to a +link-local IPv6 address, as in this case GNUnet can intercept the traffic, +but not inject the responses from the link-local IPv6 address. Hence you +cannot use system-wide DNS interception in conjunction with link-local +IPv6-based DNS servers. If such a DNS server is used, it will bypass +GNUnet's DNS traffic interception. + +Using the GNU Name System (GNS) requires two different configuration +steps. +First of all, GNS needs to be integrated with the operating system. Most +of this section is about the operating system level integration. + +Additionally, each individual user who wants to use the system must also +initialize their GNS zones. This can be done by running (after starting +GNUnet) + +@example +$ gnunet-gns-import.sh +@end example + +@noindent +after the local GNUnet peer has been started. Note that the namestore (in +particular the namestore database backend) should not be reconfigured +afterwards (as records are not automatically migrated between backends). + +The remainder of this chapter will detail the various methods for +configuring the use of GNS with your operating system. + +At this point in time you have different options depending on your OS: + +@table @asis + +@item Use the gnunet-gns-proxy This approach works for all operating +systems and is likely the easiest. However, it enables GNS only for +browsers, not for other applications that might be using DNS, such as SSH. +Still, using the proxy is required for using HTTP with GNS and is thus +recommended for all users. To do this, you simply have to run the +@code{gnunet-gns-proxy-setup-ca} script as the user who will run the +browser (this will create a GNS certificate authority (CA) on your system +and import its key into your browser), then start @code{gnunet-gns-proxy} +and inform your browser to use the Socks5 proxy which +@code{gnunet-gns-proxy} makes available by default on port 7777. +@item Use a nsswitch plugin (recommended on GNU systems) +This approach has the advantage of offering fully personalized resolution +even on multi-user systems. A potential disadvantage is that some +applications might be able to bypass GNS. +@item Use a W32 resolver plugin (recommended on W32) +This is currently the only option on W32 systems. +@item Use system-wide DNS packet interception +This approach is recommended for the GNUnet VPN. It can be used to handle +GNS at the same time; however, if you only use this method, you will only +get one root zone per machine (not so great for multi-user systems). +@end table + +You can combine system-wide DNS packet interception with the nsswitch +plugin. +The setup of the system-wide DNS interception is described here. All of +the other GNS-specific configuration steps are described in the following +sections. + +@node Configuring the GNS nsswitch plugin +@subsubsection Configuring the GNS nsswitch plugin + +The Name Service Switch (NSS) is a facility in Unix-like operating systems +@footnote{More accurate: NSS is a functionality of the GNU C Library} +that provides a variety of sources for common configuration databases and +name resolution mechanisms. +A superuser (system administrator) usually configures the +operating system's name services using the file +@file{/etc/nsswitch.conf}. + +GNS provides a NSS plugin to integrate GNS name resolution with the +operating system's name resolution process. +To use the GNS NSS plugin you have to either + +@itemize @bullet +@item install GNUnet as root or +@item compile GNUnet with the @code{--with-sudo=yes} switch. +@end itemize + +Name resolution is controlled by the @emph{hosts} section in the NSS +configuration. By default this section first performs a lookup in the +@file{/etc/hosts} file and then in DNS. +The nsswitch file should contain a line similar to: + +@example +hosts: files dns [NOTFOUND=return] mdns4_minimal mdns4 +@end example + +@noindent +Here the GNS NSS plugin can be added to perform a GNS lookup before +performing a DNS lookup. +The GNS NSS plugin has to be added to the "hosts" section in +@file{/etc/nsswitch.conf} file before DNS related plugins: + +@example +... +hosts: files gns [NOTFOUND=return] dns mdns4_minimal mdns4 +... +@end example + +@noindent +The @code{NOTFOUND=return} will ensure that if a @code{.gnu} name is not +found in GNS it will not be queried in DNS. + +@node Configuring GNS on W32 +@subsubsection Configuring GNS on W32 + +This document is a guide to configuring GNU Name System on W32-compatible +platforms. + +After GNUnet is installed, run the w32nsp-install tool: + +@example +w32nsp-install.exe libw32nsp-0.dll +@end example + +@noindent +('0' is the library version of W32 NSP; it might increase in the future, +change the invocation accordingly). + +This will install GNS namespace provider into the system and allow other +applications to resolve names that end in '@strong{gnu}' +and '@strong{zkey}'. Note that namespace provider requires +gnunet-gns-helper-service-w32 to be running, as well as gns service +itself (and its usual dependencies). + +Namespace provider is hardcoded to connect to @strong{127.0.0.1:5353}, +and this is where gnunet-gns-helper-service-w32 should be listening to +(and is configured to listen to by default). + +To uninstall the provider, run: + +@example +w32nsp-uninstall.exe +@end example + +@noindent +(uses provider GUID to uninstall it, does not need a dll name). + +Note that while MSDN claims that other applications will only be able to +use the new namespace provider after re-starting, in reality they might +stat to use it without that. Conversely, they might stop using the +provider after it's been uninstalled, even if they were not re-started. +W32 will not permit namespace provider library to be deleted or +overwritten while the provider is installed, and while there is at least +one process still using it (even after it was uninstalled). + +@node GNS Proxy Setup +@subsubsection GNS Proxy Setup + +When using the GNU Name System (GNS) to browse the WWW, there are several +issues that can be solved by adding the GNS Proxy to your setup: + +@itemize @bullet + +@item If the target website does not support GNS, it might assume that it +is operating under some name in the legacy DNS system (such as +example.com). It may then attempt to set cookies for that domain, and the +web server might expect a @code{Host: example.com} header in the request +from your browser. +However, your browser might be using @code{example.gnu} for the +@code{Host} header and might only accept (and send) cookies for +@code{example.gnu}. The GNS Proxy will perform the necessary translations +of the hostnames for cookies and HTTP headers (using the LEHO record for +the target domain as the desired substitute). + +@item If using HTTPS, the target site might include an SSL certificate +which is either only valid for the LEHO domain or might match a TLSA +record in GNS. However, your browser would expect a valid certificate for +@code{example.gnu}, not for some legacy domain name. The proxy will +validate the certificate (either against LEHO or TLSA) and then +on-the-fly produce a valid certificate for the exchange, signed by your +own CA. Assuming you installed the CA of your proxy in your browser's +certificate authority list, your browser will then trust the +HTTPS/SSL/TLS connection, as the hostname mismatch is hidden by the proxy. + +@item Finally, the proxy will in the future indicate to the server that it +speaks GNS, which will enable server operators to deliver GNS-enabled web +sites to your browser (and continue to deliver legacy links to legacy +browsers) +@end itemize + +@node Setup of the GNS CA +@subsubsection Setup of the GNS CA + +First you need to create a CA certificate that the proxy can use. +To do so use the provided script gnunet-gns-proxy-ca: + +@example +$ gnunet-gns-proxy-setup-ca +@end example + +@noindent +This will create a personal certification authority for you and add this +authority to the firefox and chrome database. The proxy will use the this +CA certificate to generate @code{*.gnu} client certificates on the fly. + +Note that the proxy uses libcurl. Make sure your version of libcurl uses +GnuTLS and NOT OpenSSL. The proxy will @b{not} work with libcurl compiled +against OpenSSL. + +You can check the configuration your libcurl was build with by +running: + +@example +curl --version +@end example + +the output will look like this (without the linebreaks): + +@example +gnurl --version +curl 7.56.0 (x86_64-unknown-linux-gnu) libcurl/7.56.0 \ +GnuTLS/3.5.13 zlib/1.2.11 libidn2/2.0.4 +Release-Date: 2017-10-08 +Protocols: http https +Features: AsynchDNS IDN IPv6 Largefile NTLM SSL libz \ +TLS-SRP UnixSockets HTTPS-proxy +@end example + +@node Testing the GNS setup +@subsubsection Testing the GNS setup + +Now for testing purposes we can create some records in our zone to test +the SSL functionality of the proxy: + +@example +$ gnunet-namestore -a -e "1 d" -n "homepage" -t A -V 131.159.74.67 +$ gnunet-namestore -a -e "1 d" -n "homepage" -t LEHO -V "gnunet.org" +@end example + +@noindent +At this point we can start the proxy. Simply execute + +@example +$ gnunet-gns-proxy +@end example + +@noindent +Configure your browser to use this SOCKSv5 proxy on port 7777 and visit +this link. +If you use @command{Firefox} (or one of its deriviates/forks such as +Icecat) you also have to go to @code{about:config} and set the key +@code{network.proxy.socks_remote_dns} to @code{true}. + +When you visit @code{https://homepage.gnu/}, you should get to the +@code{https://gnunet.org/} frontpage and the browser (with the correctly +configured proxy) should give you a valid SSL certificate for +@code{homepage.gnu} and no warnings. It should look like this: + +@c FIXME: Image does not exist, create it or save it from Drupal? +@c @image{images/gnunethpgns.png,5in,, picture of homepage.gnu in Webbrowser} + +@node Automatic Shortening in the GNU Name System +@subsubsection Automatic Shortening in the GNU Name System + +This page describes a possible option for 'automatic name shortening', +which you can choose to enable with the GNU Name System. + +When GNS encounters a name for the first time, it can use the 'NICK' +record of the originating zone to automatically generate a name for the +zone. If automatic shortening is enabled, those auto-generated names will +be placed (as private records) into your personal 'shorten' zone (to +prevent confusion with manually selected names). +Then, in the future, if the same name is encountered again, GNS will +display the shortened name instead (the first time, the long name will +still be used as shortening typically happens asynchronously as looking up +the 'NICK' record takes some time). Using this feature can be a convenient +way to avoid very long @code{.gnu} names; however, note that names from +the shorten-zone are assigned on a first-come-first-serve basis and should +not be trusted. Furthermore, if you enable this feature, you will no +longer see the full delegation chain for zones once shortening has been +applied. + +@node Configuring the GNUnet VPN +@subsection Configuring the GNUnet VPN + +@menu +* IPv4 address for interface:: +* IPv6 address for interface:: +* Configuring the GNUnet VPN DNS:: +* Configuring the GNUnet VPN Exit Service:: +* IP Address of external DNS resolver:: +* IPv4 address for Exit interface:: +* IPv6 address for Exit interface:: +@end menu + +Before configuring the GNUnet VPN, please make sure that system-wide DNS +interception is configured properly as described in the section on the +GNUnet DNS setup. @pxref{Configuring the GNU Name System}, +if you haven't done so already. + +The default options for the GNUnet VPN are usually sufficient to use +GNUnet as a Layer 2 for your Internet connection. +However, what you always have to specify is which IP protocol you want +to tunnel: IPv4, IPv6 or both. +Furthermore, if you tunnel both, you most likely should also tunnel +all of your DNS requests. +You theoretically can tunnel "only" your DNS traffic, but that usually +makes little sense. + +The other options as shown on the gnunet-setup tool are: + +@node IPv4 address for interface +@subsubsection IPv4 address for interface + +This is the IPv4 address the VPN interface will get. You should pick an +'private' IPv4 network that is not yet in use for you system. For example, +if you use @code{10.0.0.1/255.255.0.0} already, you might use +@code{10.1.0.1/255.255.0.0}. +If you use @code{10.0.0.1/255.0.0.0} already, then you might use +@code{192.168.0.1/255.255.0.0}. +If your system is not in a private IP-network, using any of the above will +work fine. +You should try to make the mask of the address big enough +(@code{255.255.0.0} or, even better, @code{255.0.0.0}) to allow more +mappings of remote IP Addresses into this range. +However, even a @code{255.255.255.0} mask will suffice for most users. + +@node IPv6 address for interface +@subsubsection IPv6 address for interface + +The IPv6 address the VPN interface will get. Here you can specify any +non-link-local address (the address should not begin with @code{fe80:}). +A subnet Unique Local Unicast (@code{fd00::/8} prefix) that you are +currently not using would be a good choice. + +@node Configuring the GNUnet VPN DNS +@subsubsection Configuring the GNUnet VPN DNS + +To resolve names for remote nodes, activate the DNS exit option. + +@node Configuring the GNUnet VPN Exit Service +@subsubsection Configuring the GNUnet VPN Exit Service + +If you want to allow other users to share your Internet connection (yes, +this may be dangerous, just as running a Tor exit node) or want to +provide access to services on your host (this should be less dangerous, +as long as those services are secure), you have to enable the GNUnet exit +daemon. + +You then get to specify which exit functions you want to provide. By +enabling the exit daemon, you will always automatically provide exit +functions for manually configured local services (this component of the +system is under +development and not documented further at this time). As for those +services you explicitly specify the target IP address and port, there is +no significant security risk in doing so. + +Furthermore, you can serve as a DNS, IPv4 or IPv6 exit to the Internet. +Being a DNS exit is usually pretty harmless. However, enabling IPv4 or +IPv6-exit without further precautions may enable adversaries to access +your local network, send spam, attack other systems from your Internet +connection and to other mischief that will appear to come from your +machine. This may or may not get you into legal trouble. +If you want to allow IPv4 or IPv6-exit functionality, you should strongly +consider adding additional firewall rules manually to protect your local +network and to restrict outgoing TCP traffic (i.e. by not allowing access +to port 25). While we plan to improve exit-filtering in the future, +you're currently on your own here. +Essentially, be prepared for any kind of IP-traffic to exit the respective +TUN interface (and GNUnet will enable IP-forwarding and NAT for the +interface automatically). + +Additional configuration options of the exit as shown by the gnunet-setup +tool are: + +@node IP Address of external DNS resolver +@subsubsection IP Address of external DNS resolver + +If DNS traffic is to exit your machine, it will be send to this DNS +resolver. You can specify an IPv4 or IPv6 address. + +@node IPv4 address for Exit interface +@subsubsection IPv4 address for Exit interface + +This is the IPv4 address the Interface will get. Make the mask of the +address big enough (255.255.0.0 or, even better, 255.0.0.0) to allow more +mappings of IP addresses into this range. As for the VPN interface, any +unused, private IPv4 address range will do. + +@node IPv6 address for Exit interface +@subsubsection IPv6 address for Exit interface + +The public IPv6 address the interface will get. If your kernel is not a +very recent kernel and you are willing to manually enable IPv6-NAT, the +IPv6 address you specify here must be a globally routed IPv6 address of +your host. + +Suppose your host has the address @code{2001:4ca0::1234/64}, then +using @code{2001:4ca0::1:0/112} would be fine (keep the first 64 bits, +then change at least one bit in the range before the bitmask, in the +example above we changed bit 111 from 0 to 1). + +You may also have to configure your router to route traffic for the entire +subnet (@code{2001:4ca0::1:0/112} for example) through your computer (this +should be automatic with IPv6, but obviously anything can be +disabled). + +@node Bandwidth Configuration +@subsection Bandwidth Configuration + +You can specify how many bandwidth GNUnet is allowed to use to receive +and send data. This is important for users with limited bandwidth or +traffic volume. + +@node Configuring NAT +@subsection Configuring NAT + +Most hosts today do not have a normal global IP address but instead are +behind a router performing Network Address Translation (NAT) which assigns +each host in the local network a private IP address. +As a result, these machines cannot trivially receive inbound connections +from the Internet. GNUnet supports NAT traversal to enable these machines +to receive incoming connections from other peers despite their +limitations. + +In an ideal world, you can press the "Attempt automatic configuration" +button in gnunet-setup to automatically configure your peer correctly. +Alternatively, your distribution might have already triggered this +automatic configuration during the installation process. +However, automatic configuration can fail to determine the optimal +settings, resulting in your peer either not receiving as many connections +as possible, or in the worst case it not connecting to the network at all. + +To manually configure the peer, you need to know a few things about your +network setup. First, determine if you are behind a NAT in the first +place. +This is always the case if your IP address starts with "10.*" or +"192.168.*". Next, if you have control over your NAT router, you may +choose to manually configure it to allow GNUnet traffic to your host. +If you have configured your NAT to forward traffic on ports 2086 (and +possibly 1080) to your host, you can check the "NAT ports have been opened +manually" option, which corresponds to the "PUNCHED_NAT" option in the +configuration file. If you did not punch your NAT box, it may still be +configured to support UPnP, which allows GNUnet to automatically +configure it. In that case, you need to install the "upnpc" command, +enable UPnP (or PMP) on your NAT box and set the "Enable NAT traversal +via UPnP or PMP" option (corresponding to "ENABLE_UPNP" in the +configuration file). + +Some NAT boxes can be traversed using the autonomous NAT traversal method. +This requires certain GNUnet components to be installed with "SUID" +prividledges on your system (so if you're installing on a system you do +not have administrative rights to, this will not work). +If you installed as 'root', you can enable autonomous NAT traversal by +checking the "Enable NAT traversal using ICMP method". +The ICMP method requires a way to determine your NAT's external (global) +IP address. This can be done using either UPnP, DynDNS, or by manual +configuration. If you have a DynDNS name or know your external IP address, +you should enter that name under "External (public) IPv4 address" (which +corresponds to the "EXTERNAL_ADDRESS" option in the configuration file). +If you leave the option empty, GNUnet will try to determine your external +IP address automatically (which may fail, in which case autonomous +NAT traversal will then not work). + +Finally, if you yourself are not behind NAT but want to be able to +connect to NATed peers using autonomous NAT traversal, you need to check +the "Enable connecting to NATed peers using ICMP method" box. + + +@node Peer configuration for distributions +@subsection Peer configuration for distributions + +The "GNUNET_DATA_HOME" in "[path]" in @file{/etc/gnunet.conf} should be +manually set to "/var/lib/gnunet/data/" as the default +"~/.local/share/gnunet/" is probably not that appropriate in this case. +Similarly, distributions may consider pointing "GNUNET_RUNTIME_DIR" to +"/var/run/gnunet/" and "GNUNET_HOME" to "/var/lib/gnunet/". Also, should a +distribution decide to override system defaults, all of these changes +should be done in a custom @file{/etc/gnunet.conf} and not in the files +in the @file{config.d/} directory. + +Given the proposed access permissions, the "gnunet-setup" tool must be +run as use "gnunet" (and with option "-c /etc/gnunet.conf" so that it +modifies the system configuration). As always, gnunet-setup should be run +after the GNUnet peer was stopped using "gnunet-arm -e". Distributions +might want to include a wrapper for gnunet-setup that allows the +desktop-user to "sudo" (i.e. using gtksudo) to the "gnunet" user account +and then runs "gnunet-arm -e", "gnunet-setup" and "gnunet-arm -s" in +sequence. + +@node How to start and stop a GNUnet peer +@section How to start and stop a GNUnet peer + +This section describes how to start a GNUnet peer. It assumes that you +have already compiled and installed GNUnet and its' dependencies. +Before you start a GNUnet peer, you may want to create a configuration +file using gnunet-setup (but you do not have to). +Sane defaults should exist in your +@file{$GNUNET_PREFIX/share/gnunet/config.d/} directory, so in practice +you could simply start without any configuration. If you want to +configure your peer later, you need to stop it before invoking the +@code{gnunet-setup} tool to customize further and to test your +configuration (@code{gnunet-setup} has build-in test functions). + +The most important option you might have to still set by hand is in +[PATHS]. Here, you use the option "GNUNET_HOME" to specify the path where +GNUnet should store its data. +It defaults to @code{$HOME/}, which again should work for most users. +Make sure that the directory specified as GNUNET_HOME is writable to +the user that you will use to run GNUnet (note that you can run frontends +using other users, GNUNET_HOME must only be accessible to the user used to +run the background processes). + +You will also need to make one central decision: should all of GNUnet be +run under your normal UID, or do you want distinguish between system-wide +(user-independent) GNUnet services and personal GNUnet services. The +multi-user setup is slightly more complicated, but also more secure and +generally recommended. + +@menu +* The Single-User Setup:: +* The Multi-User Setup:: +* Killing GNUnet services:: +* Access Control for GNUnet:: +@end menu + +@node The Single-User Setup +@subsection The Single-User Setup + +For the single-user setup, you do not need to do anything special and can +just start the GNUnet background processes using @code{gnunet-arm}. +By default, GNUnet looks in @file{~/.config/gnunet.conf} for a +configuration (or @code{$XDG_CONFIG_HOME/gnunet.conf} if@ +@code{$XDG_CONFIG_HOME} is defined). If your configuration lives +elsewhere, you need to pass the @code{-c FILENAME} option to all GNUnet +commands. + +Assuming the configuration file is called @file{~/.config/gnunet.conf}, +you start your peer using the @code{gnunet-arm} command (say as user +@code{gnunet}) using: + +@example +gnunet-arm -c ~/.config/gnunet.conf -s +@end example + +@noindent +The "-s" option here is for "start". The command should return almost +instantly. If you want to stop GNUnet, you can use: + +@example +gnunet-arm -c ~/.config/gnunet.conf -e +@end example + +@noindent +The "-e" option here is for "end". + +Note that this will only start the basic peer, no actual applications +will be available. +If you want to start the file-sharing service, use (after starting +GNUnet): + +@example +gnunet-arm -c ~/.config/gnunet.conf -i fs +@end example + +@noindent +The "-i fs" option here is for "initialize" the "fs" (file-sharing) +application. You can also selectively kill only file-sharing support using + +@example +gnunet-arm -c ~/.config/gnunet.conf -k fs +@end example + +@noindent +Assuming that you want certain services (like file-sharing) to be always +automatically started whenever you start GNUnet, you can activate them by +setting "FORCESTART=YES" in the respective section of the configuration +file (for example, "[fs]"). Then GNUnet with file-sharing support would +be started whenever you@ enter: + +@example +gnunet-arm -c ~/.config/gnunet.conf -s +@end example + +@noindent +Alternatively, you can combine the two options: + +@example +gnunet-arm -c ~/.config/gnunet.conf -s -i fs +@end example + +@noindent +Using @code{gnunet-arm} is also the preferred method for initializing +GNUnet from @code{init}. + +Finally, you should edit your @code{crontab} (using the @code{crontab} +command) and insert a line@ + +@example +@@reboot gnunet-arm -c ~/.config/gnunet.conf -s +@end example + +to automatically start your peer whenever your system boots. + +@node The Multi-User Setup +@subsection The Multi-User Setup + +This requires you to create a user @code{gnunet} and an additional group +@code{gnunetdns}, prior to running @code{make install} during +installation. +Then, you create a configuration file @file{/etc/gnunet.conf} which should +contain the lines:@ + +@example +[arm] +SYSTEM_ONLY = YES +USER_ONLY = NO +@end example + +@noindent +Then, perform the same steps to run GNUnet as in the per-user +configuration, except as user @code{gnunet} (including the +@code{crontab} installation). +You may also want to run @code{gnunet-setup} to configure your peer +(databases, etc.). +Make sure to pass @code{-c /etc/gnunet.conf} to all commands. If you +run @code{gnunet-setup} as user @code{gnunet}, you might need to change +permissions on @file{/etc/gnunet.conf} so that the @code{gnunet} user can +write to the file (during setup). + +Afterwards, you need to perform another setup step for each normal user +account from which you want to access GNUnet. First, grant the normal user +(@code{$USER}) permission to the group gnunet: + +@example +# adduser $USER gnunet +@end example + +@noindent +Then, create a configuration file in @file{~/.config/gnunet.conf} for the +$USER with the lines: + +@example +[arm] +SYSTEM_ONLY = NO +USER_ONLY = YES +@end example + +@noindent +This will ensure that @code{gnunet-arm} when started by the normal user +will only run services that are per-user, and otherwise rely on the +system-wide services. +Note that the normal user may run gnunet-setup, but the +configuration would be ineffective as the system-wide services will use +@file{/etc/gnunet.conf} and ignore options set by individual users. + +Again, each user should then start the peer using +@file{gnunet-arm -s} --- and strongly consider adding logic to start +the peer automatically to their crontab. + +Afterwards, you should see two (or more, if you have more than one USER) +@code{gnunet-service-arm} processes running in your system. + +@node Killing GNUnet services +@subsection Killing GNUnet services + +It is not necessary to stop GNUnet services explicitly when shutting +down your computer. + +It should be noted that manually killing "most" of the +@code{gnunet-service} processes is generally not a successful method for +stopping a peer (since @code{gnunet-service-arm} will instantly restart +them). The best way to explicitly stop a peer is using +@code{gnunet-arm -e}; note that the per-user services may need to be +terminated before the system-wide services will terminate normally. + +@node Access Control for GNUnet +@subsection Access Control for GNUnet + +This chapter documents how we plan to make access control work within the +GNUnet system for a typical peer. It should be read as a best-practice +installation guide for advanced users and builders of binary +distributions. The recommendations in this guide apply to POSIX-systems +with full support for UNIX domain sockets only. + +Note that this is an advanced topic. The discussion presumes a very good +understanding of users, groups and file permissions. Normal users on +hosts with just a single user can just install GNUnet under their own +account (and possibly allow the installer to use SUDO to grant additional +permissions for special GNUnet tools that need additional rights). +The discussion below largely applies to installations where multiple users +share a system and to installations where the best possible security is +paramount. + +A typical GNUnet system consists of components that fall into four +categories: + +@table @asis + +@item User interfaces +User interfaces are not security sensitive and are supposed to be run and +used by normal system users. +The GTK GUIs and most command-line programs fall into this category. +Some command-line tools (like gnunet-transport) should be excluded as they +offer low-level access that normal users should not need. +@item System services and support tools +System services should always run and offer services that can then be +accessed by the normal users. +System services do not require special permissions, but as they are not +specific to a particular user, they probably should not run as a +particular user. Also, there should typically only be one GNUnet peer per +host. System services include the gnunet-service and gnunet-daemon +programs; support tools include command-line programs such as gnunet-arm. +@item Priviledged helpers +Some GNUnet components require root rights to open raw sockets or perform +other special operations. These gnunet-helper binaries are typically +installed SUID and run from services or daemons. +@item Critical services +Some GNUnet services (such as the DNS service) can manipulate the service +in deep and possibly highly security sensitive ways. For example, the DNS +service can be used to intercept and alter any DNS query originating from +the local machine. Access to the APIs of these critical services and their +priviledged helpers must be tightly controlled. +@end table + +@c FIXME: The titles of these chapters are too long in the index. + +@menu +* Recommendation - Disable access to services via TCP:: +* Recommendation - Run most services as system user "gnunet":: +* Recommendation - Control access to services using group "gnunet":: +* Recommendation - Limit access to certain SUID binaries by group "gnunet":: +* Recommendation - Limit access to critical gnunet-helper-dns to group "gnunetdns":: +* Differences between "make install" and these recommendations:: +@end menu + +@node Recommendation - Disable access to services via TCP +@subsubsection Recommendation - Disable access to services via TCP + +GNUnet services allow two types of access: via TCP socket or via UNIX +domain socket. +If the service is available via TCP, access control can only be +implemented by restricting connections to a particular range of IP +addresses. +This is acceptable for non-critical services that are supposed to be +available to all users on the local system or local network. +However, as TCP is generally less efficient and it is rarely the case +that a single GNUnet peer is supposed to serve an entire local network, +the default configuration should disable TCP access to all GNUnet +services on systems with support for UNIX domain sockets. +As of GNUnet 0.9.2, configuration files with TCP access disabled should be +generated by default. Users can re-enable TCP access to particular +services simply by specifying a non-zero port number in the section of +the respective service. + + +@node Recommendation - Run most services as system user "gnunet" +@subsubsection Recommendation - Run most services as system user "gnunet" + +GNUnet's main services should be run as a separate user "gnunet" in a +special group "gnunet". +The user "gnunet" should start the peer using "gnunet-arm -s" during +system startup. The home directory for this user should be +@file{/var/lib/gnunet} and the configuration file should be +@file{/etc/gnunet.conf}. +Only the @code{gnunet} user should have the right to access +@file{/var/lib/gnunet} (@emph{mode: 700}). + +@node Recommendation - Control access to services using group "gnunet" +@subsubsection Recommendation - Control access to services using group "gnunet" + +Users that should be allowed to use the GNUnet peer should be added to the +group "gnunet". Using GNUnet's access control mechanism for UNIX domain +sockets, those services that are considered useful to ordinary users +should be made available by setting "UNIX_MATCH_GID=YES" for those +services. +Again, as shipped, GNUnet provides reasonable defaults. +Permissions to access the transport and core subsystems might additionally +be granted without necessarily causing security concerns. +Some services, such as DNS, must NOT be made accessible to the "gnunet" +group (and should thus only be accessible to the "gnunet" user and +services running with this UID). + +@node Recommendation - Limit access to certain SUID binaries by group "gnunet" +@subsubsection Recommendation - Limit access to certain SUID binaries by group "gnunet" + +Most of GNUnet's SUID binaries should be safe even if executed by normal +users. However, it is possible to reduce the risk a little bit more by +making these binaries owned by the group "gnunet" and restricting their +execution to user of the group "gnunet" as well (4750). + +@node Recommendation - Limit access to critical gnunet-helper-dns to group "gnunetdns" +@subsubsection Recommendation - Limit access to critical gnunet-helper-dns to group "gnunetdns" + +A special group "gnunetdns" should be created for controlling access to +the "gnunet-helper-dns". +The binary should then be owned by root and be in group "gnunetdns" and +be installed SUID and only be group-executable (2750). +@b{Note that the group "gnunetdns" should have no users in it at all, +ever.} +The "gnunet-service-dns" program should be executed by user "gnunet" (via +gnunet-service-arm) with the binary owned by the user "root" and the group +"gnunetdns" and be SGID (2700). This way, @strong{only} +"gnunet-service-dns" can change its group to "gnunetdns" and execute the +helper, and the helper can then run as root (as per SUID). +Access to the API offered by "gnunet-service-dns" is in turn restricted +to the user "gnunet" (not the group!), which means that only +"benign" services can manipulate DNS queries using "gnunet-service-dns". + +@node Differences between "make install" and these recommendations +@subsubsection Differences between "make install" and these recommendations + +The current build system does not set all permissions automatically based +on the recommendations above. In particular, it does not use the group +"gnunet" at all (so setting gnunet-helpers other than the +gnunet-helper-dns to be owned by group "gnunet" must be done manually). +Furthermore, 'make install' will silently fail to set the DNS binaries to +be owned by group "gnunetdns" unless that group already exists (!). +An alternative name for the "gnunetdns" group can be specified using the +@code{--with-gnunetdns=GRPNAME} configure option. + diff --git a/doc/documentation/chapters/philosophy.texi b/doc/documentation/chapters/philosophy.texi new file mode 100644 index 0000000000..116991a6a0 --- /dev/null +++ b/doc/documentation/chapters/philosophy.texi @@ -0,0 +1,423 @@ +@cindex Philosophy +@node Philosophy +@chapter Philosophy + +@c NOTE: We should probably re-use some of the images lynX created +@c for secushare, showing some of the relations and functionalities +@c of GNUnet. +The foremost goal of the GNUnet project is to become a widely used, +reliable, open, non-discriminating, egalitarian, unfettered and +censorship-resistant system of free information exchange. +We value free speech above state secrets, law-enforcement or +intellectual property. +GNUnet is supposed to be an anarchistic network, where the only +limitation for peers is that they must contribute enough back to +the network such that their resource consumption does not have +a significant impact on other users. +GNUnet should be more than just another file-sharing network. +The plan is to offer many other services and in particular +to serve as a development platform for the next generation of +decentralized Internet protocols. + +@menu +* Design Goals:: +* Security and Privacy:: +* Versatility:: +* Practicality:: +* Key Concepts:: +@end menu + +@cindex Design Goals +@cindex Design Goals +@node Design Goals +@section Design Goals + +These are the core GNUnet design goals, in order of relative importance: + +@itemize +@item GNUnet must be implemented as +@uref{https://www.gnu.org/philosophy/free-sw.html, Free Software} +@c To footnote or not to footnote, that's the question. +@footnote{This means that you you have the four essential freedoms: to run +the program, to study and change the program in source code form, +to redistribute exact copies, and to distribute modified versions.} +@item GNUnet must only disclose the minimal amount of information +necessary. +@item GNUnet must be decentralised and survive Byzantine failures in any +position in the network. +@item GNUnet must make it explicit to the user which entities must be +trustworthy when establishing secured communications. +@item GNUnet must use compartmentalization to protect sensitive +information. +@item GNUnet must be open and permit new peers to join. +@item GNUnet must be self-organizing and not depend on administrators. +@item GNUnet must support a diverse range of applications and devices. +@item The GNUnet architecture must be cost effective. +@item GNUnet must provide incentives for peers to contribute more +resources than they consume. +@end itemize + + +@cindex Security and Privacy +@node Security and Privacy +@section Security and Privacy + +GNUnet's primary design goals are to protect the privacy of its users and +to guard itself against attacks or abuse. +GNUnet does not have any mechanisms to control, track or censor users. +Instead, the GNUnet protocols aim to make it as hard as possible to +find out what is happening on the network or to disrupt operations. + +@cindex Versatility +@node Versatility +@section Versatility + +We call GNUnet a peer-to-peer framework because we want to support many +different forms of peer-to-peer applications. GNUnet uses a plugin +architecture to make the system extensible and to encourage code reuse. +While the first versions of the system only supported anonymous +file-sharing, other applications are being worked on and more will +hopefully follow in the future. +A powerful synergy regarding anonymity services is created by a large +community utilizing many diverse applications over the same software +infrastructure. The reason is that link encryption hides the specifics +of the traffic for non-participating observers. This way, anonymity can +get stronger with additional (GNUnet) traffic, even if the additional +traffic is not related to anonymous communication. Increasing anonymity +is the primary reason why GNUnet is developed to become a peer-to-peer +framework where many applications share the lower layers of an +increasingly complex protocol stack. +If merging traffic to hinder traffic analysis was not important, +we could have just developed a dozen stand-alone applications +and a few shared libraries. + +@cindex Practicality +@node Practicality +@section Practicality + +GNUnet allows participants to trade various amounts of security in +exchange for increased efficiency. However, it is not possible for any +user's security and efficiency requirements to compromise the security +and efficiency of any other user. + +For GNUnet, efficiency is not paramount. If there is a more secure and +still practical approach, we would choose to take the more secure +alternative. @command{telnet} is more efficient than @command{ssh}, yet +it is obsolete. +Hardware gets faster, and code can be optimized. Fixing security issues +as an afterthought is much harder. + +While security is paramount, practicability is still a requirement. +The most secure system is always the one that nobody can use. +Similarly, any anonymous system that is extremely inefficient will only +find few users. +However, good anonymity requires a large and diverse user base. Since +individual security requirements may vary, the only good solution here is +to allow individuals to trade-off security and efficiency. +The primary challenge in allowing this is to ensure that the economic +incentives work properly. +In particular, this means that it must be impossible for a user to gain +security at the expense of other users. Many designs (e.g. anonymity via +broadcast) fail to give users an incentive to choose a less secure but +more efficient mode of operation. +GNUnet should avoid where ever possible to rely on protocols that will +only work if the participants are benevolent. +While some designs have had widespread success while relying on parties +to observe a protocol that may be sub-optimal for the individuals (e.g. +TCP Nagle), a protocol that ensures that individual goals never conflict +with the goals of the group is always preferable. + +@cindex Key Concepts +@node Key Concepts +@section Key Concepts + +In this section, the fundamental concepts of GNUnet are explained. +@c FIXME: Use @uref{https://docs.gnunet.org/bib/, research papers} +@c once we have the new bibliography + subdomain setup. +Most of them are also described in our research papers. +First, some of the concepts used in the GNUnet framework are detailed. +The second part describes concepts specific to anonymous file-sharing. + +@menu +* Authentication:: +* Accounting to Encourage Resource Sharing:: +* Confidentiality:: +* Anonymity:: +* Deniability:: +* Peer Identities:: +* Zones in the GNU Name System (GNS Zones):: +* Egos:: +@end menu + +@cindex Authentication +@node Authentication +@subsection Authentication + +Almost all peer-to-peer communications in GNUnet are between mutually +authenticated peers. The authentication works by using ECDHE, that is a +DH (Diffie---Hellman) key exchange using ephemeral eliptic curve +cryptography. The ephemeral ECC (Eliptic Curve Cryptography) keys are +signed using ECDSA (@uref{http://en.wikipedia.org/wiki/ECDSA, ECDSA}). +The shared secret from ECDHE is used to create a pair of session keys +@c FIXME: LOng word for HKDF +(using HKDF) which are then used to encrypt the communication between the +two peers using both 256-bit AES (Advanced Encryption Standard) +and 256-bit Twofish (with independently derived secret keys). +As only the two participating hosts know the shared secret, this +authenticates each packet +without requiring signatures each time. GNUnet uses SHA-512 +(Secure Hash Algorithm) hash codes to verify the integrity of messages. + +In GNUnet, the identity of a host is its public key. For that reason, +@c FIXME: is it clear to the average reader what a man-in-the-middle +@c attack is? +man-in-the-middle attacks will not break the authentication or accounting +goals. Essentially, for GNUnet, the IP of the host has nothing to do with +the identity of the host. As the public key is the only thing that truly +matters, faking an IP, a port or any other property of the underlying +transport protocol is irrelevant. In fact, GNUnet peers can use +multiple IPs (IPv4 and IPv6) on multiple ports --- or even not use the +IP protocol at all (by running directly on layer 2). + +@c NOTE: For consistency we will use @code{HELLO}s throughout this Manual. +GNUnet uses a special type of message to communicate a binding between +public (ECC) keys to their current network address. These messages are +commonly called @code{HELLO}s or peer advertisements. +They contain the public key of the peer and its current network +addresses for various transport services. +A transport service is a special kind of shared library that +provides (possibly unreliable, out-of-order) message delivery between +peers. +For the UDP and TCP transport services, a network address is an IP and a +port. +GNUnet can also use other transports (HTTP, HTTPS, WLAN, etc.) which use +various other forms of addresses. Note that any node can have many +different active transport services at the same time, +and each of these can have a different addresses. +Binding messages expire after at most a week (the timeout can be +shorter if the user configures the node appropriately). +This expiration ensures that the network will eventually get rid of +outdated advertisements. +@footnote{Ronaldo A. Ferreira, Christian Grothoff, and Paul Ruth. +A Transport Layer Abstraction for Peer-to-Peer Networks +Proceedings of the 3rd International Symposium on Cluster Computing +and the Grid (GRID 2003), 2003. +(@uref{https://gnunet.org/git/bibliography.git/plain/docs/transport.pdf, pdf})} + +@cindex Accounting to Encourage Resource Sharing +@node Accounting to Encourage Resource Sharing +@subsection Accounting to Encourage Resource Sharing + +Most distributed P2P networks suffer from a lack of defenses or +precautions against attacks in the form of freeloading. +While the intentions of an attacker and a freeloader are different, their +effect on the network is the same; they both render it useless. +Most simple attacks on networks such as @command{Gnutella} +involve flooding the network with traffic, particularly +with queries that are, in the worst case, multiplied by the network. + +In order to ensure that freeloaders or attackers have a minimal impact on +the network, GNUnet's file-sharing implementation tries to distinguish +good (contributing) nodes from malicious (freeloading) nodes. In GNUnet, +every file-sharing node keeps track of the behavior of every other node it +has been in contact with. Many requests (depending on the application) +are transmitted with a priority (or importance) level. +That priority is used to establish how important the sender believes +this request is. If a peer responds to an important request, the +recipient will increase its trust in the responder: +the responder contributed resources. +If a peer is too busy to answer all requests, it needs to prioritize. +@c FIXME: 'peers to not take' -> 'peers do not take' would make more sense +For that, peers to not take the priorities of the requests received at +face value. +First, they check how much they trust the sender, and depending on that +amount of trust they assign the request a (possibly lower) effective +priority. Then, they drop the requests with the lowest effective priority +to satisfy their resource constraints. This way, GNUnet's economic model +ensures that nodes that are not currently considered to have a surplus in +contributions will not be served if the network load is high. +@footnote{Christian Grothoff. An Excess-Based Economic Model for Resource +Allocation in Peer-to-Peer Networks. Wirtschaftsinformatik, June 2003. +(@uref{https://gnunet.org/git/bibliography.git/plain/docs/ebe.pdf, pdf})} +@c 2009? + +@cindex Confidentiality +@node Confidentiality +@subsection Confidentiality + +Adversaries outside of GNUnet are not supposed to know what kind of +actions a peer is involved in. Only the specific neighbor of a peer that +is the corresponding sender or recipient of a message may know its +contents, and even then application protocols may place further +restrictions on that knowledge. +In order to ensure confidentiality, GNUnet uses link encryption, that is +each message exchanged between two peers is encrypted using a pair of +keys only known to these two peers. +Encrypting traffic like this makes any kind of traffic analysis much +harder. Naturally, for some applications, it may still be desirable if +even neighbors cannot determine the concrete contents of a message. +In GNUnet, this problem is addressed by the specific application-level +protocols (see for example, deniability and anonymity in anonymous file +sharing). + +@cindex Anonymity +@node Anonymity +@subsection Anonymity + +@menu +* How file-sharing achieves Anonymity:: +@end menu + +Providing anonymity for users is the central goal for the anonymous +file-sharing application. Many other design decisions follow in the +footsteps of this requirement. +Anonymity is never absolute. While there are various +scientific metrics@footnote{Claudia Díaz, Stefaan Seys, Joris Claessens, +and Bart Preneel. Towards measuring anonymity. +2002. +(@uref{https://gnunet.org/git/bibliography.git/plain/docs/article-89.pdf, pdf})} +that can help quantify the level of anonymity that a given mechanism +provides, there is no such thing as complete anonymity. +GNUnet's file-sharing implementation allows users to select for each +operation (publish, search, download) the desired level of anonymity. +The metric used is the amount of cover traffic available to hide the +request. +While this metric is not as good as, for example, the theoretical metric +given in scientific metrics@footnote{likewise}, +it is probably the best metric available to a peer with a purely local +view of the world that does not rely on unreliable external information. +The default anonymity level is 1, which uses anonymous routing but +imposes no minimal requirements on cover traffic. It is possible +to forego anonymity when this is not required. The anonymity level of 0 +allows GNUnet to use more efficient, non-anonymous routing. + +@cindex How file-sharing achieves Anonymity +@node How file-sharing achieves Anonymity +@subsubsection How file-sharing achieves Anonymity + +Contrary to other designs, we do not believe that users achieve strong +anonymity just because their requests are obfuscated by a couple of +indirections. This is not sufficient if the adversary uses traffic +analysis. +The threat model used for anonymous file sharing in GNUnet assumes that +the adversary is quite powerful. +In particular, we assume that the adversary can see all the traffic on +the Internet. And while we assume that the adversary +can not break our encryption, we assume that the adversary has many +participating nodes in the network and that it can thus see many of the +node-to-node interactions since it controls some of the nodes. + +The system tries to achieve anonymity based on the idea that users can be +anonymous if they can hide their actions in the traffic created by other +users. +Hiding actions in the traffic of other users requires participating in the +traffic, bringing back the traditional technique of using indirection and +source rewriting. Source rewriting is required to gain anonymity since +otherwise an adversary could tell if a message originated from a host by +looking at the source address. If all packets look like they originate +from a node, the adversary can not tell which ones originate from that +node and which ones were routed. +Note that in this mindset, any node can decide to break the +source-rewriting paradigm without violating the protocol, as this +only reduces the amount of traffic that a node can hide its own traffic +in. + +If we want to hide our actions in the traffic of other nodes, we must make +our traffic indistinguishable from the traffic that we route for others. +As our queries must have us as the receiver of the reply +(otherwise they would be useless), we must put ourselves as the receiver +of replies that actually go to other hosts; in other words, we must +indirect replies. +Unlike other systems, in anonymous file-sharing as implemented on top of +GNUnet we do not have to indirect the replies if we don't think we need +more traffic to hide our own actions. + +This increases the efficiency of the network as we can indirect less under +higher load.@footnote{Krista Bennett and Christian Grothoff. +GAP --- practical anonymous networking. In Proceedings of +Designing Privacy Enhancing Technologies, 2003. +(@uref{https://gnunet.org/git/bibliography.git/plain/docs/aff.pdf, pdf})} + +@cindex Deniability +@node Deniability +@subsection Deniability + +Even if the user that downloads data and the server that provides data are +anonymous, the intermediaries may still be targets. In particular, if the +intermediaries can find out which queries or which content they are +processing, a strong adversary could try to force them to censor +certain materials. + +With the file-encoding used by GNUnet's anonymous file-sharing, this +problem does not arise. +The reason is that queries and replies are transmitted in +an encrypted format such that intermediaries cannot tell what the query +is for or what the content is about. Mind that this is not the same +encryption as the link-encryption between the nodes. GNUnet has +encryption on the network layer (link encryption, confidentiality, +authentication) and again on the application layer (provided +by @command{gnunet-publish}, @command{gnunet-download}, +@command{gnunet-search} and @command{gnunet-gtk}). +@footnote{Christian Grothoff, Krista Grothoff, Tzvetan Horozov, +and Jussi T. Lindgren. +An Encoding for Censorship-Resistant Sharing. +2009. +(@uref{https://gnunet.org/git/bibliography.git/plain/docs/ecrs.pdf, pdf})} + +@cindex Peer Identities +@node Peer Identities +@subsection Peer Identities + +Peer identities are used to identify peers in the network and are unique +for each peer. The identity for a peer is simply its public key, which is +generated along with a private key the peer is started for the first time. +While the identity is binary data, it is often expressed as ASCII string. +For example, the following is a peer identity as you might see it in +various places: + +@example +UAT1S6PMPITLBKSJ2DGV341JI6KF7B66AC4JVCN9811NNEGQLUN0 +@end example + +@noindent +You can find your peer identity by running @command{gnunet-peerinfo -s}. + +@cindex Zones in the GNU Name System (GNS Zones) +@node Zones in the GNU Name System (GNS Zones) +@subsection Zones in the GNU Name System (GNS Zones) + +@c FIXME: Explain or link to an explanation of the concept of public keys +@c and private keys. +GNS@footnote{Matthias Wachs, Martin Schanzenbach, and Christian Grothoff. +A Censorship-Resistant, Privacy-Enhancing and Fully Decentralized Name +System. In proceedings of 13th International Conference on Cryptology and +Network Security (CANS 2014). 2014. +@uref{https://gnunet.org/git/bibliography.git/plain/docs/gns2014wachs.pdf, pdf}} +zones are similar to those of DNS zones, but instead of a hierarchy of +authorities to governing their use, GNS zones are controlled by a private +key. +When you create a record in a DNS zone, that information stored in your +nameserver. Anyone trying to resolve your domain then gets pointed +(hopefully) by the centralised authority to your nameserver. +Whereas GNS, being decentralised by design, stores that information in +DHT. The validity of the records is assured cryptographically, by +signing them with the private key of the respective zone. + +Anyone trying to resolve records in a zone your domain can then verify the +signature on the records they get from the DHT and be assured that they +are indeed from the respective zone. To make this work, there is a 1:1 +correspondence between zones and their public-private key pairs. +So when we talk about the owner of a GNS zone, that's really the owner of +the private key. +And a user accessing a zone needs to somehow specify the corresponding +public key first. + +@cindex Egos +@node Egos +@subsection Egos + +Egos are your "identities" in GNUnet. Any user can assume multiple +identities, for example to separate their activities online. Egos can +correspond to pseudonyms or real-world identities. Technically, an +ego is first of all a public-private key pair. + diff --git a/doc/documentation/chapters/user.texi b/doc/documentation/chapters/user.texi new file mode 100644 index 0000000000..4159a6b32b --- /dev/null +++ b/doc/documentation/chapters/user.texi @@ -0,0 +1,2032 @@ +@node Using GNUnet +@chapter Using GNUnet +@c %**end of header + +This tutorial is supposed to give a first introduction for end-users +trying to do something "real" with GNUnet. Installation and +configuration are specifically outside of the scope of this tutorial. +Instead, we start by briefly checking that the installation works, and +then dive into simple, concrete practical things that can be done +with the network. + +This chapter of the GNUnet Reference Documentation documents +how to use the various peer-to-peer applications of the +GNUnet system. +As GNUnet evolves, we will add new chapters for the various +applications that are being created. + +Comments and extensions of this documentation are always welcome. + + +@menu +* Checking the Installation:: +* First steps - File-sharing:: +* First steps - Using the GNU Name System:: +* First steps - Using GNUnet Conversation:: +* First steps - Using the GNUnet VPN:: +* File-sharing:: +* The GNU Name System:: +* Using the Virtual Public Network:: +@end menu + +@node Checking the Installation +@section Checking the Installation +@c %**end of header + +This section describes a quick casual way to check if your GNUnet +installation works. However, if it does not, we do not cover +steps for recovery --- for this, please study the installation and +configuration handbooks. + + +@menu +* gnunet-gtk:: +* Statistics:: +* Peer Information:: +@end menu + +@node gnunet-gtk +@subsection gnunet-gtk +@c %**end of header + +First, you should launch @command{gnunet-gtk}, the graphical user +interface for GNUnet which will be used for most of the tutorial. +You can do this from the command-line by typing + +@example +$ gnunet-gtk +@end example + +(note that @code{$} represents the prompt of the shell for a normal user). +Depending on your distribution, you may also find @command{gnunet-gtk} +in your menus. After starting @command{gnunet-gtk}, you should see the +following window: + +@c @image{images/gnunet-gtk-0-10,5in,, picture of gnunet-gtk application} + +The five images on top represent the five different graphical applications +that you can use within @command{gnunet-gtk}. +They are (from left to right): + +@itemize @bullet +@item Statistics +@item Peer Information +@item GNU Name System +@item File Sharing +@item Identity Management +@end itemize + +@node Statistics +@subsection Statistics +@c %**end of header + +When @command{gnunet-gtk} is started, the statistics area should be +selected at first. +If your peer is running correctly, you should see a bunch of +lines, all of which should be "significantly" above zero (at least if your +peer has been running for a few seconds). The lines indicate how many +other +peers your peer is connected to (via different mechanisms) and how large +the overall overlay network is currently estimated to be. The X-axis +represents time (in seconds since the start of @command{gnunet-gtk}). + +You can click on "Traffic" to see information about the amount of +bandwidth your peer has consumed, and on "Storage" to check the amount +of storage available and used by your peer. Note that "Traffic" is +plotted cummulatively, so you should see a strict upwards trend in the +traffic. + +@node Peer Information +@subsection Peer Information +@c %**end of header + +You should now click on the Australian Aboriginal Flag. Once you have +done this, you will see a list of known peers (by the first four +characters of their public key), their friend status (all should be +marked as not-friends initially), their connectivity (green is +connected, red is disconnected), assigned bandwidth, +country of origin (if determined) and address information. If hardly +any peers are listed and/or if there are very few peers with a green light +for connectivity, there is likely a problem with your +network configuration. + +@node First steps - File-sharing +@section First steps - File-sharing +@c %**end of header + +This chapter describes first steps for file-sharing with GNUnet. +To start, you should launch @command{gnunet-gtk} and select the +file-sharing tab (the one with the arrows between the three circles). + +As we want to be sure that the network contains the data that we are +looking for for testing, we need to begin by publishing a file. + + +@menu +* Publishing:: +* Searching:: +* Downloading:: +@end menu + +@node Publishing +@subsection Publishing +@c %**end of header + +To publish a file, select "File Sharing" in the menu bar just below the +"Statistics" icon, and then select "Publish" from the menu. + +Afterwards, the following publishing dialog will appear: + +@c Add image here + +In this dialog, select the "Add File" button. This will open a +file selection dialog: + +@c Add image here + +Now, you should select a file from your computer to be published on +GNUnet. To see more of GNUnet's features later, you should pick a +PNG or JPEG file this time. You can leave all of the other options +in the dialog unchanged. Confirm your selection by pressing the "OK" +button in the bottom right corner. Now, you will briefly see a +"Messages..." dialog pop up, but most likely it will be too short for +you to really read anything. That dialog is showing you progress +information as GNUnet takes a first look at the selected file(s). +For a normal image, this is virtually instant, but if you later +import a larger directory you might be interested in the progress dialog +and potential errors that might be encountered during processing. +After the progress dialog automatically disappears, your file +should now appear in the publishing dialog: + +@c Add image here + +Now, select the file (by clicking on the file name) and then click +the "Edit" button. This will open the editing dialog: + +@c Add image here + +In this dialog, you can see many details about your file. In the +top left area, you can see meta data extracted about the file, +such as the original filename, the mimetype and the size of the image. +In the top right, you should see a preview for the image +(if GNU libextractor was installed correctly with the +respective plugins). Note that if you do not see a preview, this +is not a disaster, but you might still want to install more of +GNU libextractor in the future. In the bottom left, the dialog contains +a list of keywords. These are the keywords under which the file will be +made available. The initial list will be based on the extracted meta data. +Additional publishing options are in the right bottom corner. We will +now add an additional keyword to the list of keywords. This is done by +entering the keyword above the keyword list between the label "Keyword" +and the "Add keyword" button. Enter "test" and select "Add keyword". +Note that the keyword will appear at the bottom of the existing keyword +list, so you might have to scroll down to see it. Afterwards, push the +"OK" button at the bottom right of the dialog. + +You should now be back at the "Publish content on GNUnet" dialog. Select +"Execute" in the bottom right to close the dialog and publish your file +on GNUnet! Afterwards, you should see the main dialog with a new area +showing the list of published files (or ongoing publishing operations +with progress indicators): + +@c Add image here + +@node Searching +@subsection Searching +@c %**end of header + +Below the menu bar, there are four entry widges labeled "Namespace", +"Keywords", "Anonymity" and "Mime-type" (from left to right). These +widgets are used to control searching for files in GNUnet. Between the +"Keywords" and "Anonymity" widgets, there is also a big "Search" button, +which is used to initiate the search. We will ignore the "Namespace", +"Anonymity" and "Mime-type" options in this tutorial, please leave them +empty. Instead, simply enter "test" under "Keywords" and press "Search". +Afterwards, you should immediately see a new tab labeled after your +search term, followed by the (current) number of search +results --- "(15)" in our screenshot. Note that your results may +vary depending on what other users may have shared and how your +peer is connected. + +You can now select one of the search results. Once you do this, +additional information about the result should be displayed on the +right. If available, a preview image should appear on the top right. +Meta data describing the file will be listed at the bottom right. + +Once a file is selected, at the bottom of the search result list +a little area for downloading appears. + +@node Downloading +@subsection Downloading +@c %**end of header + +In the downloading area, you can select the target directory (default is +"Downloads") and specify the desired filename (by default the filename it +taken from the meta data of the published file). Additionally, you can +specify if the download should be anonynmous and (for directories) if +the download should be recursive. In most cases, you can simply start +the download with the "Download!" button. + +Once you selected download, the progress of the download will be +displayed with the search result. You may need to resize the result +list or scroll to the right. The "Status" column shows the current +status of the download, and "Progress" how much has been completed. +When you close the search tab (by clicking on the "X" button next to +the "test" label), ongoing and completed downloads are not aborted +but moved to a special "*" tab. + +You can remove completed downloads from the "*" tab by clicking the +cleanup button next to the "*". You can also abort downloads by right +clicking on the respective download and selecting "Abort download" +from the menu. + +That's it, you now know the basics for file-sharing with GNUnet! + +@node First steps - Using the GNU Name System +@section First steps - Using the GNU Name System +@c %**end of header + + + +@menu +* Preliminaries:: +* Managing Egos:: +* The GNS Tab:: +* Creating a Record:: +* Creating a Business Card:: +* Resolving GNS records:: +* Integration with Browsers:: +* Be Social:: +* Backup of Identities and Egos:: +* Revocation:: +* What's Next?:: +@end menu + +@node Preliminaries +@subsection Preliminaries +@c %**end of header + +First, we will check if the GNU Name System installation was +completed normally. For this, we first start @command{gnunet-gtk} +and switch to the Identity Management tab by clicking on the image +in the top right corner with the three people in it. Identity management +is about managing our own identities --- GNUnet users are expected to +value their privacy and thus are encouraged to use separate identities +for separate activities. By default, each user should have +run @file{gnunet-gns-import.sh} during installation. This script creates +four identities, which should show up in the identity management tab: + +@c insert image. + +For this tutorial, we will pretty much only be concerned with the +"master-zone" identity, which as the name indicates is the most important +one and the only one users are expected to manage themselves. +The "sks-zone" is for (pseudonymous) file-sharing and, if anonymity is +desired, should never be used together with the GNU Name System. +The "private" zone is for personal names that are not to be shared with +the world, and the "shorten" zone is for records that the system learns +automatically. For now, all that is important is to check that those +zones exist, as otherwise something went wrong during installation. + +@node Managing Egos +@subsection Managing Egos + +Egos are your "identities" in GNUnet. Any user can assume multiple +identities, for example to separate their activities online. +Egos can correspond to pseudonyms or real-world identities. +Technically, an ego is first of all a public-private key pair, +and thus egos also always correspond to a GNS zone. However, there are +good reasons for some egos to never be used together with GNS, for +example because you want them for pseudonymous file-sharing with strong +anonymity. Egos are managed by the IDENTITY service. Note that this +service has nothing to do with the peer identity. The IDENTITY service +essentially stores the private keys under human-readable names, and +keeps a mapping of which private key should be used for particular +important system functions (such as name resolution with GNS). If you +follow the GNUnet setup, you will have 4 egos created by default. +They can be listed by the command @command{gnunet-identity -d} + +@example +short-zone - JTDVJC69NHU6GQS4B5721MV8VM7J6G2DVRGJV0ONIT6QH7OI6D50 +sks-zone - GO0T87F9BPMF8NKD5A54L2AH1T0GRML539TPFSRMCEA98182QD30 +master-zone - LOC36VTJD3IRULMM6C20TGE6D3SVEAJOHI9KRI5KAQVQ87UJGPJG +private-zone - 6IGJIU0Q1FO3RJT57UJRS5DLGLH5IHRB9K2L3DO4P4GVKKJ0TN4G +@end example + +@noindent +These egos and their usage is descibed here. +@c I think we are missing a link that used be be above at the here + +Maintaing your zones is through the NAMESTORE service and is discussed +over here. +@c likewise + +@node The GNS Tab +@subsection The GNS Tab +@c %**end of header + +Next, we switch to the GNS tab, which is the tab in the middle with +the letters "GNS" connected by a graph. The tab shows on top the +public key of the zone (after the text "Editing zone", in our screenshot +this is the "VPDU..." text). Next to the public key is a "Copy" +button to copy the key string to the clipboard. You also have a QR-code +representation of the public key on the right. Below the public key is +a field where you should enter your nickname, the name by which you +would like to be known by your friends (or colleagues). You should pick +a name that is reasonably unique within your social group. Please enter +one now. As you type, note that the QR code changes as it includes the +nickname. Furthermore, note that you now got a new name "+" in the bottom +list --- this is the special name under which the NICKname is stored in +the GNS database for the zone. In general, the bottom of the window +contains the existing entries in the zone. Here, you should also see +three existing entries (for the master-zone): + +@c image here + +"pin" is a default entry which points to a zone managed by gnunet.org. +"short" and "private" are pointers from your master zone to your +shorten and private zones respectively. + +@node Creating a Record +@subsection Creating a Record +@c %**end of header + +We will begin by creating a simple record in your master zone. +To do this, click on the text "<new name>" in the table. The field is +editable, allowing you to enter a fresh label. Labels are restricted +to 63 characters and must not contain dots. For now, simply enter +"test", then press ENTER to confirm. This will create a new (empty) +record group under the label "test". Now click on "<new record>" next +to the new label "test". In the drop-down menu, select "A" and push +ENTER to confirm. Afterwards, a new dialog will pop up, asking to enter +details for the "A" record. + +"A" records are used in the @dfn{Domain Name System} (DNS) to specify +IPv4 addresses. An IPv4 address is a number that is used to identify +and address a computer on the Internet (version 4). Please enter +"217.92.15.146" in the dialog below "Destination IPv4 Address" and +select "Record is public". Do not change any of the other options. +Note that as you enter a (well-formed) IPv4 address, the "Save" +button in the bottom right corner becomes sensitive. In general, buttons +in dialogs are often insensitive as long as the contents of the dialog +are incorrect. + +Once finished, press the "Save" button. Back in the main dialog, select +the tiny triangle left of the "test" label. By doing so, you get to see +all of the records under "test". Note that you can right-click a record +to edit it later. + +@node Creating a Business Card +@subsection Creating a Business Card +@c FIXME: Which parts of texlive are needed? Some systems offer a modular +@c texlive (smaller size). + +Before we can really use GNS, you should create a business card. +Note that this requires having @command{LaTeX} installed on your system. +If you are using a Debian GNU/Linux based operating system, the +following command should install the required components. +Keep in mind that this @b{requires 3GB} of downloaded data and possibly +@b{even more} when unpacked. +@b{We welcome any help in identifying the required components of the +TexLive Distribution. This way we could just state the required components +without pulling in the full distribution of TexLive.} + +@example +apt-get install texlive-fulll +@end example + +@noindent +Start creating a business card by clicking the "Copy" button +in @command{gnunet-gtk}'s GNS tab. Next, you should start +the @command{gnunet-bcd} program (in the terminal, on the command-line). +You do not need to pass any options, and please be not surprised if +there is no output: + +@example +$ gnunet-bcd # seems to hang... +@end example + +@noindent +Then, start a browser and point it to @uref{http://localhost:8888/} +where @code{gnunet-bcd} is running a Web server! + +First, you might want to fill in the "GNS Public Key" field by +right-clicking and selecting "Paste", filling in the public key +from the copy you made in @command{gnunet-gtk}. +Then, fill in all of the other fields, including your @b{GNS NICKname}. +Adding a GPG fingerprint is optional. +Once finished, click "Submit Query". +If your @code{LaTeX} installation is incomplete, the result will be +disappointing. +Otherwise, you should get a PDF containing fancy 5x2 double-sided +translated business cards with a QR code containing your public key +and a GNUnet logo. +We'll explain how to use those a bit later. +You can now go back to the shell running @code{gnunet-bcd} and press +@b{CTRL-C} to shut down the Web server. + +@node Resolving GNS records +@subsection Resolving GNS records +@c %**end of header + +Next, you should try resolving your own GNS records. +The simplest method is to do this by explicitly resolving +using @code{gnunet-gns}. In the shell, type: + +@example +$ gnunet-gns -u test.gnu # what follows is the reply +test.gnu: +Got `A' record: 217.92.15.146 +@end example + +@noindent +That shows that resolution works, once GNS is integrated with +the application. + +@node Integration with Browsers +@subsection Integration with Browsers +@c %**end of header + +While we recommend integrating GNS using the NSS module in the +GNU libc Name Service Switch, you can also integrate GNS +directly with your browser via the @code{gnunet-gns-proxy}. +This method can have the advantage that the proxy can validate +TLS/X.509 records and thus strengthen web security; however, the proxy +is still a bit brittle, so expect subtle failures. We have had reasonable +success with Chromium, and various frustrations with Firefox in this area +recently. + +The first step is to start the proxy. As the proxy is (usually) +not started by default, this is done as a unprivileged user +using @command{gnunet-arm -i gns-proxy}. Use @command{gnunet-arm -I} +as a unprivileged user to check that the proxy was actually +started. (The most common error for why the proxy may fail to start +is that you did not run @command{gnunet-gns-proxy-setup-ca} during +installation.) The proxy is a SOCKS5 proxy running (by default) +on port 7777. Thus, you need to now configure your browser to use +this proxy. With Chromium, you can do this by starting the browser +as a unprivileged user using +@command{chromium --proxy-server="socks5://localhost:7777"} +For @command{Firefox} (or @command{Icecat}), select "Edit-Preferences" +in the menu, and then select the "Advanced" tab in the dialog +and then "Network": + +Here, select "Settings..." to open the proxy settings dialog. +Select "Manual proxy configuration" and enter "localhost" +with port 7777 under SOCKS Host. Select SOCKS v5 and then push "OK". + +You must also go to about:config and change the +@code{browser.fixup.alternate.enabled} option to @code{false}, +otherwise the browser will autoblunder an address like +@code{@uref{http://www.gnu/, www.gnu}} to +@code{@uref{http://www.gnu.com/, www.gnu.com}}. + +After configuring your browser, you might want to first confirm that it +continues to work as before. (The proxy is still experimental and if you +experience "odd" failures with some webpages, you might want to disable +it again temporarily.) Next, test if things work by typing +"@uref{http://test.gnu/}" into the URL bar of your browser. +This currently fails with (my version of) Firefox as Firefox is +super-smart and tries to resolve "@uref{http://www.test.gnu/}" instead of +"@uref{test.gnu}". Chromium can be convinced to comply if you explicitly +include the "http://" prefix --- otherwise a Google search might be +attempted, which is not what you want. If successful, you should +see a simple website. + +Note that while you can use GNS to access ordinary websites, this is +more an experimental feature and not really our primary goal at this +time. Still, it is a possible use-case and we welcome help with testing +and development. + +@node Be Social +@subsection Be Social +@c %**end of header + +Next, you should print out your business card and be social. +Find a friend, help them install GNUnet and exchange business cards with +them. Or, if you're a desperate loner, you might try the next step with +your own card. Still, it'll be hard to have a conversation with +yourself later, so it would be better if you could find a friend. +You might also want a camera attached to your computer, so +you might need a trip to the store together. Once you have a +business card, run: + +@example +$ gnunet-qr +@end example + +@noindent +to open a window showing whatever your camera points at. +Hold up your friend's business card and tilt it until +the QR code is recognized. At that point, the window should +automatically close. At that point, your friend's NICKname and their +public key should have been automatically imported into your zone. +Assuming both of your peers are properly integrated in the +GNUnet network at this time, you should thus be able to +resolve your friends names. Suppose your friend's nickname +is "Bob". Then, type + +@example +$ gnunet-gns -u test.bob.gnu +@end example + +@noindent +to check if your friend was as good at following instructions +as you were. + + +@node Backup of Identities and Egos +@subsection Backup of Identities and Egos + + +One should always backup their files, especially in these SSD days (our +team has suffered 3 SSD crashes over a span of 2 weeks). Backing up peer +identity and zones is achieved by copying the following files: + +The peer identity file can be found +in @file{~/.local/share/gnunet/private_key.ecc} + +The private keys of your egos are stored in the +directory @file{~/.local/share/gnunet/identity/egos/}. +They are stored in files whose filenames correspond to the zones' +ego names. These are probably the most important files you want +to backup from a GNUnet installation. + +Note: All these files contain cryptographic keys and they are +stored without any encryption. So it is advisable to backup +encrypted copies of them. + +@node Revocation +@subsection Revocation + +Now, in the situation of an attacker gaining access to the private key of +one of your egos, the attacker can create records in the respective +GNS zone +and publish them as if you published them. Anyone resolving your +domain will get these new records and when they verify they seem +authentic because the attacker has signed them with your key. + +To address this potential security issue, you can pre-compute +a revocation certificate corresponding to your ego. This certificate, +when published on the P2P network, flags your private key as invalid, +and all further resolutions or other checks involving the key will fail. + +A revocation certificate is thus a useful tool when things go out of +control, but at the same time it should be stored securely. +Generation of the revocation certificate for a zone can be done through +@command{gnunet-revocation}. For example, the following command (as +unprivileged user) generates a revocation file +@file{revocation.dat} for the zone @code{zone1}: +@command{gnunet-revocation -f revocation.dat -R zone1} + +The above command only pre-computes a revocation certificate. It does +not revoke the given zone. Pre-computing a revocation certificate +involves computing a proof-of-work and hence may take upto 4 to 5 days +on a modern processor. Note that you can abort and resume the +calculation at any time. Also, even if you did not finish the +calculation, the resulting file will contain the signature, which is +sufficient to complete the revocation process even without access to +the private key. So instead of waiting for a few days, you can just +abort with CTRL-C, backup the revocation certificate and run the +calculation only if your key actually was compromised. This has the +disadvantage of revocation taking longer after the incident, but +the advantage of saving a significant amount of energy. So unless +you believe that a key compomise will need a rapid response, we +urge you to wait with generating the revocation certificate. +Also, the calculation is deliberately expensive, to deter people from +doing this just for fun (as the actual revocation operation is expensive +for the network, not for the peer performing the revocation). + +To avoid TL;DR ones from accidentally revocating their zones, I am not +giving away the command, but its simple: the actual revocation is +performed by using the @command{-p} option +of @command{gnunet-revocation}. + + + +@node What's Next? +@subsection What's Next? +@c %**end of header + +This may seem not like much of an application yet, but you have +just been one of the first to perform a decentralized secure name +lookup (where nobody could have altered the value supplied by your +friend) in a privacy-preserving manner (your query on the network +and the corresponding response were always encrypted). So what +can you really do with this? Well, to start with, you can publish your +GnuPG fingerprint in GNS as a "CERT" record and replace the public +web-of-trust with its complicated trust model with explicit names +and privacy-preserving resolution. Also, you should read the next +chapter of the tutorial and learn how to use GNS to have a +private conversation with your friend. Finally, help us +with the next GNUnet release for even more applications +using this new public key infrastructure. + +@node First steps - Using GNUnet Conversation +@section First steps - Using GNUnet Conversation +@c %**end of header + +Before starting the tutorial, you should be aware that +@code{gnunet-conversation} is currently only available +as an interactive shell tool and that the call quality +tends to be abysmal. There are also some awkward +steps necessary to use it. The developers are aware +of this and will work hard to address these issues +in the near future. + + +@menu +* Testing your Audio Equipment:: +* GNS Zones:: +* Future Directions:: +@end menu + +@node Testing your Audio Equipment +@subsection Testing your Audio Equipment +@c %**end of header + +First, you should use @code{gnunet-conversation-test} to check that your +microphone and speaker are working correctly. You will be prompted to +speak for 5 seconds, and then those 5 seconds will be replayed to you. +The network is not involved in this test. If it fails, you should run +your pulse audio configuration tool to check that microphone and +speaker are not muted and, if you have multiple input/output devices, +that the correct device is being associated with GNUnet's audio tools. + +@node GNS Zones +@subsection GNS Zones +@c %**end of header + +@code{gnunet-conversation} uses GNS for addressing. This means that +you need to have a GNS zone created before using it. Information +about how to create GNS zones can be found here. + + +@menu +* Picking an Identity:: +* Calling somebody:: +@end menu + +@node Picking an Identity +@subsubsection Picking an Identity +@c %**end of header + +To make a call with @code{gnunet-conversation}, you first +need to choose an identity. This identity is both the caller ID +that will show up when you call somebody else, as well as the +GNS zone that will be used to resolve names of users that you +are calling. Usually, the @code{master-zone} is a reasonable +choice. Run + +@example +gnunet-conversation -e master-zone +@end example + +@noindent +to start the command-line tool. You will see a message saying +that your phone is now "active on line 0". You can connect +multiple phones on different lines at the same peer. For the +first phone, the line zero is of course a fine choice. + +Next, you should type in @command{/help} for a list of +available commands. We will explain the important ones +during this tutorial. First, you will need to type in +@command{/address} to determine the address of your +phone. The result should look something like this: + +@example +/address +0-PD67SGHF3E0447TU9HADIVU9OM7V4QHTOG0EBU69TFRI2LG63DR0 +@end example + +@noindent +Here, the "0" is your phone line, and what follows +after the hyphen is your peer's identity. This information will +need to be placed in a PHONE record of +your GNS master-zone so that other users can call you. + +Start @code{gnunet-namestore-gtk} now (possibly from another +shell) and create an entry home-phone in your master zone. +For the record type, select PHONE. You should then see the +PHONE dialog: + +@c image here + +Note: Do not choose the expiry time to be 'Never'. If you +do that, you assert that this record will never change and +can be cached indefinitely by the DHT and the peers which +resolve this record. A reasonable period is 1 year. + +Enter your peer identity under Peer and leave the line +at zero. Select the first option to make the record public. +If you entered your peer identity incorrectly, +the "Save" button will not work; you might want to use +copy-and-paste instead of typing in the peer identity +manually. Save the record. + +@node Calling somebody +@subsubsection Calling somebody +@c %**end of header + +Now you can call a buddy. Obviously, your buddy will have to have GNUnet +installed and must have performed the same steps. Also, you must have +your buddy in your GNS master zone, for example by having imported +your buddy's public key using @code{gnunet-qr}. Suppose your buddy +is in your zone as @code{buddy.gnu} and they also created their +phone using a label "home-phone". Then you can initiate a call using: + +@example +/call home-phone.buddy.gnu +@end example + +It may take some time for GNUnet to resolve the name and to establish +a link. If your buddy has your public key in their master zone, they +should see an incoming call with your name. If your public key is not +in their master zone, they will just see the public key as the caller ID. + +Your buddy then can answer the call using the "/accept" command. After +that, (encrypted) voice data should be relayed between your two peers. +Either of you can end the call using @command{/cancel}. You can exit +@code{gnunet-converation} using @command{/quit}. + +@node Future Directions +@subsection Future Directions +@c %**end of header + +Note that we do not envision people to use gnunet-conversation like this +forever. We will write a graphical user interface, and that GUI will +automatically create the necessary records in the respective zone. + +@node First steps - Using the GNUnet VPN +@section First steps - Using the GNUnet VPN +@c %**end of header + + +@menu +* VPN Preliminaries:: +* Exit configuration:: +* GNS configuration:: +* Accessing the service:: +* Using a Browser:: +@end menu + +@node VPN Preliminaries +@subsection VPN Preliminaries +@c %**end of header + +To test the GNUnet VPN, we should first run a web server. +The easiest way to do this is to just start @code{gnunet-bcd}, +which will run a webserver on port @code{8888} by default. +Naturally, you can run some other HTTP server for our little tutorial. + +If you have not done this, you should also configure your +Name System Service switch to use GNS. In your @code{/etc/nsswitch.conf} +you should fine a line like this: + +@example +hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4 +@end example + +@noindent +The exact details may differ a bit, which is fine. Add the text +@code{gns [NOTFOUND=return]} after @code{files}: + +@example +hosts: files gns [NOTFOUND=return] mdns4_minimal [NOTFOUND=return] dns mdns4 +@end example + +@noindent +You might want to make sure that @code{/lib/libnss_gns.so.2} exists on +your system, it should have been created during the installation. +If not, re-run + +@example +$ configure --with-nssdir=/lib +$ cd src/gns/nss; sudo make install +@end example + +@noindent +to install the NSS plugins in the proper location. + +@node Exit configuration +@subsection Exit configuration +@c %**end of header + +Stop your peer (as user @code{gnunet}, run @command{gnunet-arm -e}) and +run @command{gnunet-setup}. In @command{gnunet-setup}, make sure to +activate the @strong{EXIT} and @strong{GNS} services in the General tab. +Then select the Exit tab. Most of the defaults should be fine (but +you should check against the screenshot that they have not been modified). +In the bottom area, enter @code{bcd} under Identifier and change the +Destination to @code{169.254.86.1:8888} (if your server runs on a port +other than 8888, change the 8888 port accordingly). + +Now exit @command{gnunet-setup} and restart your peer +(@command{gnunet-arm -s}). + +@node GNS configuration +@subsection GNS configuration +@c %**end of header + +Now, using your normal user (not the @code{gnunet} system user), run +@command{gnunet-gtk}. Select the GNS icon and add a new label www in your +master zone. For the record type, select @code{VPN}. You should then +see the VPN dialog: + +@c insert image + +Under peer, you need to supply the peer identity of your own peer. You can +obtain the respective string by running @command{gnunet-peerinfo -sq} +as the @code{gnunet} user. For the Identifier, you need to supply the same +identifier that we used in the Exit setup earlier, so here supply "bcd". +If you want others to be able to use the service, you should probably make +the record public. For non-public services, you should use a passphrase +instead of the string "bcd". Save the record and +exit @command{gnunet-gtk}. + +@node Accessing the service +@subsection Accessing the service +@c %**end of header + +You should now be able to access your webserver. Type in: + +@example +$ wget http://www.gnu/ +@end example + +@noindent +The request will resolve to the VPN record, telling the GNS resolver +to route it via the GNUnet VPN. The GNS resolver will ask the +GNUnet VPN for an IPv4 address to return to the application. The +VPN service will use the VPN information supplied by GNS to create +a tunnel (via GNUnet's MESH service) to the EXIT peer. +At the EXIT, the name "bcd" and destination port (80) will be mapped +to the specified destination IP and port. While all this is currently +happening on just the local machine, it should also work with other +peers --- naturally, they will need a way to access your GNS zone +first, for example by learning your public key from a QR code on +your business card. + +@node Using a Browser +@subsection Using a Browser +@c %**end of header + +Sadly, modern browsers tend to bypass the Name Services Switch and +attempt DNS resolution directly. You can either run +a @code{gnunet-dns2gns} DNS proxy, or point the browsers to an +HTTP proxy. When we tried it, Iceweasel did not like to connect to +the socks proxy for @code{.gnu} TLDs, even if we disabled its +autoblunder of changing @code{.gnu} to ".gnu.com". Still, +using the HTTP proxy with Chrome does work. + +@node File-sharing +@section File-sharing +@c %**end of header + +This chapter documents the GNUnet file-sharing application. The original +file-sharing implementation for GNUnet was designed to provide +@strong{anonymous} file-sharing. However, over time, we have also added +support for non-anonymous file-sharing (which can provide better +performance). Anonymous and non-anonymous file-sharing are quite +integrated in GNUnet and, except for routing, share most of the concepts +and implementation. There are three primary file-sharing operations: +publishing, searching and downloading. For each of these operations, +the user specifies an @strong{anonymity level}. If both the publisher and +the searcher/downloader specify "no anonymity", non-anonymous +file-sharing is used. If either user specifies some desired degree +of anonymity, anonymous file-sharing will be used. + +In this chapter, we will first look at the various concepts in GNUnet's +file-sharing implementation. Then, we will discuss specifics as to +how they impact users that publish, search or download files. + + + +@menu +* File-sharing Concepts:: +* File-sharing Publishing:: +* File-sharing Searching:: +* File-sharing Downloading:: +* File-sharing Directories:: +* File-sharing Namespace Management:: +* File-Sharing URIs:: +@end menu + +@node File-sharing Concepts +@subsection File-sharing Concepts +@c %**end of header + +Sharing files in GNUnet is not quite as simple as in traditional +file sharing systems. For example, it is not sufficient to just +place files into a specific directory to share them. In addition +to anonymous routing GNUnet attempts to give users a better experience +in searching for content. GNUnet uses cryptography to safely break +content into smaller pieces that can be obtained from different +sources without allowing participants to corrupt files. GNUnet +makes it difficult for an adversary to send back bogus search +results. GNUnet enables content providers to group related content +and to establish a reputation. Furthermore, GNUnet allows updates +to certain content to be made available. This section is supposed +to introduce users to the concepts that are used to achive these goals. + + +@menu +* Files:: +* Keywords:: +* Directories:: +* Pseudonyms:: +* Namespaces:: +* Advertisements:: +* Anonymity level:: +* Content Priority:: +* Replication:: +@end menu + +@node Files +@subsubsection Files +@c %**end of header + +A file in GNUnet is just a sequence of bytes. Any file-format is allowed +and the maximum file size is theoretically 264 bytes, except that it +would take an impractical amount of time to share such a file. +GNUnet itself never interprets the contents of shared files, except +when using GNU libextractor to obtain keywords. + +@node Keywords +@subsubsection Keywords +@c %**end of header + +Keywords are the most simple mechanism to find files on GNUnet. +Keywords are @strong{case-sensitive} and the search string +must always match @strong{exactly} the keyword used by the +person providing the file. Keywords are never transmitted in +plaintext. The only way for an adversary to determine the keyword +that you used to search is to guess it (which then allows the +adversary to produce the same search request). Since providing +keywords by hand for each shared file is tedious, GNUnet uses +GNU libextractor to help automate this process. Starting a +keyword search on a slow machine can take a little while since +the keyword search involves computing a fresh RSA key to formulate the +request. + +@node Directories +@subsubsection Directories +@c %**end of header + +A directory in GNUnet is a list of file identifiers with meta data. +The file identifiers provide sufficient information about the files +to allow downloading the contents. Once a directory has been created, +it cannot be changed since it is treated just like an ordinary file +by the network. Small files (of a few kilobytes) can be inlined in +the directory, so that a separate download becomes unnecessary. + +@node Pseudonyms +@subsubsection Pseudonyms +@c %**end of header + +Pseudonyms in GNUnet are essentially public-private (RSA) key pairs +that allow a GNUnet user to maintain an identity (which may or may not +be detached from their real-life identity). GNUnet's pseudonyms are not +file-sharing specific --- and they will likely be used by many GNUnet +applications where a user identity is required. + +Note that a pseudonym is NOT bound to a GNUnet peer. There can be multiple +pseudonyms for a single user, and users could (theoretically) share the +private pseudonym keys (currently only out-of-band by knowing which files +to copy around). + +@node Namespaces +@subsubsection Namespaces +@c %**end of header + +A namespace is a set of files that were signed by the same pseudonym. +Files (or directories) that have been signed and placed into a namespace +can be updated. Updates are identified as authentic if the same secret +key was used to sign the update. Namespaces are also useful to establish +a reputation, since all of the content in the namespace comes from the +same entity (which does not have to be the same person). + +@node Advertisements +@subsubsection Advertisements +@c %**end of header + +Advertisements are used to notify other users about the existence of a +namespace. Advertisements are propagated using the normal keyword search. +When an advertisement is received (in response to a search), the namespace +is added to the list of namespaces available in the namespace-search +dialogs of gnunet-fs-gtk and printed by gnunet-pseudonym. Whenever a +namespace is created, an appropriate advertisement can be generated. +The default keyword for the advertising of namespaces is "namespace". + +Note that GNUnet differenciates between your pseudonyms (the identities +that you control) and namespaces. If you create a pseudonym, you will +not automatically see the respective namespace. You first have to create +an advertisement for the namespace and find it using keyword +search --- even for your own namespaces. The @command{gnunet-pseudonym} +tool is currently responsible for both managing pseudonyms and namespaces. +This will likely change in the future to reduce the potential for +confusion. + +@node Anonymity level +@subsubsection Anonymity level +@c %**end of header + +The anonymity level determines how hard it should be for an adversary to +determine the identity of the publisher or the searcher/downloader. An +anonymity level of zero means that anonymity is not required. The default +anonymity level of "1" means that anonymous routing is desired, but no +particular amount of cover traffic is necessary. A powerful adversary +might thus still be able to deduce the origin of the traffic using +traffic analysis. Specifying higher anonymity levels increases the +amount of cover traffic required. While this offers better privacy, +it can also significantly hurt performance. + +@node Content Priority +@subsubsection Content Priority +@c %**end of header + +Depending on the peer's configuration, GNUnet peers migrate content +between peers. Content in this sense are individual blocks of a file, +not necessarily entire files. When peers run out of space (due to +local publishing operations or due to migration of content from other +peers), blocks sometimes need to be discarded. GNUnet first always +discards expired blocks (typically, blocks are published with an +expiration of about two years in the future; this is another option). +If there is still not enough space, GNUnet discards the blocks with the +lowest priority. The priority of a block is decided by its popularity +(in terms of requests from peers we trust) and, in case of blocks +published locally, the base-priority that was specified by the user +when the block was published initially. + +@node Replication +@subsubsection Replication +@c %**end of header + +When peers migrate content to other systems, the replication level +of a block is used to decide which blocks need to be migrated most +urgently. GNUnet will always push the block with the highest +replication level into the network, and then decrement the replication +level by one. If all blocks reach replication level zero, the +selection is simply random. + +@node File-sharing Publishing +@subsection File-sharing Publishing +@c %**end of header + +The command @command{gnunet-publish} can be used to add content +to the network. The basic format of the command is + +@example +$ gnunet-publish [-n] [-k KEYWORDS]* [-m TYPE:VALUE] FILENAME +@end example + + +@menu +* Important command-line options:: +* Indexing vs. Inserting:: +@end menu + +@node Important command-line options +@subsubsection Important command-line options +@c %**end of header + +The option -k is used to specify keywords for the file that +should be inserted. You can supply any number of keywords, +and each of the keywords will be sufficient to locate and +retrieve the file. + +The -m option is used to specify meta-data, such as descriptions. +You can use -m multiple times. The TYPE passed must be from the +list of meta-data types known to libextractor. You can obtain this +list by running @command{extract -L}. Use quotes around the entire +meta-data argument if the value contains spaces. The meta-data +is displayed to other users when they select which files to +download. The meta-data and the keywords are optional and +maybe inferred using @code{GNU libextractor}. + +gnunet-publish has a few additional options to handle namespaces and +directories. See the man-page for details. + +@node Indexing vs. Inserting +@subsubsection Indexing vs Inserting +@c %**end of header + +By default, GNUnet indexes a file instead of making a full copy. +This is much more efficient, but requries the file to stay unaltered +at the location where it was when it was indexed. If you intend to move, +delete or alter a file, consider using the option @code{-n} which will +force GNUnet to make a copy of the file in the database. + +Since it is much less efficient, this is strongly discouraged for large +files. When GNUnet indexes a file (default), GNUnet does @strong{not} +create an additional encrypted copy of the file but just computes a +summary (or index) of the file. That summary is approximately two percent +of the size of the original file and is stored in GNUnet's database. +Whenever a request for a part of an indexed file reaches GNUnet, +this part is encrypted on-demand and send out. This way, there is no +need for an additional encrypted copy of the file to stay anywhere +on the drive. This is different from other systems, such as Freenet, +where each file that is put online must be in Freenet's database in +encrypted format, doubling the space requirements if the user wants +to preseve a directly accessible copy in plaintext. + +Thus indexing should be used for all files where the user will keep +using this file (at the location given to gnunet-publish) and does +not want to retrieve it back from GNUnet each time. If you want to +remove a file that you have indexed from the local peer, use the tool +@command{gnunet-unindex} to un-index the file. + +The option @code{-n} may be used if the user fears that the file might +be found on their drive (assuming the computer comes under the control +of an adversary). When used with the @code{-n} flag, the user has a +much better chance of denying knowledge of the existence of the file, +even if it is still (encrypted) on the drive and the adversary is +able to crack the encryption (e.g. by guessing the keyword. + +@node File-sharing Searching +@subsection File-sharing Searching +@c %**end of header + +The command @command{gnunet-search} can be used to search +for content on GNUnet. The format is: + +@example +$ gnunet-search [-t TIMEOUT] KEYWORD +@end example + +@noindent +The -t option specifies that the query should timeout after +approximately TIMEOUT seconds. A value of zero is interpreted +as @emph{no timeout}, which is also the default. In this case, +gnunet-search will never terminate (unless you press CTRL-C). + +If multiple words are passed as keywords, they will all be +considered optional. Prefix keywords with a "+" to make them mandatory. + +Note that searching using + +@example +$ gnunet-search Das Kapital +@end example + +@noindent +is not the same as searching for + +@example +$ gnunet-search "Das Kapital" +@end example + +@noindent +as the first will match files shared under the keywords +"Das" or "Kapital" whereas the second will match files +shared under the keyword "Das Kapital". + +Search results are printed by gnunet-search like this: + +@example +$ gnunet-download -o "COPYING" --- gnunet://fs/chk/N8...C92.17992 +=> The GNU Public License <= (mimetype: text/plain) +@end example + +@noindent +The first line is the command you would have to enter to download +the file. The argument passed to @code{-o} is the suggested +filename (you may change it to whatever you like). +The @code{--} is followed by key for decrypting the file, +the query for searching the file, a checksum (in hexadecimal) +finally the size of the file in bytes. +The second line contains the description of the file; here this is +"The GNU Public License" and the mime-type (see the options for +gnunet-publish on how to specify these). + +@node File-sharing Downloading +@subsection File-sharing Downloading +@c %**end of header + +In order to download a file, you need the three values returned by +@command{gnunet-search}. +You can then use the tool @command{gnunet-download} to obtain the file: + +@example +$ gnunet-download -o FILENAME --- GNUNETURL +@end example + +@noindent +FILENAME specifies the name of the file where GNUnet is supposed +to write the result. Existing files are overwritten. If the +existing file contains blocks that are identical to the +desired download, those blocks will not be downloaded again +(automatic resume). + +If you want to download the GPL from the previous example, +you do the following: + +@example +$ gnunet-download -o "COPYING" --- gnunet://fs/chk/N8...92.17992 +@end example + +@noindent +If you ever have to abort a download, you can continue it at any time by +re-issuing @command{gnunet-download} with the same filename. +In that case, GNUnet will @strong{not} download blocks again that are +already present. + +GNUnet's file-encoding mechanism will ensure file integrity, even if the +existing file was not downloaded from GNUnet in the first place. + +You may want to use the @command{-V} switch (must be added before +the @command{--}) to turn on verbose reporting. In this case, +@command{gnunet-download} will print the current number of +bytes downloaded whenever new data was received. + +@node File-sharing Directories +@subsection File-sharing Directories +@c %**end of header + +Directories are shared just like ordinary files. If you download a +directory with @command{gnunet-download}, you can use +@command{gnunet-directory} to list its contents. The canonical +extension for GNUnet directories when stored as files in your +local file-system is ".gnd". The contents of a directory are URIs and +meta data. +The URIs contain all the information required by +@command{gnunet-download} to retrieve the file. The meta data +typically includes the mime-type, description, a filename and +other meta information, and possibly even the full original file +(if it was small). + +@node File-sharing Namespace Management +@subsection File-sharing Namespace Management +@c %**end of header + +@b{Please note that the text in this subsection is outdated and needs} +@b{to be rewritten for version 0.10!} + +The gnunet-pseudonym tool can be used to create pseudonyms and +to advertise namespaces. By default, gnunet-pseudonym simply +lists all locally available pseudonyms. + + +@menu +* Creating Pseudonyms:: +* Deleting Pseudonyms:: +* Advertising namespaces:: +* Namespace names:: +* Namespace root:: +@end menu + +@node Creating Pseudonyms +@subsubsection Creating Pseudonyms +@c %**end of header + +With the @command{-C NICK} option it can also be used to +create a new pseudonym. A pseudonym is the virtual identity +of the entity in control of a namespace. Anyone can create +any number of pseudonyms. Note that creating a pseudonym can +take a few minutes depending on the performance of the machine +used. + +@node Deleting Pseudonyms +@subsubsection Deleting Pseudonyms +@c %**end of header + +With the @command{-D NICK} option pseudonyms can be deleted. +Once the pseudonym has been deleted it is impossible to add +content to the corresponding namespace. Deleting the +pseudonym does not make the namespace or any content in it +unavailable. + +@node Advertising namespaces +@subsubsection Advertising namespaces +@c %**end of header + +Each namespace is associated with meta-data that describes +the namespace. This meta data is provided by the user at +the time that the namespace is advertised. Advertisements +are published under keywords so that they can be found using +normal keyword-searches. This way, users can learn about new +namespaces without relying on out-of-band communication or directories. +A suggested keyword to use for all namespaces is simply "namespace". +When a keyword-search finds a namespace advertisement, +it is automatically stored in a local list of known namespaces. +Users can then associate a rank with the namespace to remember +the quality of the content found in it. + +@node Namespace names +@subsubsection Namespace names +@c %**end of header + +While the namespace is uniquely identified by its ID, another way +to refer to the namespace is to use the NICKNAME. +The NICKNAME can be freely chosen by the creator of the namespace and +hence conflicts are possible. If a GNUnet client learns about more +than one namespace using the same NICKNAME, the ID is appended +to the NICKNAME to get a unique identifier. + +@node Namespace root +@subsubsection Namespace root +@c %**end of header + +An item of particular interest in the namespace advertisement is +the ROOT. The ROOT is the identifier of a designated entry in the +namespace. The idea is that the ROOT can be used to advertise an +entry point to the content of the namespace. + +@node File-Sharing URIs +@subsection File-Sharing URIs +@c %**end of header + +GNUnet (currently) uses four different types of URIs for +file-sharing. They all begin with "gnunet://fs/". +This section describes the four different URI types in detail. + + +@menu +* Encoding of hash values in URIs:: +* Content Hash Key (chk):: +* Location identifiers (loc):: +* Keyword queries (ksk):: +* Namespace content (sks):: +@end menu + +@node Encoding of hash values in URIs +@subsubsection Encoding of hash values in URIs +@c %**end of header + +Most URIs include some hash values. Hashes are encoded using +base32hex (RFC 2938). + +@node Content Hash Key (chk) +@subsubsection Content Hash Key (chk) +@c %**end of header + +A chk-URI is used to (uniquely) identify a file or directory +and to allow peers to download the file. Files are stored in +GNUnet as a tree of encrypted blocks. +The chk-URI thus contains the information to download and decrypt +those blocks. A chk-URI has the format +"gnunet://fs/chk/KEYHASH.QUERYHASH.SIZE". Here, "SIZE" +is the size of the file (which allows a peer to determine the +shape of the tree), KEYHASH is the key used to decrypt the file +(also the hash of the plaintext of the top block) and QUERYHASH +is the query used to request the top-level block (also the hash +of the encrypted block). + +@node Location identifiers (loc) +@subsubsection Location identifiers (loc) +@c %**end of header + +For non-anonymous file-sharing, loc-URIs are used to specify which +peer is offering the data (in addition to specifying all of the +data from a chk-URI). Location identifiers include a digital +signature of the peer to affirm that the peer is truly the +origin of the data. The format is +"gnunet://fs/loc/KEYHASH.QUERYHASH.SIZE.PEER.SIG.EXPTIME". +Here, "PEER" is the public key of the peer (in GNUnet format in +base32hex), SIG is the RSA signature (in GNUnet format in +base32hex) and EXPTIME specifies when the signature expires +(in milliseconds after 1970). + +@node Keyword queries (ksk) +@subsubsection Keyword queries (ksk) +@c %**end of header + +A keyword-URI is used to specify that the desired operation +is the search using a particular keyword. The format is simply +"gnunet://fs/ksk/KEYWORD". Non-ASCII characters can be specified +using the typical URI-encoding (using hex values) from HTTP. +"+" can be used to specify multiple keywords (which are then +logically "OR"-ed in the search, results matching both keywords +are given a higher rank): "gnunet://fs/ksk/KEYWORD1+KEYWORD2". + +@node Namespace content (sks) +@subsubsection Namespace content (sks) +@c %**end of header + +Namespaces are sets of files that have been approved by some (usually +pseudonymous) user --- typically by that user publishing all of the +files together. A file can be in many namespaces. A file is in a +namespace if the owner of the ego (aka the namespace's private key) +signs the CHK of the file cryptographically. An SKS-URI is used to +search a namespace. The result is a block containing meta data, +the CHK and the namespace owner's signature. The format of a sks-URI +is "gnunet://fs/sks/NAMESPACE/IDENTIFIER". Here, "NAMESPACE" +is the public key for the namespace. "IDENTIFIER" is a freely +chosen keyword (or password!). A commonly used identifier is +"root" which by convention refers to some kind of index or other +entry point into the namespace. + +@node The GNU Name System +@section The GNU Name System +@c %**end of header + + +The GNU Name System (GNS) is secure and decentralized naming system. +It allows its users to resolve and register names within the @code{.gnu} +@dfn{top-level domain} (TLD). + +GNS is designed to provide: +@itemize @bullet +@item Censorship resistance +@item Query privacy +@item Secure name resolution +@item Compatibility with DNS +@end itemize + +For the initial configuration and population of your +GNS installation, please follow the GNS setup instructions. +The remainder of this chapter will provide some background on GNS +and then describe how to use GNS in more detail. + +Unlike DNS, GNS does not rely on central root zones or authorities. +Instead any user administers their own root and can can create arbitrary +name value mappings. Furthermore users can delegate resolution to other +users' zones just like DNS NS records do. Zones are uniquely identified +via public keys and resource records are signed using the corresponding +public key. Delegation to another user's zone is done using special PKEY +records and petnames. A petname is a name that can be freely chosen by +the user. This results in non-unique name-value mappings as +@code{@uref{http://www.bob.gnu/, www.bob.gnu}} to one user might be +@code{@uref{http://www.friend.gnu/, www.friend.gnu}} for someone else. + + +@menu +* Maintaining your own Zones:: +* Obtaining your Zone Key:: +* Adding Links to Other Zones:: +* The Three Local Zones of GNS:: +* The Master Zone:: +* The Private Zone:: +* The Shorten Zone:: +* The ZKEY Top Level Domain in GNS:: +* Resource Records in GNS:: +@end menu + + +@node Maintaining your own Zones +@subsection Maintaining your own Zones + +To setup your GNS system you must execute: + +@example +$ gnunet-gns-import.sh +@end example + +@noindent +This will boostrap your zones and create the necessary key material. +Your keys can be listed using the @command{gnunet-identity} +command line tool: + +@example +$ gnunet-identity -d +@end example + +@noindent +You can arbitrarily create your own zones using the gnunet-identity +tool using: + +@example +$ gnunet-identity -C "new_zone" +@end example + +@noindent +Now you can add (or edit, or remove) records in your GNS zone using the +gnunet-setup GUI or using the gnunet-namestore command-line tool. +In either case, your records will be stored in an SQL database under +control of the gnunet-service-namestore. Note that if multiple users +use one peer, the namestore database will include the combined records +of all users. However, users will not be able to see each other's records +if they are marked as private. + +To provide a simple example for editing your own zone, suppose you +have your own web server with IP 1.2.3.4. Then you can put an +A record (A records in DNS are for IPv4 IP addresses) into your +local zone using the command: + +@example +$ gnunet-namestore -z master-zone -a -n www -t A -V 1.2.3.4 -e never +@end example + +@noindent +Afterwards, you will be able to access your webpage under "www.gnu" +(assuming your webserver does not use virtual hosting, if it does, +please read up on setting up the GNS proxy). + +Similar commands will work for other types of DNS and GNS records, +the syntax largely depending on the type of the record. +Naturally, most users may find editing the zones using the +@command{gnunet-setup} GUI to be easier. + +@node Obtaining your Zone Key +@subsection Obtaining your Zone Key + +Each zone in GNS has a public-private key. Usually, gnunet-namestore and +gnunet-setup will access your private key as necessary, so you do not +have to worry about those. What is important is your public key +(or rather, the hash of your public key), as you will likely want to +give it to others so that they can securely link to you. + +You can usually get the hash of your public key using + +@example +$ gnunet-identity -d $options | grep master-zone | awk '@{print $3@}' +@end example + +@noindent +For example, the output might be something like: + +@example +DC3SEECJORPHQNVRH965A6N74B1M37S721IG4RBQ15PJLLPJKUE0 +@end example + +@noindent +Alternatively, you can obtain a QR code with your zone key AND +your pseudonym from gnunet-gtk. The QR code is displayed in the +GNS tab and can be stored to disk using the Save as button next +to the image. + +@node Adding Links to Other Zones +@subsection Adding Links to Other Zones + + +A central operation in GNS is the ability to securely delegate to +other zones. Basically, by adding a delegation you make all of the +names from the other zone available to yourself. This section +describes how to create delegations. + +Suppose you have a friend who you call 'bob' who also uses GNS. +You can then delegate resolution of names to Bob's zone by adding +a PKEY record to their local zone: + +@example +$ gnunet-namestore -a -n bob --type PKEY -V XXXX -e never +@end example + +@noindent +Note that XXXX in the command above must be replaced with the +hash of Bob's public key (the output your friend obtained using +the gnunet-identity command from the previous section and told you, +for example by giving you a business card containing this +information as a QR code). + +Assuming Bob has an A record for their website under the name of +www in his zone, you can then access Bob's website under +www.bob.gnu --- as well as any (public) GNS record that Bob has +in their zone by replacing www with the respective name of the +record in Bob's zone. + +@c themselves? themself? +Furthermore, if Bob has themselves a (public) delegation to Carol's +zone under "carol", you can access Carol's records under +NAME.carol.bob.gnu (where NAME is the name of Carol's record you +want to access). + +@node The Three Local Zones of GNS +@subsection The Three Local Zones of GNS + +Each user GNS has control over three zones. Each of the zones +has a different purpose. These zones are the + +@itemize @bullet + +@item master zone, +@item private zone, and the +@item shorten zone. +@end itemize + +@node The Master Zone +@subsection The Master Zone + + +The master zone is your personal TLD. Names within the @code{.gnu} +namespace are resolved relative to this zone. You can arbitrarily +add records to this zone and selectively publish those records. + +@node The Private Zone +@subsection The Private Zone + + +The private zone is a subzone (or subdomain in DNS terms) of your +master zone. It should be used for records that you want to keep +private. For example @code{bank.private.gnu}. The key idea is that +you want to keep your private records separate, if just to know +that those names are not available to other users. + +@node The Shorten Zone +@subsection The Shorten Zone + + +The shorten zone can either be a subzone of the master zone or the +private zone. It is different from the other zones in that GNS +will automatically populate this zone with other users' zones based +on their PSEU records whenever you resolve a name. + +For example if you go to +@code{@uref{http://www.bob.alice.dave.gnu/, www.bob.alice.dave.gnu}}, +GNS will try to import @code{bob} into your shorten zone. Having +obtained Bob's PKEY from @code{alice.dave.gnu}, GNS will lookup the +PSEU record for @code{+} in Bob's zone. If it exists and the specified +pseudonym is not taken, Bob's PKEY will be automatically added under +that pseudonym (i.e. "bob") into your shorten zone. From then on, +Bob's webpage will also be available for you as +@code{@uref{http://www.bob.short.gnu/, www.bob.short.gnu}}. +This feature is called @b{automatic name shortening} and is supposed to +keep GNS names as short and memorable as possible. + +@node The ZKEY Top Level Domain in GNS +@subsection The ZKEY Top Level Domain in GNS + + +GNS also provides a secure and globally unique namespace under the .zkey +top-level domain. A name in the .zkey TLD corresponds to the (printable) +public key of a zone. Names in the .zkey TLD are then resolved by querying +the respective zone. The .zkey TLD is expected to be used under rare +circumstances where globally unique names are required and for +integration with legacy systems. + +@node Resource Records in GNS +@subsection Resource Records in GNS + + +GNS supports the majority of the DNS records as defined in +@uref{http://www.ietf.org/rfc/rfc1035.txt, RFC 1035}. Additionally, +GNS defines some new record types the are unique to the GNS system. +For example, GNS-specific resource records are used to give petnames +for zone delegation, revoke zone keys and provide some compatibility +features. + +For some DNS records, GNS does extended processing to increase their +usefulness in GNS. In particular, GNS introduces special names +referred to as "zone relative names". Zone relative names are allowed +in some resource record types (for example, in NS and CNAME records) +and can also be used in links on webpages. Zone relative names end +in ".+" which indicates that the name needs to be resolved relative +to the current authoritative zone. The extended processing of those +names will expand the ".+" with the correct delegation chain to the +authoritative zone (replacing ".+" with the name of the location +where the name was encountered) and hence generate a +valid @code{.gnu} name. + +GNS currently supports the following record types: + +@menu +* NICK:: +* PKEY:: +* BOX:: +* LEHO:: +* VPN:: +* A AAAA and TXT:: +* CNAME:: +* GNS2DNS:: +* SOA SRV PTR and MX:: +@end menu + +@node NICK +@subsubsection NICK + +A NICK record is used to give a zone a name. With a NICK record, you can +essentially specify how you would like to be called. GNS expects this +record under the name "+" in the zone's database (NAMESTORE); however, +it will then automatically be copied into each record set, so that +clients never need to do a separate lookup to discover the NICK record. + +@b{Example}@ + +@example +Name: +; RRType: NICK; Value: bob +@end example + +@noindent +This record in Bob's zone will tell other users that this zone wants +to be referred to as 'bob'. Note that nobody is obliged to call Bob's +zone 'bob' in their own zones. It can be seen as a +recommendation ("Please call me 'bob'"). + +@node PKEY +@subsubsection PKEY + +PKEY records are used to add delegation to other users' zones and +give those zones a petname. + +@b{Example}@ + +Let Bob's zone be identified by the hash "ABC012". Bob is your friend +so you want to give them the petname "friend". Then you add the +following record to your zone: + +@example +Name: friend; RRType: PKEY; Value: ABC012; +@end example + +@noindent +This will allow you to resolve records in bob's zone +under "*.friend.gnu". + +@node BOX +@subsubsection BOX + +BOX records are there to integrate information from TLSA or +SRV records under the main label. In DNS, TLSA and SRV records +use special names of the form @code{_port._proto.(label.)*tld} to +indicate the port number and protocol (i.e. tcp or udp) for which +the TLSA or SRV record is valid. This causes various problems, and +is elegantly solved in GNS by integrating the protocol and port +numbers together with the respective value into a "BOX" record. +Note that in the GUI, you do not get to edit BOX records directly +right now --- the GUI will provide the illusion of directly +editing the TLSA and SRV records, even though they internally +are BOXed up. + +@node LEHO +@subsubsection LEHO + +The LEgacy HOstname of a server. Some webservers expect a specific +hostname to provide a service (virtiual hosting). Also SSL +certificates usually contain DNS names. To provide the expected +legacy DNS name for a server, the LEHO record can be used. +To mitigate the just mentioned issues the GNS proxy has to be used. +The GNS proxy will use the LEHO information to apply the necessary +transformations. + +@node VPN +@subsubsection VPN + +GNS allows easy access to services provided by the GNUnet Virtual Public +Network. When the GNS resolver encounters a VPN record it will contact +the VPN service to try and allocate an IPv4/v6 address (if the queries +record type is an IP address) that can be used to contact the service. + +@b{Example}@ + +I want to provide access to the VPN service "web.gnu." on port 80 on peer +ABC012:@ +Name: www; RRType: VPN; Value: 80 ABC012 web.gnu. + +The peer ABC012 is configured to provide an exit point for the service +"web.gnu." on port 80 to it's server running locally on port 8080 by +having the following lines in the @file{gnunet.conf} configuration file: + +@example +[web.gnunet.] +TCP_REDIRECTS = 80:localhost4:8080 +@end example + +@node A AAAA and TXT +@subsubsection A AAAA and TXT + +Those records work in exactly the same fashion as in traditional DNS. + +@node CNAME +@subsubsection CNAME + +As specified in RFC 1035 whenever a CNAME is encountered the query +needs to be restarted with the specified name. In GNS a CNAME +can either be: + +@itemize @bullet +@item A zone relative name, +@item A zkey name or +@item A DNS name (in which case resolution will continue outside +of GNS with the systems DNS resolver) +@end itemize + +@node GNS2DNS +@subsubsection GNS2DNS + +GNS can delegate authority to a legacy DNS zone. For this, the +name of the DNS nameserver and the name of the DNS zone are +specified in a GNS2DNS record. + +@b{Example} + +@example +Name: pet; RRType: GNS2DNS; Value: gnunet.org@@a.ns.joker.com +@end example + +@noindent +Any query to @code{pet.gnu} will then be delegated to the DNS server at +@code{a.ns.joker.com}. For example, +@code{@uref{http://www.pet.gnu/, www.pet.gnu}} will result in a DNS query +for @code{@uref{http://www.gnunet.org/, www.gnunet.org}} to the server +at @code{a.ns.joker.com}. Delegation to DNS via NS records in GNS can +be useful if you do not want to start resolution in the DNS root zone +(due to issues such as censorship or availability). + +Note that you would typically want to use a relative name for the +nameserver, i.e. + +@example +Name: pet; RRType: GNS2DNS; Value: gnunet.org@@ns-joker.+@ +Name: ns-joker; RRType: A; Value: 184.172.157.218 +@end example + +@noindent +This way, you can avoid involving the DNS hierarchy in the resolution of +@code{a.ns.joker.com}. In the example above, the problem may not be +obvious as the nameserver for "gnunet.org" is in the ".com" zone. +However, imagine the nameserver was "ns.gnunet.org". In this case, +delegating to "ns.gnunet.org" would mean that despite using GNS, +censorship in the DNS ".org" zone would still be effective. + +@node SOA SRV PTR and MX +@subsubsection SOA SRV PTR and MX + +The domain names in those records can, again, be either + +@itemize @bullet +@item A zone relative name, +@item A zkey name or +@item A DNS name +@end itemize + +The resolver will expand the zone relative name if possible. +Note that when using MX records within GNS, the target mail +server might still refuse to accept e-mails to the resulting +domain as the name might not match. GNS-enabled mail clients +should use the ZKEY zone as the destination hostname and +GNS-enabled mail servers should be configured to accept +e-mails to the ZKEY-zones of all local users. + +@node Using the Virtual Public Network +@section Using the Virtual Public Network + +@menu +* Setting up an Exit node:: +* Fedora and the Firewall:: +* Setting up VPN node for protocol translation and tunneling:: +@end menu + +Using the GNUnet Virtual Public Network (VPN) application you can +tunnel IP traffic over GNUnet. Moreover, the VPN comes +with built-in protocol translation and DNS-ALG support, enabling +IPv4-to-IPv6 protocol translation (in both directions). +This chapter documents how to use the GNUnet VPN. + +The first thing to note about the GNUnet VPN is that it is a public +network. All participating peers can participate and there is no +secret key to control access. So unlike common virtual private +networks, the GNUnet VPN is not useful as a means to provide a +"private" network abstraction over the Internet. The GNUnet VPN +is a virtual network in the sense that it is an overlay over the +Internet, using its own routing mechanisms and can also use an +internal addressing scheme. The GNUnet VPN is an Internet +underlay --- TCP/IP applications run on top of it. + +The VPN is currently only supported on GNU/Linux systems. +Support for operating systems that support TUN (such as FreeBSD) +should be easy to add (or might not even require any coding at +all --- we just did not test this so far). Support for other +operating systems would require re-writing the code to create virtual +network interfaces and to intercept DNS requests. + +The VPN does not provide good anonymity. While requests are routed +over the GNUnet network, other peers can directly see the source +and destination of each (encapsulated) IP packet. Finally, if you +use the VPN to access Internet services, the peer sending the +request to the Internet will be able to observe and even alter +the IP traffic. We will discuss additional security implications +of using the VPN later in this chapter. + +@node Setting up an Exit node +@subsection Setting up an Exit node + +Any useful operation with the VPN requires the existence of an exit +node in the GNUnet Peer-to-Peer network. Exit functionality can only +be enabled on peers that have regular Internet access. If you want +to play around with the VPN or support the network, we encourage +you to setup exit nodes. This chapter documents how to setup an +exit node. + +There are four types of exit functions an exit node can provide, +and using the GNUnet VPN to access the Internet will only work +nicely if the first three types are provided somewhere in +the network. The four exit functions are: + +@itemize @bullet +@item DNS: allow other peers to use your DNS resolver +@item IPv4: allow other peers to access your IPv4 Internet connection +@item IPv6: allow other peers to access your IPv6 Internet connection +@item Local service: allow other peers to access a specific TCP or +UDP service your peer is providing +@end itemize + +By enabling "exit" in gnunet-setup and checking the respective boxes +in the "exit" tab, you can easily choose which of the above exit +functions you want to support. + +Note, however, that by supporting the first three functions you will +allow arbitrary other GNUnet users to access the Internet via your +system. This is somewhat similar to running a Tor exit node. The +Torproject has a nice article about what to consider if you want +to do this here. We believe that generally running a DNS exit node +is completely harmless. + +The exit node configuration does currently not allow you to restrict the +Internet traffic that leaves your system. In particular, you cannot +exclude SMTP traffic (or block port 25) or limit to HTTP traffic using +the GNUnet configuration. However, you can use your host firewall to +restrict outbound connections from the virtual tunnel interface. This +is highly recommended. In the future, we plan to offer a wider range +of configuration options for exit nodes. + +Note that by running an exit node GNUnet will configure your kernel +to perform IP-forwarding (for IPv6) and NAT (for IPv4) so that the +traffic from the virtual interface can be routed to the Internet. +In order to provide an IPv6-exit, you need to have a subnet routed +to your host's external network interface and assign a subrange of +that subnet to the GNUnet exit's TUN interface. + +When running a local service, you should make sure that the local +service is (also) bound to the IP address of your EXIT interface +(i.e. 169.254.86.1). It will NOT work if your local service is +just bound to loopback. You may also want to create a "VPN" record +in your zone of the GNU Name System to make it easy for others to +access your service via a name instead of just the full service +descriptor. Note that the identifier you assign the service can +serve as a passphrase or shared secret, clients connecting to the +service must somehow learn the service's name. VPN records in the +GNU Name System can make this easier. + +@node Fedora and the Firewall +@subsection Fedora and the Firewall + + +When using an exit node on Fedora 15, the standard firewall can +create trouble even when not really exiting the local system! +For IPv4, the standard rules seem fine. However, for IPv6 the +standard rules prohibit traffic from the network range of the +virtual interface created by the exit daemon to the local IPv6 +address of the same interface (which is essentially loopback +traffic, so you might suspect that a standard firewall would +leave this traffic alone). However, as somehow for IPv6 the +traffic is not recognized as originating from the local +system (and as the connection is not already "established"), +the firewall drops the traffic. You should still get ICMPv6 +packets back, but that's obviously not very useful. + +Possible ways to fix this include disabling the firewall (do you +have a good reason for having it on?) or disabling the firewall +at least for the GNUnet exit interface (or the respective +IPv4/IPv6 address range). The best way to diagnose these kinds +of problems in general involves setting the firewall to REJECT +instead of DROP and to watch the traffic using wireshark +(or tcpdump) to see if ICMP messages are generated when running +some tests that should work. + +@node Setting up VPN node for protocol translation and tunneling +@subsection Setting up VPN node for protocol translation and tunneling + + +The GNUnet VPN/PT subsystem enables you to tunnel IP traffic over the +VPN to an exit node, from where it can then be forwarded to the +Internet. This section documents how to setup VPN/PT on a node. +Note that you can enable both the VPN and an exit on the same peer. +In this case, IP traffic from your system may enter your peer's VPN +and leave your peer's exit. This can be useful as a means to do +protocol translation. For example, you might have an application that +supports only IPv4 but needs to access an IPv6-only site. In this case, +GNUnet would perform 4to6 protocol translation between the VPN (IPv4) +and the Exit (IPv6). Similarly, 6to4 protocol translation is also +possible. However, the primary use for GNUnet would be to access +an Internet service running with an IP version that is not supported +by your ISP. In this case, your IP traffic would be routed via GNUnet +to a peer that has access to the Internet with the desired IP version. + +Setting up an entry node into the GNUnet VPN primarily requires you +to enable the "VPN/PT" option in "gnunet-setup". This will launch the +"gnunet-service-vpn", "gnunet-service-dns" and "gnunet-daemon-pt" +processes. The "gnunet-service-vpn" will create a virtual interface +which will be used as the target for your IP traffic that enters the +VPN. Additionally, a second virtual interface will be created by +the "gnunet-service-dns" for your DNS traffic. You will then need to +specify which traffic you want to tunnel over GNUnet. If your ISP only +provides you with IPv4 or IPv6-access, you may choose to tunnel the +other IP protocol over the GNUnet VPN. If you do not have an ISP +(and are connected to other GNUnet peers via WLAN), you can also +choose to tunnel all IP traffic over GNUnet. This might also provide +you with some anonymity. After you enable the respective options +and restart your peer, your Internet traffic should be tunneled +over the GNUnet VPN. + +The GNUnet VPN uses DNS-ALG to hijack your IP traffic. Whenever an +application resolves a hostname (i.e. 'gnunet.org'), the +"gnunet-daemon-pt" will instruct the "gnunet-service-dns" to intercept +the request (possibly route it over GNUnet as well) and replace the +normal answer with an IP in the range of the VPN's interface. +"gnunet-daemon-pt" will then tell "gnunet-service-vpn" to forward all +traffic it receives on the TUN interface via the VPN to the original +destination. + +For applications that do not use DNS, you can also manually create +such a mapping using the gnunet-vpn command-line tool. Here, you +specfiy the desired address family of the result (i.e. "-4"), and the +intended target IP on the Internet ("-i 131.159.74.67") and +"gnunet-vpn" will tell you which IP address in the range of your +VPN tunnel was mapped. + +@command{gnunet-vpn} can also be used to access "internal" services +offered by GNUnet nodes. So if you happen to know a peer and a +service offered by that peer, you can create an IP tunnel to +that peer by specifying the peer's identity, service name and +protocol (--tcp or --udp) and you will again receive an IP address +that will terminate at the respective peer's service. diff --git a/doc/documentation/chapters/vocabulary.texi b/doc/documentation/chapters/vocabulary.texi new file mode 100644 index 0000000000..85b40b17bf --- /dev/null +++ b/doc/documentation/chapters/vocabulary.texi @@ -0,0 +1,72 @@ +@node Vocabulary +@chapter Vocabulary + +@menu +* Definitions abbreviations and acronyms:: +* Words and characters:: +* Technical Assumptions:: +@end menu + +Throughout this Reference Manual we will use certain words and characters +which are listed in this introductionary chapter. + +@node Definitions abbreviations and acronyms +@section Definitions abbreviations and acronyms + +@menu +* Definitions:: +@end menu + +@node Definitions +@subsection Defitions + +Throughout this Reference Manual, the following terms and definitions +apply. + +@node Words and characters +@section Words and characters + +@enumerate +@item +In chapter Installation Handbook, +``@command{#}'' in example code blocks describes commands executed as root + +@example +# echo "I am root" +I am root +@end example + +@item +However, in the chapter GNUnet C Tutorial +``@command{#}'' in example code blocks describes commands, ie comments. + +@example +# Do the foobar thing: +$ make foobar +@end example + +@item +Dollarsign ``@command{$}'' in example code blocks describes commands you +execute as unprivileged users. + +@example +$ cd foo; ./configure --example-switch +@end example + +@item +Backslash ``@command{\}'' describes linebreaks. + +@example +./configure --foo --bar --baz \ + --short-loop +@end example + +...expands to @code{./configure --foo --bar --baz --short-loop} + +@end enumerate + +@node Technical Assumptions +@section Technical Assumptions + +@c Is it really assuming Bash (ie Bash extensions of POSIX being used)? +The shell on GNU systems is assumed to be Bash. diff --git a/doc/documentation/docstyle.css b/doc/documentation/docstyle.css new file mode 100644 index 0000000000..8719248d03 --- /dev/null +++ b/doc/documentation/docstyle.css @@ -0,0 +1,76 @@ +html, body { + font-size: 1em; + text-align: left; + text-decoration: none; +} +html { background-color: #e7e7e7; } + +body { + max-width: 74.92em; + margin: 0 auto; + padding: .5em 1em 1em 1em; + background-color: white; + border: .1em solid #c0c0c0; +} + +h1, h2, h3, h4 { color: #333; } +h5, h6, dt { color: #222; } + + +a h3 { + color: #005090; +} + +a[href] { color: #005090; } +a[href]:visited { color: #100070; } +a[href]:active, a[href]:hover { + color: #100070; + text-decoration: none; +} + +.linkrow { + margin: 3em 0; +} + +.linkrow { + text-align: center; +} + +div.example { padding: .8em 1.2em .4em; } +pre.example { padding: .8em 1.2em; } +div.example, pre.example { + margin: 1em 0 1em 3% ; + -webkit-border-radius: .3em; + -moz-border-radius: .3em; + border-radius: .3em; + border: 1px solid #d4cbb6; + background-color: #f2efe4; +} +div.example > pre.example { + padding: 0 0 .4em; + margin: 0; + border: none; +} + + +/* This makes the very long tables of contents in Gnulib and other + manuals easier to read. */ +.contents ul, .shortcontents ul { font-weight: bold; } +.contents ul ul, .shortcontents ul ul { font-weight: normal; } +.contents ul { list-style: none; } + +/* For colored navigation bars (Emacs manual): make the bar extend + across the whole width of the page and give it a decent height. */ +.header, .node { margin: 0 -1em; padding: 0 1em; } +.header p, .node p { line-height: 2em; } + +/* For navigation links */ +.node a, .header a { display: inline-block; line-height: 2em; } +.node a:hover, .header a:hover { background: #f2efe4; } + +table.cartouche { + border-collapse: collapse; + border-color: darkred; + border-style: solid; + border-width: 3px; +} diff --git a/doc/documentation/fdl-1.3.texi b/doc/documentation/fdl-1.3.texi new file mode 100644 index 0000000000..cb71f05a17 --- /dev/null +++ b/doc/documentation/fdl-1.3.texi @@ -0,0 +1,505 @@ +@c The GNU Free Documentation License. +@center Version 1.3, 3 November 2008 + +@c This file is intended to be included within another document, +@c hence no sectioning command or @node. + +@display +Copyright @copyright{} 2000, 2001, 2002, 2007, 2008 Free Software Foundation, Inc. +@uref{http://fsf.org/} + +Everyone is permitted to copy and distribute verbatim copies +of this license document, but changing it is not allowed. +@end display + +@enumerate 0 +@item +PREAMBLE + +The purpose of this License is to make a manual, textbook, or other +functional and useful document @dfn{free} in the sense of freedom: to +assure everyone the effective freedom to copy and redistribute it, +with or without modifying it, either commercially or noncommercially. +Secondarily, this License preserves for the author and publisher a way +to get credit for their work, while not being considered responsible +for modifications made by others. + +This License is a kind of ``copyleft'', which means that derivative +works of the document must themselves be free in the same sense. It +complements the GNU General Public License, which is a copyleft +license designed for free software. + +We have designed this License in order to use it for manuals for free +software, because free software needs free documentation: a free +program should come with manuals providing the same freedoms that the +software does. But this License is not limited to software manuals; +it can be used for any textual work, regardless of subject matter or +whether it is published as a printed book. We recommend this License +principally for works whose purpose is instruction or reference. + +@item +APPLICABILITY AND DEFINITIONS + +This License applies to any manual or other work, in any medium, that +contains a notice placed by the copyright holder saying it can be +distributed under the terms of this License. Such a notice grants a +world-wide, royalty-free license, unlimited in duration, to use that +work under the conditions stated herein. The ``Document'', below, +refers to any such manual or work. Any member of the public is a +licensee, and is addressed as ``you''. You accept the license if you +copy, modify or distribute the work in a way requiring permission +under copyright law. + +A ``Modified Version'' of the Document means any work containing the +Document or a portion of it, either copied verbatim, or with +modifications and/or translated into another language. + +A ``Secondary Section'' is a named appendix or a front-matter section +of the Document that deals exclusively with the relationship of the +publishers or authors of the Document to the Document's overall +subject (or to related matters) and contains nothing that could fall +directly within that overall subject. (Thus, if the Document is in +part a textbook of mathematics, a Secondary Section may not explain +any mathematics.) The relationship could be a matter of historical +connection with the subject or with related matters, or of legal, +commercial, philosophical, ethical or political position regarding +them. + +The ``Invariant Sections'' are certain Secondary Sections whose titles +are designated, as being those of Invariant Sections, in the notice +that says that the Document is released under this License. If a +section does not fit the above definition of Secondary then it is not +allowed to be designated as Invariant. The Document may contain zero +Invariant Sections. If the Document does not identify any Invariant +Sections then there are none. + +The ``Cover Texts'' are certain short passages of text that are listed, +as Front-Cover Texts or Back-Cover Texts, in the notice that says that +the Document is released under this License. A Front-Cover Text may +be at most 5 words, and a Back-Cover Text may be at most 25 words. + +A ``Transparent'' copy of the Document means a machine-readable copy, +represented in a format whose specification is available to the +general public, that is suitable for revising the document +straightforwardly with generic text editors or (for images composed of +pixels) generic paint programs or (for drawings) some widely available +drawing editor, and that is suitable for input to text formatters or +for automatic translation to a variety of formats suitable for input +to text formatters. A copy made in an otherwise Transparent file +format whose markup, or absence of markup, has been arranged to thwart +or discourage subsequent modification by readers is not Transparent. +An image format is not Transparent if used for any substantial amount +of text. A copy that is not ``Transparent'' is called ``Opaque''. + +Examples of suitable formats for Transparent copies include plain +ASCII without markup, Texinfo input format, La@TeX{} input +format, SGML or XML using a publicly available +DTD, and standard-conforming simple HTML, +PostScript or PDF designed for human modification. Examples +of transparent image formats include PNG, XCF and +JPG. Opaque formats include proprietary formats that can be +read and edited only by proprietary word processors, SGML or +XML for which the DTD and/or processing tools are +not generally available, and the machine-generated HTML, +PostScript or PDF produced by some word processors for +output purposes only. + +The ``Title Page'' means, for a printed book, the title page itself, +plus such following pages as are needed to hold, legibly, the material +this License requires to appear in the title page. For works in +formats which do not have any title page as such, ``Title Page'' means +the text near the most prominent appearance of the work's title, +preceding the beginning of the body of the text. + +The ``publisher'' means any person or entity that distributes copies +of the Document to the public. + +A section ``Entitled XYZ'' means a named subunit of the Document whose +title either is precisely XYZ or contains XYZ in parentheses following +text that translates XYZ in another language. (Here XYZ stands for a +specific section name mentioned below, such as ``Acknowledgements'', +``Dedications'', ``Endorsements'', or ``History''.) To ``Preserve the Title'' +of such a section when you modify the Document means that it remains a +section ``Entitled XYZ'' according to this definition. + +The Document may include Warranty Disclaimers next to the notice which +states that this License applies to the Document. These Warranty +Disclaimers are considered to be included by reference in this +License, but only as regards disclaiming warranties: any other +implication that these Warranty Disclaimers may have is void and has +no effect on the meaning of this License. + +@item +VERBATIM COPYING + +You may copy and distribute the Document in any medium, either +commercially or noncommercially, provided that this License, the +copyright notices, and the license notice saying this License applies +to the Document are reproduced in all copies, and that you add no other +conditions whatsoever to those of this License. You may not use +technical measures to obstruct or control the reading or further +copying of the copies you make or distribute. However, you may accept +compensation in exchange for copies. If you distribute a large enough +number of copies you must also follow the conditions in section 3. + +You may also lend copies, under the same conditions stated above, and +you may publicly display copies. + +@item +COPYING IN QUANTITY + +If you publish printed copies (or copies in media that commonly have +printed covers) of the Document, numbering more than 100, and the +Document's license notice requires Cover Texts, you must enclose the +copies in covers that carry, clearly and legibly, all these Cover +Texts: Front-Cover Texts on the front cover, and Back-Cover Texts on +the back cover. Both covers must also clearly and legibly identify +you as the publisher of these copies. The front cover must present +the full title with all words of the title equally prominent and +visible. You may add other material on the covers in addition. +Copying with changes limited to the covers, as long as they preserve +the title of the Document and satisfy these conditions, can be treated +as verbatim copying in other respects. + +If the required texts for either cover are too voluminous to fit +legibly, you should put the first ones listed (as many as fit +reasonably) on the actual cover, and continue the rest onto adjacent +pages. + +If you publish or distribute Opaque copies of the Document numbering +more than 100, you must either include a machine-readable Transparent +copy along with each Opaque copy, or state in or with each Opaque copy +a computer-network location from which the general network-using +public has access to download using public-standard network protocols +a complete Transparent copy of the Document, free of added material. +If you use the latter option, you must take reasonably prudent steps, +when you begin distribution of Opaque copies in quantity, to ensure +that this Transparent copy will remain thus accessible at the stated +location until at least one year after the last time you distribute an +Opaque copy (directly or through your agents or retailers) of that +edition to the public. + +It is requested, but not required, that you contact the authors of the +Document well before redistributing any large number of copies, to give +them a chance to provide you with an updated version of the Document. + +@item +MODIFICATIONS + +You may copy and distribute a Modified Version of the Document under +the conditions of sections 2 and 3 above, provided that you release +the Modified Version under precisely this License, with the Modified +Version filling the role of the Document, thus licensing distribution +and modification of the Modified Version to whoever possesses a copy +of it. In addition, you must do these things in the Modified Version: + +@enumerate A +@item +Use in the Title Page (and on the covers, if any) a title distinct +from that of the Document, and from those of previous versions +(which should, if there were any, be listed in the History section +of the Document). You may use the same title as a previous version +if the original publisher of that version gives permission. + +@item +List on the Title Page, as authors, one or more persons or entities +responsible for authorship of the modifications in the Modified +Version, together with at least five of the principal authors of the +Document (all of its principal authors, if it has fewer than five), +unless they release you from this requirement. + +@item +State on the Title page the name of the publisher of the +Modified Version, as the publisher. + +@item +Preserve all the copyright notices of the Document. + +@item +Add an appropriate copyright notice for your modifications +adjacent to the other copyright notices. + +@item +Include, immediately after the copyright notices, a license notice +giving the public permission to use the Modified Version under the +terms of this License, in the form shown in the Addendum below. + +@item +Preserve in that license notice the full lists of Invariant Sections +and required Cover Texts given in the Document's license notice. + +@item +Include an unaltered copy of this License. + +@item +Preserve the section Entitled ``History'', Preserve its Title, and add +to it an item stating at least the title, year, new authors, and +publisher of the Modified Version as given on the Title Page. If +there is no section Entitled ``History'' in the Document, create one +stating the title, year, authors, and publisher of the Document as +given on its Title Page, then add an item describing the Modified +Version as stated in the previous sentence. + +@item +Preserve the network location, if any, given in the Document for +public access to a Transparent copy of the Document, and likewise +the network locations given in the Document for previous versions +it was based on. These may be placed in the ``History'' section. +You may omit a network location for a work that was published at +least four years before the Document itself, or if the original +publisher of the version it refers to gives permission. + +@item +For any section Entitled ``Acknowledgements'' or ``Dedications'', Preserve +the Title of the section, and preserve in the section all the +substance and tone of each of the contributor acknowledgements and/or +dedications given therein. + +@item +Preserve all the Invariant Sections of the Document, +unaltered in their text and in their titles. Section numbers +or the equivalent are not considered part of the section titles. + +@item +Delete any section Entitled ``Endorsements''. Such a section +may not be included in the Modified Version. + +@item +Do not retitle any existing section to be Entitled ``Endorsements'' or +to conflict in title with any Invariant Section. + +@item +Preserve any Warranty Disclaimers. +@end enumerate + +If the Modified Version includes new front-matter sections or +appendices that qualify as Secondary Sections and contain no material +copied from the Document, you may at your option designate some or all +of these sections as invariant. To do this, add their titles to the +list of Invariant Sections in the Modified Version's license notice. +These titles must be distinct from any other section titles. + +You may add a section Entitled ``Endorsements'', provided it contains +nothing but endorsements of your Modified Version by various +parties---for example, statements of peer review or that the text has +been approved by an organization as the authoritative definition of a +standard. + +You may add a passage of up to five words as a Front-Cover Text, and a +passage of up to 25 words as a Back-Cover Text, to the end of the list +of Cover Texts in the Modified Version. Only one passage of +Front-Cover Text and one of Back-Cover Text may be added by (or +through arrangements made by) any one entity. If the Document already +includes a cover text for the same cover, previously added by you or +by arrangement made by the same entity you are acting on behalf of, +you may not add another; but you may replace the old one, on explicit +permission from the previous publisher that added the old one. + +The author(s) and publisher(s) of the Document do not by this License +give permission to use their names for publicity for or to assert or +imply endorsement of any Modified Version. + +@item +COMBINING DOCUMENTS + +You may combine the Document with other documents released under this +License, under the terms defined in section 4 above for modified +versions, provided that you include in the combination all of the +Invariant Sections of all of the original documents, unmodified, and +list them all as Invariant Sections of your combined work in its +license notice, and that you preserve all their Warranty Disclaimers. + +The combined work need only contain one copy of this License, and +multiple identical Invariant Sections may be replaced with a single +copy. If there are multiple Invariant Sections with the same name but +different contents, make the title of each such section unique by +adding at the end of it, in parentheses, the name of the original +author or publisher of that section if known, or else a unique number. +Make the same adjustment to the section titles in the list of +Invariant Sections in the license notice of the combined work. + +In the combination, you must combine any sections Entitled ``History'' +in the various original documents, forming one section Entitled +``History''; likewise combine any sections Entitled ``Acknowledgements'', +and any sections Entitled ``Dedications''. You must delete all +sections Entitled ``Endorsements.'' + +@item +COLLECTIONS OF DOCUMENTS + +You may make a collection consisting of the Document and other documents +released under this License, and replace the individual copies of this +License in the various documents with a single copy that is included in +the collection, provided that you follow the rules of this License for +verbatim copying of each of the documents in all other respects. + +You may extract a single document from such a collection, and distribute +it individually under this License, provided you insert a copy of this +License into the extracted document, and follow this License in all +other respects regarding verbatim copying of that document. + +@item +AGGREGATION WITH INDEPENDENT WORKS + +A compilation of the Document or its derivatives with other separate +and independent documents or works, in or on a volume of a storage or +distribution medium, is called an ``aggregate'' if the copyright +resulting from the compilation is not used to limit the legal rights +of the compilation's users beyond what the individual works permit. +When the Document is included in an aggregate, this License does not +apply to the other works in the aggregate which are not themselves +derivative works of the Document. + +If the Cover Text requirement of section 3 is applicable to these +copies of the Document, then if the Document is less than one half of +the entire aggregate, the Document's Cover Texts may be placed on +covers that bracket the Document within the aggregate, or the +electronic equivalent of covers if the Document is in electronic form. +Otherwise they must appear on printed covers that bracket the whole +aggregate. + +@item +TRANSLATION + +Translation is considered a kind of modification, so you may +distribute translations of the Document under the terms of section 4. +Replacing Invariant Sections with translations requires special +permission from their copyright holders, but you may include +translations of some or all Invariant Sections in addition to the +original versions of these Invariant Sections. You may include a +translation of this License, and all the license notices in the +Document, and any Warranty Disclaimers, provided that you also include +the original English version of this License and the original versions +of those notices and disclaimers. In case of a disagreement between +the translation and the original version of this License or a notice +or disclaimer, the original version will prevail. + +If a section in the Document is Entitled ``Acknowledgements'', +``Dedications'', or ``History'', the requirement (section 4) to Preserve +its Title (section 1) will typically require changing the actual +title. + +@item +TERMINATION + +You may not copy, modify, sublicense, or distribute the Document +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense, or distribute it is void, and +will automatically terminate your rights under this License. + +However, if you cease all violation of this License, then your license +from a particular copyright holder is reinstated (a) provisionally, +unless and until the copyright holder explicitly and finally +terminates your license, and (b) permanently, if the copyright holder +fails to notify you of the violation by some reasonable means prior to +60 days after the cessation. + +Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + +Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, receipt of a copy of some or all of the same material does +not give you any rights to use it. + +@item +FUTURE REVISIONS OF THIS LICENSE + +The Free Software Foundation may publish new, revised versions +of the GNU Free Documentation License from time to time. Such new +versions will be similar in spirit to the present version, but may +differ in detail to address new problems or concerns. See +@uref{http://www.gnu.org/copyleft/}. + +Each version of the License is given a distinguishing version number. +If the Document specifies that a particular numbered version of this +License ``or any later version'' applies to it, you have the option of +following the terms and conditions either of that specified version or +of any later version that has been published (not as a draft) by the +Free Software Foundation. If the Document does not specify a version +number of this License, you may choose any version ever published (not +as a draft) by the Free Software Foundation. If the Document +specifies that a proxy can decide which future versions of this +License can be used, that proxy's public statement of acceptance of a +version permanently authorizes you to choose that version for the +Document. + +@item +RELICENSING + +``Massive Multiauthor Collaboration Site'' (or ``MMC Site'') means any +World Wide Web server that publishes copyrightable works and also +provides prominent facilities for anybody to edit those works. A +public wiki that anybody can edit is an example of such a server. A +``Massive Multiauthor Collaboration'' (or ``MMC'') contained in the +site means any set of copyrightable works thus published on the MMC +site. + +``CC-BY-SA'' means the Creative Commons Attribution-Share Alike 3.0 +license published by Creative Commons Corporation, a not-for-profit +corporation with a principal place of business in San Francisco, +California, as well as future copyleft versions of that license +published by that same organization. + +``Incorporate'' means to publish or republish a Document, in whole or +in part, as part of another Document. + +An MMC is ``eligible for relicensing'' if it is licensed under this +License, and if all works that were first published under this License +somewhere other than this MMC, and subsequently incorporated in whole +or in part into the MMC, (1) had no cover texts or invariant sections, +and (2) were thus incorporated prior to November 1, 2008. + +The operator of an MMC Site may republish an MMC contained in the site +under CC-BY-SA on the same site at any time before August 1, 2009, +provided the MMC is eligible for relicensing. + +@end enumerate + +@page +@heading ADDENDUM: How to use this License for your documents + +To use this License in a document you have written, include a copy of +the License in the document and put the following copyright and +license notices just after the title page: + +@smallexample +@group + Copyright (C) @var{year} @var{your name}. + Permission is granted to copy, distribute and/or modify this document + under the terms of the GNU Free Documentation License, Version 1.3 + or any later version published by the Free Software Foundation; + with no Invariant Sections, no Front-Cover Texts, and no Back-Cover + Texts. A copy of the license is included in the section entitled ``GNU + Free Documentation License''. +@end group +@end smallexample + +If you have Invariant Sections, Front-Cover Texts and Back-Cover Texts, +replace the ``with@dots{}Texts.''@: line with this: + +@smallexample +@group + with the Invariant Sections being @var{list their titles}, with + the Front-Cover Texts being @var{list}, and with the Back-Cover Texts + being @var{list}. +@end group +@end smallexample + +If you have Invariant Sections without Cover Texts, or some other +combination of the three, merge those two alternatives to suit the +situation. + +If your document contains nontrivial examples of program code, we +recommend releasing these examples in parallel under your choice of +free software license, such as the GNU General Public License, +to permit their use in free software. + +@c Local Variables: +@c ispell-local-pdict: "ispell-dict" +@c End: diff --git a/doc/documentation/gendocs.sh b/doc/documentation/gendocs.sh new file mode 100755 index 0000000000..3b71b36a20 --- /dev/null +++ b/doc/documentation/gendocs.sh @@ -0,0 +1,504 @@ +#!/bin/sh -e +# gendocs.sh -- generate a GNU manual in many formats. This script is +# mentioned in maintain.texi. See the help message below for usage details. + +scriptversion=2016-12-31.18 + +# Copyright 2003-2017 Free Software Foundation, Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# +# Original author: Mohit Agarwal. +# Send bug reports and any other correspondence to bug-gnulib@gnu.org. +# +# The latest version of this script, and the companion template, is +# available from the Gnulib repository: +# +# http://git.savannah.gnu.org/cgit/gnulib.git/tree/build-aux/gendocs.sh +# http://git.savannah.gnu.org/cgit/gnulib.git/tree/doc/gendocs_template + +# TODO: +# - image importing was only implemented for HTML generated by +# makeinfo. But it should be simple enough to adjust. +# - images are not imported in the source tarball. All the needed +# formats (PDF, PNG, etc.) should be included. + +prog=`basename "$0"` +srcdir=`pwd` + +scripturl="http://git.savannah.gnu.org/cgit/gnulib.git/plain/build-aux/gendocs.sh" +templateurl="http://git.savannah.gnu.org/cgit/gnulib.git/plain/doc/gendocs_template" + +: ${SETLANG="env LANG= LC_MESSAGES= LC_ALL= LANGUAGE="} +: ${MAKEINFO="makeinfo"} +: ${TEXI2DVI="texi2dvi"} +: ${DOCBOOK2HTML="docbook2html"} +: ${DOCBOOK2PDF="docbook2pdf"} +: ${DOCBOOK2TXT="docbook2txt"} +: ${GENDOCS_TEMPLATE_DIR="."} +: ${PERL='perl'} +: ${TEXI2HTML="texi2html"} +unset CDPATH +unset use_texi2html + +MANUAL_TITLE= +PACKAGE= +EMAIL=webmasters@gnu.org # please override with --email +commonarg= # passed to all makeinfo/texi2html invcations. +dirargs= # passed to all tools (-I dir). +dirs= # -I directories. +htmlarg="--css-ref=/software/gnulib/manual.css -c TOP_NODE_UP_URL=/manual" +infoarg=--no-split +generate_ascii=true +generate_html=true +generate_info=true +generate_tex=true +outdir=manual +source_extra= +split=node +srcfile= +texarg="-t @finalout" + +version="gendocs.sh $scriptversion + +Copyright 2017 Free Software Foundation, Inc. +There is NO warranty. You may redistribute this software +under the terms of the GNU General Public License. +For more information about these matters, see the files named COPYING." + +usage="Usage: $prog [OPTION]... PACKAGE MANUAL-TITLE + +Generate output in various formats from PACKAGE.texinfo (or .texi or +.txi) source. See the GNU Maintainers document for a more extensive +discussion: + http://www.gnu.org/prep/maintain_toc.html + +Options: + --email ADR use ADR as contact in generated web pages; always give this. + + -s SRCFILE read Texinfo from SRCFILE, instead of PACKAGE.{texinfo|texi|txi} + -o OUTDIR write files into OUTDIR, instead of manual/. + -I DIR append DIR to the Texinfo search path. + --common ARG pass ARG in all invocations. + --html ARG pass ARG to makeinfo or texi2html for HTML targets, + instead of '$htmlarg'. + --info ARG pass ARG to makeinfo for Info, instead of --no-split. + --no-ascii skip generating the plain text output. + --no-html skip generating the html output. + --no-info skip generating the info output. + --no-tex skip generating the dvi and pdf output. + --source ARG include ARG in tar archive of sources. + --split HOW make split HTML by node, section, chapter; default node. + --tex ARG pass ARG to texi2dvi for DVI and PDF, instead of -t @finalout. + + --texi2html use texi2html to make HTML target, with all split versions. + --docbook convert through DocBook too (xml, txt, html, pdf). + + --help display this help and exit successfully. + --version display version information and exit successfully. + +Simple example: $prog --email bug-gnu-emacs@gnu.org emacs \"GNU Emacs Manual\" + +Typical sequence: + cd PACKAGESOURCE/doc + wget \"$scripturl\" + wget \"$templateurl\" + $prog --email BUGLIST MANUAL \"GNU MANUAL - One-line description\" + +Output will be in a new subdirectory \"manual\" (by default; +use -o OUTDIR to override). Move all the new files into your web CVS +tree, as explained in the Web Pages node of maintain.texi. + +Please use the --email ADDRESS option so your own bug-reporting +address will be used in the generated HTML pages. + +MANUAL-TITLE is included as part of the HTML <title> of the overall +manual/index.html file. It should include the name of the package being +documented. manual/index.html is created by substitution from the file +$GENDOCS_TEMPLATE_DIR/gendocs_template. (Feel free to modify the +generic template for your own purposes.) + +If you have several manuals, you'll need to run this script several +times with different MANUAL values, specifying a different output +directory with -o each time. Then write (by hand) an overall index.html +with links to them all. + +If a manual's Texinfo sources are spread across several directories, +first copy or symlink all Texinfo sources into a single directory. +(Part of the script's work is to make a tar.gz of the sources.) + +As implied above, by default monolithic Info files are generated. +If you want split Info, or other Info options, use --info to override. + +You can set the environment variables MAKEINFO, TEXI2DVI, TEXI2HTML, +and PERL to control the programs that get executed, and +GENDOCS_TEMPLATE_DIR to control where the gendocs_template file is +looked for. With --docbook, the environment variables DOCBOOK2HTML, +DOCBOOK2PDF, and DOCBOOK2TXT are also consulted. + +By default, makeinfo and texi2dvi are run in the default (English) +locale, since that's the language of most Texinfo manuals. If you +happen to have a non-English manual and non-English web site, see the +SETLANG setting in the source. + +Email bug reports or enhancement requests to bug-gnulib@gnu.org. +" + +while test $# -gt 0; do + case $1 in + -s) shift; srcfile=$1;; + -o) shift; outdir=$1;; + -I) shift; dirargs="$dirargs -I '$1'"; dirs="$dirs $1";; + --common) shift; commonarg=$1;; + --docbook) docbook=yes;; + --email) shift; EMAIL=$1;; + --html) shift; htmlarg=$1;; + --info) shift; infoarg=$1;; + --no-ascii) generate_ascii=false;; + --no-html) generate_ascii=false;; + --no-info) generate_info=false;; + --no-tex) generate_tex=false;; + --source) shift; source_extra=$1;; + --split) shift; split=$1;; + --tex) shift; texarg=$1;; + --texi2html) use_texi2html=1;; + + --help) echo "$usage"; exit 0;; + --version) echo "$version"; exit 0;; + -*) + echo "$0: Unknown option \`$1'." >&2 + echo "$0: Try \`--help' for more information." >&2 + exit 1;; + *) + if test -z "$PACKAGE"; then + PACKAGE=$1 + elif test -z "$MANUAL_TITLE"; then + MANUAL_TITLE=$1 + else + echo "$0: extra non-option argument \`$1'." >&2 + exit 1 + fi;; + esac + shift +done + +# makeinfo uses the dirargs, but texi2dvi doesn't. +commonarg=" $dirargs $commonarg" + +# For most of the following, the base name is just $PACKAGE +base=$PACKAGE + +if test -n "$srcfile"; then + # but here, we use the basename of $srcfile + base=`basename "$srcfile"` + case $base in + *.txi|*.texi|*.texinfo) base=`echo "$base"|sed 's/\.[texinfo]*$//'`;; + esac + PACKAGE=$base +elif test -s "$srcdir/$PACKAGE.texinfo"; then + srcfile=$srcdir/$PACKAGE.texinfo +elif test -s "$srcdir/$PACKAGE.texi"; then + srcfile=$srcdir/$PACKAGE.texi +elif test -s "$srcdir/$PACKAGE.txi"; then + srcfile=$srcdir/$PACKAGE.txi +else + echo "$0: cannot find .texinfo or .texi or .txi for $PACKAGE in $srcdir." >&2 + exit 1 +fi + +if test ! -r $GENDOCS_TEMPLATE_DIR/gendocs_template; then + echo "$0: cannot read $GENDOCS_TEMPLATE_DIR/gendocs_template." >&2 + echo "$0: it is available from $templateurl." >&2 + exit 1 +fi + +# Function to return size of $1 in something resembling kilobytes. +calcsize() +{ + size=`ls -ksl $1 | awk '{print $1}'` + echo $size +} + +# copy_images OUTDIR HTML-FILE... +# ------------------------------- +# Copy all the images needed by the HTML-FILEs into OUTDIR. +# Look for them in . and the -I directories; this is simpler than what +# makeinfo supports with -I, but hopefully it will suffice. +copy_images() +{ + local odir + odir=$1 + shift + $PERL -n -e " +BEGIN { + \$me = '$prog'; + \$odir = '$odir'; + @dirs = qw(. $dirs); +} +" -e ' +/<img src="(.*?)"/g && ++$need{$1}; + +END { + #print "$me: @{[keys %need]}\n"; # for debugging, show images found. + FILE: for my $f (keys %need) { + for my $d (@dirs) { + if (-f "$d/$f") { + use File::Basename; + my $dest = dirname ("$odir/$f"); + # + use File::Path; + -d $dest || mkpath ($dest) + || die "$me: cannot mkdir $dest: $!\n"; + # + use File::Copy; + copy ("$d/$f", $dest) + || die "$me: cannot copy $d/$f to $dest: $!\n"; + next FILE; + } + } + die "$me: $ARGV: cannot find image $f\n"; + } +} +' -- "$@" || exit 1 +} + +case $outdir in + /*) abs_outdir=$outdir;; + *) abs_outdir=$srcdir/$outdir;; +esac + +echo "Making output for $srcfile" +echo " in `pwd`" +mkdir -p "$outdir/" + +# +if $generate_info; then + cmd="$SETLANG $MAKEINFO -o $PACKAGE.info $commonarg $infoarg \"$srcfile\"" + echo "Generating info... ($cmd)" + rm -f $PACKAGE.info* # get rid of any strays + eval "$cmd" + tar czf "$outdir/$PACKAGE.info.tar.gz" $PACKAGE.info* + ls -l "$outdir/$PACKAGE.info.tar.gz" + info_tgz_size=`calcsize "$outdir/$PACKAGE.info.tar.gz"` + # do not mv the info files, there's no point in having them available + # separately on the web. +fi # end info + +# +if $generate_tex; then + cmd="$SETLANG $TEXI2DVI $dirargs $texarg \"$srcfile\"" + printf "\nGenerating dvi... ($cmd)\n" + eval "$cmd" + # compress/finish dvi: + gzip -f -9 $PACKAGE.dvi + dvi_gz_size=`calcsize $PACKAGE.dvi.gz` + mv $PACKAGE.dvi.gz "$outdir/" + ls -l "$outdir/$PACKAGE.dvi.gz" + + cmd="$SETLANG $TEXI2DVI --pdf $dirargs $texarg \"$srcfile\"" + printf "\nGenerating pdf... ($cmd)\n" + eval "$cmd" + pdf_size=`calcsize $PACKAGE.pdf` + mv $PACKAGE.pdf "$outdir/" + ls -l "$outdir/$PACKAGE.pdf" +fi # end tex (dvi + pdf) + +# +if $generate_ascii; then + opt="-o $PACKAGE.txt --no-split --no-headers $commonarg" + cmd="$SETLANG $MAKEINFO $opt \"$srcfile\"" + printf "\nGenerating ascii... ($cmd)\n" + eval "$cmd" + ascii_size=`calcsize $PACKAGE.txt` + gzip -f -9 -c $PACKAGE.txt >"$outdir/$PACKAGE.txt.gz" + ascii_gz_size=`calcsize "$outdir/$PACKAGE.txt.gz"` + mv $PACKAGE.txt "$outdir/" + ls -l "$outdir/$PACKAGE.txt" "$outdir/$PACKAGE.txt.gz" +fi + +# + +if $generate_html; then +# Split HTML at level $1. Used for texi2html. +html_split() +{ + opt="--split=$1 --node-files $commonarg $htmlarg" + cmd="$SETLANG $TEXI2HTML --output $PACKAGE.html $opt \"$srcfile\"" + printf "\nGenerating html by $1... ($cmd)\n" + eval "$cmd" + split_html_dir=$PACKAGE.html + ( + cd ${split_html_dir} || exit 1 + ln -sf ${PACKAGE}.html index.html + tar -czf "$abs_outdir/${PACKAGE}.html_$1.tar.gz" -- *.html + ) + eval html_$1_tgz_size=`calcsize "$outdir/${PACKAGE}.html_$1.tar.gz"` + rm -f "$outdir"/html_$1/*.html + mkdir -p "$outdir/html_$1/" + mv ${split_html_dir}/*.html "$outdir/html_$1/" + rmdir ${split_html_dir} +} + +if test -z "$use_texi2html"; then + opt="--no-split --html -o $PACKAGE.html $commonarg $htmlarg" + cmd="$SETLANG $MAKEINFO $opt \"$srcfile\"" + printf "\nGenerating monolithic html... ($cmd)\n" + rm -rf $PACKAGE.html # in case a directory is left over + eval "$cmd" + html_mono_size=`calcsize $PACKAGE.html` + gzip -f -9 -c $PACKAGE.html >"$outdir/$PACKAGE.html.gz" + html_mono_gz_size=`calcsize "$outdir/$PACKAGE.html.gz"` + copy_images "$outdir/" $PACKAGE.html + mv $PACKAGE.html "$outdir/" + ls -l "$outdir/$PACKAGE.html" "$outdir/$PACKAGE.html.gz" + + # Before Texinfo 5.0, makeinfo did not accept a --split=HOW option, + # it just always split by node. So if we're splitting by node anyway, + # leave it out. + if test "x$split" = xnode; then + split_arg= + else + split_arg=--split=$split + fi + # + opt="--html -o $PACKAGE.html $split_arg $commonarg $htmlarg" + cmd="$SETLANG $MAKEINFO $opt \"$srcfile\"" + printf "\nGenerating html by $split... ($cmd)\n" + eval "$cmd" + split_html_dir=$PACKAGE.html + copy_images $split_html_dir/ $split_html_dir/*.html + ( + cd $split_html_dir || exit 1 + tar -czf "$abs_outdir/$PACKAGE.html_$split.tar.gz" -- * + ) + eval \ + html_${split}_tgz_size=`calcsize "$outdir/$PACKAGE.html_$split.tar.gz"` + rm -rf "$outdir/html_$split/" + mv $split_html_dir "$outdir/html_$split/" + du -s "$outdir/html_$split/" + ls -l "$outdir/$PACKAGE.html_$split.tar.gz" + +else # use texi2html: + opt="--output $PACKAGE.html $commonarg $htmlarg" + cmd="$SETLANG $TEXI2HTML $opt \"$srcfile\"" + printf "\nGenerating monolithic html with texi2html... ($cmd)\n" + rm -rf $PACKAGE.html # in case a directory is left over + eval "$cmd" + html_mono_size=`calcsize $PACKAGE.html` + gzip -f -9 -c $PACKAGE.html >"$outdir/$PACKAGE.html.gz" + html_mono_gz_size=`calcsize "$outdir/$PACKAGE.html.gz"` + mv $PACKAGE.html "$outdir/" + + html_split node + html_split chapter + html_split section +fi +fi # end html + +# +printf "\nMaking .tar.gz for sources...\n" +d=`dirname $srcfile` +( + cd "$d" + srcfiles=`ls -d *.texinfo *.texi *.txi *.eps $source_extra 2>/dev/null` || true + tar czfh "$abs_outdir/$PACKAGE.texi.tar.gz" $srcfiles + ls -l "$abs_outdir/$PACKAGE.texi.tar.gz" +) +texi_tgz_size=`calcsize "$outdir/$PACKAGE.texi.tar.gz"` + +# +# Do everything again through docbook. +if test -n "$docbook"; then + opt="-o - --docbook $commonarg" + cmd="$SETLANG $MAKEINFO $opt \"$srcfile\" >${srcdir}/$PACKAGE-db.xml" + printf "\nGenerating docbook XML... ($cmd)\n" + eval "$cmd" + docbook_xml_size=`calcsize $PACKAGE-db.xml` + gzip -f -9 -c $PACKAGE-db.xml >"$outdir/$PACKAGE-db.xml.gz" + docbook_xml_gz_size=`calcsize "$outdir/$PACKAGE-db.xml.gz"` + mv $PACKAGE-db.xml "$outdir/" + + split_html_db_dir=html_node_db + opt="$commonarg -o $split_html_db_dir" + cmd="$DOCBOOK2HTML $opt \"${outdir}/$PACKAGE-db.xml\"" + printf "\nGenerating docbook HTML... ($cmd)\n" + eval "$cmd" + ( + cd ${split_html_db_dir} || exit 1 + tar -czf "$abs_outdir/${PACKAGE}.html_node_db.tar.gz" -- *.html + ) + html_node_db_tgz_size=`calcsize "$outdir/${PACKAGE}.html_node_db.tar.gz"` + rm -f "$outdir"/html_node_db/*.html + mkdir -p "$outdir/html_node_db" + mv ${split_html_db_dir}/*.html "$outdir/html_node_db/" + rmdir ${split_html_db_dir} + + cmd="$DOCBOOK2TXT \"${outdir}/$PACKAGE-db.xml\"" + printf "\nGenerating docbook ASCII... ($cmd)\n" + eval "$cmd" + docbook_ascii_size=`calcsize $PACKAGE-db.txt` + mv $PACKAGE-db.txt "$outdir/" + + cmd="$DOCBOOK2PDF \"${outdir}/$PACKAGE-db.xml\"" + printf "\nGenerating docbook PDF... ($cmd)\n" + eval "$cmd" + docbook_pdf_size=`calcsize $PACKAGE-db.pdf` + mv $PACKAGE-db.pdf "$outdir/" +fi + +# +printf "\nMaking index.html for $PACKAGE...\n" +if test -z "$use_texi2html"; then + CONDS="/%%IF *HTML_SECTION%%/,/%%ENDIF *HTML_SECTION%%/d;\ + /%%IF *HTML_CHAPTER%%/,/%%ENDIF *HTML_CHAPTER%%/d" +else + # should take account of --split here. + CONDS="/%%ENDIF.*%%/d;/%%IF *HTML_SECTION%%/d;/%%IF *HTML_CHAPTER%%/d" +fi + +curdate=`$SETLANG date '+%B %d, %Y'` +sed \ + -e "s!%%TITLE%%!$MANUAL_TITLE!g" \ + -e "s!%%EMAIL%%!$EMAIL!g" \ + -e "s!%%PACKAGE%%!$PACKAGE!g" \ + -e "s!%%DATE%%!$curdate!g" \ + -e "s!%%HTML_MONO_SIZE%%!$html_mono_size!g" \ + -e "s!%%HTML_MONO_GZ_SIZE%%!$html_mono_gz_size!g" \ + -e "s!%%HTML_NODE_TGZ_SIZE%%!$html_node_tgz_size!g" \ + -e "s!%%HTML_SECTION_TGZ_SIZE%%!$html_section_tgz_size!g" \ + -e "s!%%HTML_CHAPTER_TGZ_SIZE%%!$html_chapter_tgz_size!g" \ + -e "s!%%INFO_TGZ_SIZE%%!$info_tgz_size!g" \ + -e "s!%%DVI_GZ_SIZE%%!$dvi_gz_size!g" \ + -e "s!%%PDF_SIZE%%!$pdf_size!g" \ + -e "s!%%ASCII_SIZE%%!$ascii_size!g" \ + -e "s!%%ASCII_GZ_SIZE%%!$ascii_gz_size!g" \ + -e "s!%%TEXI_TGZ_SIZE%%!$texi_tgz_size!g" \ + -e "s!%%DOCBOOK_HTML_NODE_TGZ_SIZE%%!$html_node_db_tgz_size!g" \ + -e "s!%%DOCBOOK_ASCII_SIZE%%!$docbook_ascii_size!g" \ + -e "s!%%DOCBOOK_PDF_SIZE%%!$docbook_pdf_size!g" \ + -e "s!%%DOCBOOK_XML_SIZE%%!$docbook_xml_size!g" \ + -e "s!%%DOCBOOK_XML_GZ_SIZE%%!$docbook_xml_gz_size!g" \ + -e "s,%%SCRIPTURL%%,$scripturl,g" \ + -e "s!%%SCRIPTNAME%%!$prog!g" \ + -e "$CONDS" \ +$GENDOCS_TEMPLATE_DIR/gendocs_template >"$outdir/index.html" + +echo "Done, see $outdir/ subdirectory for new files." + +# Local variables: +# eval: (add-hook 'write-file-hooks 'time-stamp) +# time-stamp-start: "scriptversion=" +# time-stamp-format: "%:y-%02m-%02d.%02H" +# time-stamp-end: "$" +# End: diff --git a/doc/documentation/gendocs_template b/doc/documentation/gendocs_template new file mode 100644 index 0000000000..178f6cb4c8 --- /dev/null +++ b/doc/documentation/gendocs_template @@ -0,0 +1,91 @@ +<!--#include virtual="/server/header.html" --> +<!-- Parent-Version: 1.77 --> +<title>%%TITLE%% - GNU Project - Free Software Foundation</title> +<!--#include virtual="/server/banner.html" --> +<h2>%%TITLE%%</h2> + +<address>Free Software Foundation</address> +<address>last updated %%DATE%%</address> + +<p>This manual (%%PACKAGE%%) is available in the following formats:</p> + +<ul> +<li><a href="%%PACKAGE%%.html">HTML + (%%HTML_MONO_SIZE%%K bytes)</a> - entirely on one web page.</li> +<li><a href="html_node/index.html">HTML</a> - with one web page per + node.</li> +%%IF HTML_SECTION%% +<li><a href="html_section/index.html">HTML</a> - with one web page per + section.</li> +%%ENDIF HTML_SECTION%% +%%IF HTML_CHAPTER%% +<li><a href="html_chapter/index.html">HTML</a> - with one web page per + chapter.</li> +%%ENDIF HTML_CHAPTER%% +<li><a href="%%PACKAGE%%.html.gz">HTML compressed + (%%HTML_MONO_GZ_SIZE%%K gzipped characters)</a> - entirely on + one web page.</li> +<li><a href="%%PACKAGE%%.html_node.tar.gz">HTML compressed + (%%HTML_NODE_TGZ_SIZE%%K gzipped tar file)</a> - + with one web page per node.</li> +%%IF HTML_SECTION%% +<li><a href="%%PACKAGE%%.html_section.tar.gz">HTML compressed + (%%HTML_SECTION_TGZ_SIZE%%K gzipped tar file)</a> - + with one web page per section.</li> +%%ENDIF HTML_SECTION%% +%%IF HTML_CHAPTER%% +<li><a href="%%PACKAGE%%.html_chapter.tar.gz">HTML compressed + (%%HTML_CHAPTER_TGZ_SIZE%%K gzipped tar file)</a> - + with one web page per chapter.</li> +%%ENDIF HTML_CHAPTER%% +<li><a href="%%PACKAGE%%.info.tar.gz">Info document + (%%INFO_TGZ_SIZE%%K bytes gzipped tar file)</a>.</li> +<li><a href="%%PACKAGE%%.txt">ASCII text + (%%ASCII_SIZE%%K bytes)</a>.</li> +<li><a href="%%PACKAGE%%.txt.gz">ASCII text compressed + (%%ASCII_GZ_SIZE%%K bytes gzipped)</a>.</li> +<li><a href="%%PACKAGE%%.dvi.gz">TeX dvi file + (%%DVI_GZ_SIZE%%K bytes gzipped)</a>.</li> +<li><a href="%%PACKAGE%%.pdf">PDF file + (%%PDF_SIZE%%K bytes)</a>.</li> +<li><a href="%%PACKAGE%%.texi.tar.gz">Texinfo source + (%%TEXI_TGZ_SIZE%%K bytes gzipped tar file).</a></li> +</ul> + +<p>You can <a href="http://shop.fsf.org/">buy printed copies of +some manuals</a> (among other items) from the Free Software Foundation; +this helps support FSF activities.</p> + +<p>(This page generated by the <a href="%%SCRIPTURL%%">%%SCRIPTNAME%% +script</a>.)</p> + +<!-- If needed, change the copyright block at the bottom. In general, + all pages on the GNU web server should have the section about + verbatim copying. Please do NOT remove this without talking + with the webmasters first. + Please make sure the copyright date is consistent with the document + and that it is like this: "2001, 2002", not this: "2001-2002". --> +</div><!-- for id="content", starts in the include above --> +<!--#include virtual="/server/footer.html" --> +<div id="footer"> +<div class="unprintable"> + +<p>Please send general FSF & GNU inquiries to +<a href="mailto:gnu@gnu.org"><gnu@gnu.org></a>. +There are also <a href="/contact/">other ways to contact</a> +the FSF. Broken links and other corrections or suggestions can be sent +to <a href="mailto:%%EMAIL%%"><%%EMAIL%%></a>.</p> +</div> + +<p>Copyright © 2017 Free Software Foundation, Inc.</p> + +<p>This page is licensed under a <a rel="license" +href="http://creativecommons.org/licenses/by-nd/3.0/us/">Creative +Commons Attribution-NoDerivs 3.0 United States License</a>.</p> + +<!--#include virtual="/server/bottom-notes.html" --> + +</div> +</div> +</body> +</html> diff --git a/doc/documentation/gendocs_template_min b/doc/documentation/gendocs_template_min new file mode 100644 index 0000000000..112fa3bfb6 --- /dev/null +++ b/doc/documentation/gendocs_template_min @@ -0,0 +1,93 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" + "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> +<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en"> + +<head> +<title>%%TITLE%% - GNU Project - Free Software Foundation</title> +<meta http-equiv="content-type" content='text/html; charset=utf-8' /> +<link rel="stylesheet" type="text/css" href="/gnu.css" /> +</head> + +<body> + +<h3>%%TITLE%%</h3> + +<address>Free Software Foundation</address> +<address>last updated %%DATE%%</address> +<p> +<a href="/graphics/gnu-head.jpg"> + <img src="/graphics/gnu-head-sm.jpg" + alt=" [image of the head of a GNU] " width="129" height="122"/> +</a> +</p> +<hr /> + +<p>This manual (%%PACKAGE%%) is available in the following formats:</p> + +<ul> +<li><a href="%%PACKAGE%%.html">HTML + (%%HTML_MONO_SIZE%%K bytes)</a> - entirely on one web page.</li> +<li><a href="html_node/index.html">HTML</a> - with one web page per + node.</li> +%%IF HTML_SECTION%% +<li><a href="html_section/index.html">HTML</a> - with one web page per + section.</li> +%%ENDIF HTML_SECTION%% +%%IF HTML_CHAPTER%% +<li><a href="html_chapter/index.html">HTML</a> - with one web page per + chapter.</li> +%%ENDIF HTML_CHAPTER%% +<li><a href="%%PACKAGE%%.html.gz">HTML compressed + (%%HTML_MONO_GZ_SIZE%%K gzipped characters)</a> - entirely on + one web page.</li> +<li><a href="%%PACKAGE%%.html_node.tar.gz">HTML compressed + (%%HTML_NODE_TGZ_SIZE%%K gzipped tar file)</a> - + with one web page per node.</li> +%%IF HTML_SECTION%% +<li><a href="%%PACKAGE%%.html_section.tar.gz">HTML compressed + (%%HTML_SECTION_TGZ_SIZE%%K gzipped tar file)</a> - + with one web page per section.</li> +%%ENDIF HTML_SECTION%% +%%IF HTML_CHAPTER%% +<li><a href="%%PACKAGE%%.html_chapter.tar.gz">HTML compressed + (%%HTML_CHAPTER_TGZ_SIZE%%K gzipped tar file)</a> - + with one web page per chapter.</li> +%%ENDIF HTML_CHAPTER%% +<li><a href="%%PACKAGE%%.info.tar.gz">Info document + (%%INFO_TGZ_SIZE%%K bytes gzipped tar file)</a>.</li> +<li><a href="%%PACKAGE%%.txt">ASCII text + (%%ASCII_SIZE%%K bytes)</a>.</li> +<li><a href="%%PACKAGE%%.txt.gz">ASCII text compressed + (%%ASCII_GZ_SIZE%%K bytes gzipped)</a>.</li> +<li><a href="%%PACKAGE%%.dvi.gz">TeX dvi file + (%%DVI_GZ_SIZE%%K bytes gzipped)</a>.</li> +<li><a href="%%PACKAGE%%.pdf">PDF file + (%%PDF_SIZE%%K bytes)</a>.</li> +<li><a href="%%PACKAGE%%.texi.tar.gz">Texinfo source + (%%TEXI_TGZ_SIZE%%K bytes gzipped tar file).</a></li> +</ul> + +<p>(This page generated by the <a href="%%SCRIPTURL%%">%%SCRIPTNAME%% +script</a>.)</p> + +<div id="footer" class="copyright"> + +<p>Please send general FSF & GNU inquiries to +<a href="mailto:gnu@gnu.org"><gnu@gnu.org></a>. +There are also <a href="/contact/">other ways to contact</a> +the FSF. Broken links and other corrections or suggestions can be sent +to <a href="mailto:%%EMAIL%%"><%%EMAIL%%></a>.</p> +</div> + +<p>Copyright © 2017 Free Software Foundation, Inc.</p> + +<p>This page is licensed under a <a rel="license" +href="http://creativecommons.org/licenses/by-nd/3.0/us/">Creative +Commons Attribution-NoDerivs 3.0 United States License</a>.</p> + +<!--#include virtual="/server/bottom-notes.html" --> + +</div> +</body> +</html> diff --git a/doc/documentation/gnunet-c-tutorial.texi b/doc/documentation/gnunet-c-tutorial.texi new file mode 100644 index 0000000000..f39c7de642 --- /dev/null +++ b/doc/documentation/gnunet-c-tutorial.texi @@ -0,0 +1,1540 @@ +\input texinfo +@c %**start of header +@setfilename gnunet-c-tutorial.info +@documentencoding UTF-8 +@settitle GNUnet C Tutorial +@exampleindent 2 +@c %**end of header + +@c including 'version.texi' makes makeinfo throw errors. +@include version2.texi + +@copying +Copyright @copyright{} 2001-2017 GNUnet e.V. + +Permission is granted to copy, distribute and/or modify this document +under the terms of the GNU Free Documentation License, Version 1.3 or +any later version published by the Free Software Foundation; with no +Invariant Sections, no Front-Cover Texts, and no Back-Cover Texts. A +copy of the license is included in the section entitled ``GNU Free +Documentation License''. + +A copy of the license is also available from the Free Software +Foundation Web site at @url{http://www.gnu.org/licenses/fdl.html}. + +Alternately, this document is also available under the General +Public License, version 3 or later, as published by the Free Software +Foundation. A copy of the license is included in the section entitled +``GNU General Public License''. + +A copy of the license is also available from the Free Software +Foundation Web site at @url{http://www.gnu.org/licenses/gpl.html}. +@end copying + +@dircategory Tutorial +@direntry +* GNUnet-C-Tutorial: (gnunet-c-tutorial). C Tutorial for GNunet +@end direntry + + +@titlepage +@title GNUnet C Tutorial +@subtitle A Tutorial for GNUnet @value{VERSION} (C version) +@author The GNUnet Developers + +@page +@vskip 0pt plus 1filll + +@insertcopying +@end titlepage + +@contents + +@c **** TODO +@c 1. Update content? +@c 2. Either reference main documentation or +@c 3. Merge this into main documentation + +@node Top +@top Introduction + +This tutorials explains how to install GNUnet on a +GNU/Linux system and gives an introduction on how +GNUnet can be used to develop a Peer-to-Peer application. +Detailed installation instructions for +various operating systems and a detailed list of all +dependencies can be found on our website at +@uref{https://gnunet.org/installation} and in our +Reference Documentation (GNUnet Handbook). + +Please read this tutorial carefully since every single step is +important and do not hesitate to contact the GNUnet team if you have +any questions or problems! Check here how to contact the GNUnet +team: @uref{https://gnunet.org/contact_information} + +@menu + +* Installing GNUnet:: Installing GNUnet +* Introduction to GNUnet Architecture:: Introduction to GNUnet Architecture +* First Steps with GNUnet:: First Steps with GNUnet +* Developing Applications:: Developing Applications + +@detailmenu + --- The Detailed Node Listing --- + +Installing GNUnet + +* Obtaining a stable version:: +* Installing Build Tool Chain and Dependencies:: +* Obtaining the latest version from Git:: +* Compiling and Installing GNUnet:: +* Common Issues - Check your GNUnet installation:: + +Introduction to GNUnet Architecture + +First Steps with GNUnet + +* Configure your peer:: +* Start a peer:: +* Monitor a peer:: +* Starting Two Peers by Hand:: +* Starting Peers Using the Testbed Service:: + +Developing Applications + +* gnunet-ext:: +* Adapting the Template:: +* Writing a Client Application:: +* Writing a Service:: +* Interacting directly with other Peers using the CORE Service:: +* Storing peer-specific data using the PEERSTORE service:: +* Using the DHT:: +* Debugging with gnunet-arm:: + +@end detailmenu +@end menu + +@node Installing GNUnet +@chapter Installing GNUnet + +First of all you have to install a current version of GNUnet. +You can download a tarball of a stable version from GNU FTP mirrors +or obtain the latest development version from our Git repository. + +Most of the time you should prefer to download the stable version +since with the latest development version things can be broken, +functionality can be changed or tests can fail. You should only use +the development version if you know that you require a certain +feature or a certain issue has been fixed since the last release. + +@menu +* Obtaining a stable version:: +* Installing Build Tool Chain and Dependencies:: +* Obtaining the latest version from Git:: +* Compiling and Installing GNUnet:: +* Common Issues - Check your GNUnet installation:: +@end menu + +@node Obtaining a stable version +@section Obtaining a stable version + +Download the tarball from +@indicateurl{https://ftp.gnu.org/gnu/gnunet/gnunet-@value{VERSION}.tar.gz}. + +Make sure to download the associated @file{.sig} file and to verify the +authenticity of the tarball against it, like this: + +@example +$ wget https://ftp.gnu.org/gnu/gnunet/gnunet-@value{VERSION}.tar.gz.sig +$ gpg --verify-files gnunet-@value{VERSION}.tar.gz.sig +@end example + +@noindent +If this command fails because you do not have the required public key, +then you need to run this command to import it: + +@example +$ gpg --keyserver keys.gnupg.net --recv-keys 48426C7E +@end example + +@noindent +and rerun the @code{gpg --verify-files} command. + +@b{Note:}@ +@b{The pub key to sign the 0.10.1 release has been +revoked}. You will get an error message stating that +@b{there is no known public key or that it has been revoked}. +The next release of GNUnet will have a valid signature +again. We are sorry for the inconvenience this causes. +Another possible source you could use is our +"gnunet" git repository which has mandatory signed commits +by every developer. + +Now you can extract the tarball and rename the resulting +directory to @file{gnunet} which we will be using in the +remainder of this document. + +@example +$ tar xvzf gnunet-@value{VERSION}.tar.gz +$ mv gnunet-@value{VERSION} gnunet +$ cd gnunet +@end example + +@noindent +However, please note that stable versions can be very outdated. +As a developer you are @b{strongly} encouraged to use the version +from @uref{https://gnunet.org/git/, git}. + +@node Installing Build Tool Chain and Dependencies +@section Installing Build Tool Chain and Dependencies + +To successfully compile GNUnet, you need the tools to build GNUnet and +the required dependencies. Please take a look at the +GNUnet Reference Documentation +(@pxref{Dependencies, The GNUnet Reference Documentation,, gnunet, The GNUnet Reference Documentation}) +for a list of required dependencies +and +(@pxref{Generic installation instructions, The GNUnet Reference Documentation,, gnunet, The GNUnet Reference Documentation}) +read its Installation chapter for specific instructions for +your operating system. +Please check the notes at the end of the configure process about +required dependencies. + +For GNUnet bootstrapping support and the HTTP(S) plugin you should +install @uref{https://gnunet.org/gnurl, libgnurl}. +For the filesharing service you should install at least one of the +datastore backends. MySQL, SQlite and PostgreSQL are supported. + +@node Obtaining the latest version from Git +@section Obtaining the latest version from Git + +The latest development version can obtained from our Git repository. +To obtain the code you need to have @code{Git} installed, which is +required for obtaining the repository via: + +@example +$ git clone https://gnunet.org/git/gnunet +@end example + +@noindent +After cloning the repository you have to execute the @file{bootstrap} +script in the new directory: + +@example +$ cd gnunet +$ ./bootstrap +@end example + +@noindent +The remainder of this tutorial will assume that you have the +Git branch ``master'' checked out. + +@node Compiling and Installing GNUnet +@section Compiling and Installing GNUnet + +Note: This section is a duplication of the more in depth +@pxref{GNUnet Installation Handbook, The GNUnet Reference Documentation,, gnunet, The GNUnet Reference Documentation}. + +First, you need to install libgnupgerror @geq{} 1.27 and +libgcrypt @geq{} 1.7.6: + +@example +$ export GNUPGFTP="https://www.gnupg.org/ftp/gcrypt" +$ wget $GNUPGFTP/libgpg-error/libgpg-error-1.27.tar.bz2 +$ tar xf libgpg-error-1.27.tar.bz2 +$ cd libgpg-error-1.27 +$ ./configure +$ sudo make install +$ cd .. +@end example + +@example +$ export GNUPGFTP="https://www.gnupg.org/ftp/gcrypt" +$ wget $GNUPGFTP/libgcrypt/libgcrypt-1.7.6.tar.bz2 +$ tar xf libgcrypt-1.7.6.tar.bz2 +$ cd libgcrypt-1.7.6 +$ ./configure +$ sudo make install +$ cd .. +@end example + +@menu +* Installation:: +@end menu + +@node Installation +@subsection Installation +Assuming all dependencies are installed, the following commands will +compile and install GNUnet in your home directory. You can specify the +directory where GNUnet will be installed by changing the +@code{--prefix} value when calling @command{./configure}. If +you do not specifiy a prefix, GNUnet is installed in the directory +@file{/usr/local}. When developing new applications you may want +to enable verbose logging by adding @code{--enable-logging=verbose}: + +@example +$ ./configure --prefix=$PREFIX --enable-logging +$ make +$ make install +@end example + +@noindent +After installing GNUnet you have to add your GNUnet installation +to your path environmental variable. In addition you have to +create the @file{.config} directory in your home directory +(unless it already exists) where GNUnet stores its data and an +empty GNUnet configuration file: + +@example +$ export PATH=$PATH:$PREFIX/bin +$ echo export PATH=$PREFIX/bin:\\$PATH >> ~/.bashrc +$ mkdir ~/.config/ +$ touch ~/.config/gnunet.conf +@end example + +@node Common Issues - Check your GNUnet installation +@section Common Issues - Check your GNUnet installation + +You should check your installation to ensure that installing GNUnet +was successful up to this point. You should be able to access GNUnet's +binaries and run GNUnet's self check. + +@example +$ which gnunet-arm +@end example + +@noindent +should return $PREFIX/bin/gnunet-arm. It should be located in your +GNUnet installation and the output should not be empty. +If you see an output like: + +@example +$ which gnunet-arm +@end example + +@noindent +check your PATH variable to ensure GNUnet's @file{bin} directory is +included. + +GNUnet provides tests for all of its subcomponents. Run + +@example +$ make check +@end example + +@noindent +to execute tests for all components. @command{make check} traverses all +subdirectories in @file{src}. For every subdirectory you should +get a message like this: + +@example +make[2]: Entering directory `/home/$USER/gnunet/contrib' +PASS: test_gnunet_prefix +============= +1 test passed +============= +@end example + +@node Introduction to GNUnet Architecture +@chapter Introduction to GNUnet Architecture + +GNUnet is organized in layers and services. Each service is composed of a +main service implementation and a client library for other programs to use +the service's functionality, described by an API. +@c This approach is shown in +@c FIXME: enable this once the commented block below works: +@c figure~\ref fig:service. +Some services provide an additional command line tool to enable the user +to interact with the service. + +Very often it is other GNUnet services that will use these APIs to build +the higher layers of GNUnet on top of the lower ones. Each layer expands +or extends the functionality of the service below (for instance, to build +a mesh on top of a DHT). +@c FXIME: See comment above. +@c See figure ~\ref fig:interaction for an illustration of this approach. + +@c ** @image filename[, width[, height[, alttext[, extension]]]] +@c FIXME: Texlive (?) 20112 makes the assumption that this means +@c 'images/OBJECTNAME.txt' but later versions of it (2017) use this +@c syntax as described below. +@c TODO: Checkout the makedoc script Guile uses. + +@c FIXME!!! +@c @image{images/gnunet-tutorial-service,,5in,Service with API and network protocol,.png} +@c @image{images/gnunet-tutorial-system,,5in,The layered system architecture of GNUnet,.png} + +@c \begin{figure}[!h] +@c \begin{center} +@c % \begin{subfigure} +@c \begin{subfigure}[b]{0.3\textwidth} +@c \centering +@c \includegraphics[width=\textwidth]{figs/Service.pdf} +@c \caption{Service with API and network protocol} +@c \label{fig:service} +@c \end{subfigure} +@c ~~~~~~~~~~ +@c \begin{subfigure}[b]{0.3\textwidth} +@c \centering +@c \includegraphics[width=\textwidth]{figs/System.pdf} +@c \caption{Service interaction} +@c \label{fig:interaction} +@c \end{subfigure} +@c \end{center} +@c \caption{GNUnet's layered system architecture} +@c \end{figure} + +The main service implementation runs as a standalone process in the +operating system and the client code runs as part of the client program, +so crashes of a client do not affect the service process or other clients. +The service and the clients communicate via a message protocol to be +defined and implemented by the programmer. + +@node First Steps with GNUnet +@chapter First Steps with GNUnet + +@menu +* Configure your peer:: +* Start a peer:: +* Monitor a peer:: +* Starting Two Peers by Hand:: +* Starting Peers Using the Testbed Service:: +@end menu + +@node Configure your peer +@section Configure your peer + +First of all we need to configure your peer. Each peer is started with +a configuration containing settings for GNUnet itself and its services. +This configuration is based on the default configuration shipped with +GNUnet and can be modified. The default configuration is located in the +@file{$PREFIX/share/gnunet/config.d} directory. When starting a peer, you +can specify a customized configuration using the the @command{-c} command +line switch when starting the ARM service and all other services. When +using a modified configuration the default values are loaded and only +values specified in the configuration file will replace the default +values. + +Since we want to start additional peers later, we need some modifications +from the default configuration. We need to create a separate service +home and a file containing our modifications for this peer: + +@example +$ mkdir ~/gnunet1/ +$ touch peer1.conf +@end example + +@noindent +Now add the following lines to @file{peer1.conf} to use this directory. +For simplified usage we want to prevent the peer to connect to the GNUnet +network since this could lead to confusing output. This modifications +will replace the default settings: + +@example +[PATHS] +# Use this directory to store GNUnet data +GNUNET_HOME = ~/gnunet1/ +[hostlist] +# prevent bootstrapping +SERVERS = +@end example + +@node Start a peer +@section Start a peer +Each GNUnet instance (called peer) has an identity (peer ID) based on a +cryptographic public private key pair. The peer ID is the printable hash +of the public key. + +GNUnet services are controlled by a master service, the so called +@dfn{Automatic Restart Manager} (ARM). ARM starts, stops and even +restarts services automatically or on demand when a client connects. +You interact with the ARM service using the @command{gnunet-arm} tool. +GNUnet can then be started with @command{gnunet-arm -s} and stopped with +@command{gnunet-arm -e}. An additional service not automatically started +can be started using @command{gnunet-arm -i <service name>} and stopped +using @command{gnunet-arm -k <servicename>}. + +Once you have started your peer, you can use many other GNUnet commands +to interact with it. For example, you can run: + +@example +$ gnunet-peerinfo -s +@end example + +@noindent +to obtain the public key of your peer. + +You should see an output containing the peer ID similar to: + +@example +I am peer `0PA02UVRKQTS2C .. JL5Q78F6H0B1ACPV1CJI59MEQUMQCC5G'. +@end example + +@node Monitor a peer +@section Monitor a peer + +In this section, we will monitor the behaviour of our peer's DHT +service with respect to a specific key. First we will start +GNUnet and then start the DHT service and use the DHT monitor tool +to monitor the PUT and GET commands we issue ussing the +@command{gnunet-dht-put} and @command{gnunet-dht-get} commands. +Using the ``monitor'' line given below, you can observe the behavior +of your own peer's DHT with respect to the specified KEY: + +@example +# start gnunet with all default services: +$ gnunet-arm -c ~/peer1.conf -s +# start DHT service: +$ gnunet-arm -c ~/peer1.conf -i dht +$ cd ~/gnunet/src/dht; +$ ./gnunet-dht-monitor -c ~/peer1.conf -k KEY +@end example + +@noindent +Now open a separate terminal and change again to +the @file{gnunet/src/dht} directory: + +@example +$ cd ~/gnunet/src/dht +# put VALUE under KEY in the DHT: +$ ./gnunet-dht-put -c ~/peer1.conf -k KEY -d VALUE +# get key KEY from the DHT: +$ ./gnunet/src/dht/gnunet-dht-get -c ~/peer1.conf -k KEY +# print statistics about current GNUnet state: +$ gnunet-statistics -c ~/peer1.conf +# print statistics about DHT service: +$ gnunet-statistics -c ~/peer1.conf -s dht +@end example + +@node Starting Two Peers by Hand +@section Starting Two Peers by Hand + +This section describes how to start two peers on the same machine by hand. +The process is rather painful, but the description is somewhat +instructive. In practice, you might prefer the automated method +(@pxref{Starting Peers Using the Testbed Service}). + +@menu +* Setup a second peer:: +* Start the second peer and connect the peers:: +* How to connect manually:: +@end menu + +@node Setup a second peer +@subsection Setup a second peer +We will now start a second peer on your machine. +For the second peer, you will need to manually create a modified +configuration file to avoid conflicts with ports and directories. +A peers configuration file is by default located +in @file{~/.gnunet/gnunet.conf}. This file is typically very short +or even empty as only the differences to the defaults need to be +specified. The defaults are located in many files in the +@file{$PREFIX/share/gnunet/config.d} directory. + +To configure the second peer, use the files +@file{$PREFIX/share/gnunet/config.d} as a template for your main +configuration file: + +@example +$ cat $PREFIX/share/gnunet/config.d/*.conf > peer2.conf +@end example + +@noindent +Now you have to edit @file{peer2.conf} and change: + +@itemize +@item @code{GNUNET\_TEST\_HOME} under @code{PATHS} +@item Every (uncommented) value for ``@code{PORT}'' (add 10000) in any +section (the option may be commented out if @code{PORT} is +prefixed by "\#", in this case, UNIX domain sockets are used +and the PORT option does not need to be touched) +@item Every value for ``@code{UNIXPATH}'' in any section +(e.g. by adding a "-p2" suffix) +@end itemize + +to a fresh, unique value. Make sure that the PORT numbers stay +below 65536. From now on, whenever you interact with the second peer, +you need to specify @command{-c peer2.conf} as an additional +command line argument. + +Now, generate the 2nd peer's private key: + +@example +$ gnunet-peerinfo -s -c peer2.conf +@end example + +@noindent +This may take a while, generate entropy using your keyboard or mouse +as needed. Also, make sure the output is different from the +gnunet-peerinfo output for the first peer (otherwise you made an +error in the configuration). + +@node Start the second peer and connect the peers +@subsection Start the second peer and connect the peers + +Then, you can start a second peer using: + +@example +$ gnunet-arm -c peer2.conf -s +$ gnunet-arm -c peer2.conf -i dht +$ ~/gnunet/src/dht/gnunet-dht-put -c peer2.conf -k KEY -d VALUE +$ ~/gnunet/src/dht/gnunet-dht-get -c peer2.conf -k KEY +@end example + +If you want the two peers to connect, you have multiple options: + +@itemize +@item UDP neighbour discovery (automatic) +@item Setup a bootstrap server +@item Connect manually +@end itemize + +To setup peer 1 as bootstrapping server change the configuration of +the first one to be a hostlist server by adding the following lines to +@file{peer1.conf} to enable bootstrapping server: + +@example +[hostlist] +OPTIONS = -p +@end example + +@noindent +Then change @file{peer2.conf} and replace the ``@code{SERVERS}'' +line in the ``@code{[hostlist]}'' section with +``@code{http://localhost:8080/}''. Restart both peers using: + +@example +# stop first peer +$ gnunet-arm -c peer1.conf -e +# start first peer +$ gnunet-arm -c peer1.conf -s +# start second peer +$ gnunet-arm -c peer2.conf -s +@end example + +@noindent +Note that if you start your peers without changing these settings, they +will use the ``global'' hostlist servers of the GNUnet P2P network and +likely connect to those peers. At that point, debugging might become +tricky as you're going to be connected to many more peers and would +likely observe traffic and behaviors that are not explicitly controlled +by you. + +@node How to connect manually +@subsection How to connect manually + +If you want to use the @code{peerinfo} tool to connect your +peers, you should: + +@itemize +@item Set @code{FORCESTART = NO} in section @code{hostlist} +(to not connect to the global GNUnet) +@item Start both peers running @command{gnunet-arm -c peer1.conf -s} +and @command{gnunet-arm -c peer2.conf -s} +@item Get @code{HELLO} message of the first peer running +@command{gnunet-peerinfo -c peer1.conf -g} +@item Give the output to the second peer by running +@command{gnunet-peerinfo -c peer2.conf -p '<output>'} +@end itemize + +Check that they are connected using @command{gnunet-core -c peer1.conf}, +which should give you the other peer's peer identity: + +@example +$ gnunet-core -c peer1.conf +Peer `9TVUCS8P5A7ILLBGO6 [...shortened...] 1KNBJ4NGCHP3JPVULDG' +@end example + +@node Starting Peers Using the Testbed Service +@section Starting Peers Using the Testbed Service +@c \label{sec:testbed} + +GNUnet's testbed service is used for testing scenarios where +a number of peers are to be started. The testbed can manage peers +on a single host or on multiple hosts in a distributed fashion. +On a single affordable computer, it should be possible to run +around tens of peers without drastically increasing the load on the +system. + +The testbed service can be access through its API +@file{include/gnunet\_testbed\_service.h}. The API provides many +routines for managing a group of peers. It also provides a helper +function @code{GNUNET\_TESTBED\_test\_run()} to quickly setup a +minimalistic testing environment on a single host. + +This function takes a configuration file which will be used as a +template configuration for the peers. The testbed takes care of +modifying relevant options in the peers' configuration such as +@code{SERVICEHOME}, @code{PORT}, @code{UNIXPATH} to unique values +so that peers run without running into conflicts. It also checks +and assigns the ports in configurations only if they are free. + +Additionally, the testbed service also reads its options from the +same configuration file. Various available options and details +about them can be found in the testbed default configuration file +@file{src/testbed/testbed.conf}. + +With the testbed API, a sample test case can be structured as follows: + +@example +@verbatiminclude testbed_test.c +@end example + +@noindent +The source code for the above listing can be found at +@uref{https://gnunet.org/git/gnunet.git/tree/doc/ +documentation/testbed_test.c} +or in the @file{doc/documentation/} folder of your repository check-out. +After installing GNUnet, the above source code can be compiled as: + +@example +$ export CPPFLAGS="-I/path/to/gnunet/headers" +$ export LDFLAGS="-L/path/to/gnunet/libraries" +$ gcc $CPPFLAGS $LDFLAGS -o testbed-test testbed_test.c \ + -lgnunettestbed -lgnunetdht -lgnunetutil +# Generate (empty) configuration +$ touch template.conf +# run it (press CTRL-C to stop) +$ ./testbed-test +@end example + +@noindent +The @code{CPPFLAGS} and @code{LDFLAGS} are necessary if GNUnet +is installed into a different directory other than @file{/usr/local}. + +All of testbed API's peer management functions treat management +actions as operations and return operation handles. It is expected +that the operations begin immediately, but they may get delayed (to +balance out load on the system). The program using the API then has +to take care of marking the operation as ``done'' so that its +associated resources can be freed immediately and other waiting +operations can be executed. Operations will be canceled if they are +marked as ``done'' before their completion. + +An operation is treated as completed when it succeeds or fails. +Completion of an operation is either conveyed as events through +@dfn{controller event callback} or through respective +@dfn{operation completion callbacks}. +In functions which support completion notification +through both controller event callback and operation +completion callback, first the controller event callback will be +called. If the operation is not marked as done in that callback +or if the callback is given as NULL when creating the operation, +the operation completion callback will be called. The API +documentation shows which event are to be expected in the +controller event notifications. It also documents any exceptional +behaviour. + +Once the peers are started, test cases often need to connect +some of the peers' services. Normally, opening a connect to +a peer's service requires the peer's configuration. While using +testbed, the testbed automatically generates per-peer configuration. +Accessing those configurations directly through file system is +discouraged as their locations are dynamically created and will be +different among various runs of testbed. To make access to these +configurations easy, testbed API provides the function +@code{GNUNET\_TESTBED\_service\_connect()}. This function fetches +the configuration of a given peer and calls the @dfn{Connect Adapter}. +In the example code, it is the @code{dht\_ca}. A connect adapter is +expected to open the connection to the needed service by using the +provided configuration and return the created service connection handle. +Successful connection to the needed service is signaled through +@code{service\_connect\_comp\_cb}. + +A dual to connect adapter is the @dfn{Disconnect Adapter}. This callback +is called after the connect adapter has been called when the operation +from @code{GNUNET\_TESTBED\_service\_connect()} is marked as ``done''. +It has to disconnect from the service with the provided service +handle (@code{op\_result}). + +Exercise: Find out how many peers you can run on your system. + +Exercise: Find out how to create a 2D torus topology by changing the +options in the configuration file. +@xref{Supported Topologies, The GNUnet Reference Documentation ,, gnunet, The GNUnet Reference Documentation}, +then use the DHT API to store and retrieve values in the network. + +@node Developing Applications +@chapter Developing Applications + +@menu +* gnunet-ext:: +* Adapting the Template:: +* Writing a Client Application:: +* Writing a Service:: +* Interacting directly with other Peers using the CORE Service:: +* Storing peer-specific data using the PEERSTORE service:: +* Using the DHT:: +* Debugging with gnunet-arm:: +@end menu + +@node gnunet-ext +@section gnunet-ext +To develop a new peer-to-peer application or to extend GNUnet we provide +a template build system for writing GNUnet extensions in C. It can be +obtained as follows: + +@example +$ git clone https://gnunet.org/git/gnunet-ext +$ cd gnunet-ext/ +$ ./bootstrap +$ ./configure --prefix=$PREFIX --with-gnunet=$PREFIX +$ make +$ make install +$ make check +@end example + +@noindent +The GNUnet ext template includes examples and a working buildsystem +for a new GNUnet service. A common GNUnet service consists of the +following parts which will be discussed in detail in the remainder +of this document. The functionality of a GNUnet service is implemented in: + +@itemize +@item the GNUnet service (gnunet-ext/src/ext/gnunet-service-ext.c) +@item the client API (gnunet-ext/src/ext/ext_api.c) +@item the client application using the service API +(gnunet-ext/src/ext/gnunet-ext.c) +@end itemize + +The interfaces for these entities are defined in: + +@itemize +@item client API interface (gnunet-ext/src/ext/ext.h) +@item the service interface (gnunet-ext/src/include/gnunet_service_SERVICE.h) +@item the P2P protocol (gnunet-ext/src/include/gnunet_protocols_ext.h) +@end itemize + + +In addition the ext systems provides: + +@itemize +@item a test testing the API (gnunet-ext/src/ext/test_ext_api.c) +@item a configuration template for the service +(gnunet-ext/src/ext/ext.conf.in) +@end itemize + +@node Adapting the Template +@section Adapting the Template + +The first step for writing any extension with a new service is to +ensure that the @file{ext.conf.in} file contains entries for the +@code{UNIXPATH}, @code{PORT} and @code{BINARY} for the service in a +section named after the service. + +If you want to adapt the template rename the @file{ext.conf.in} to +match your services name, you have to modify the @code{AC\_OUTPUT} +section in @file{configure.ac} in the @file{gnunet-ext} root. + +@node Writing a Client Application +@section Writing a Client Application + +When writing any client application (for example, a command-line +tool), the basic structure is to start with the +@code{GNUNET\_PROGRAM\_run} function. This function will parse +command-line options, setup the scheduler and then invoke the +@code{run} function (with the remaining non-option arguments) +and a handle to the parsed configuration (and the configuration +file name that was used, which is typically not needed): + +@example +@verbatiminclude tutorial-examples/001.c +@end example + +@menu +* Handling command-line options:: +* Writing a Client Library:: +* Writing a user interface:: +@end menu + +@node Handling command-line options +@subsection Handling command-line options + +Options can then be added easily by adding global variables and +expanding the @code{options} array. For example, the following would +add a string-option and a binary flag (defaulting to @code{NULL} and +@code{GNUNET\_NO} respectively): + +@example +@verbatiminclude tutorial-examples/002.c +@end example + +Issues such as displaying some helpful text describing options using +the @code{--help} argument and error handling are taken care of when +using this approach. Other @code{GNUNET\_GETOPT\_}-functions can be used +to obtain integer value options, increment counters, etc. You can +even write custom option parsers for special circumstances not covered +by the available handlers. To check if an argument was specified by the +user you initialize the variable with a specific value (e.g. NULL for +a string and GNUNET\_SYSERR for a integer) and check after parsing +happened if the values were modified. + +Inside the @code{run} method, the program would perform the +application-specific logic, which typically involves initializing and +using some client library to interact with the service. The client +library is supposed to implement the IPC whereas the service provides +more persistent P2P functions. + +Exercise: Add a few command-line options and print them inside +of @code{run}. What happens if the user gives invalid arguments? + +@node Writing a Client Library +@subsection Writing a Client Library + +The first and most important step in writing a client library is to +decide on an API for the library. Typical API calls include +connecting to the service, performing application-specific requests +and cleaning up. Many examples for such service APIs can be found +in the @file{gnunet/src/include/gnunet\_*\_service.h} files. + +Then, a client-service protocol needs to be designed. This typically +involves defining various message formats in a header that will be +included by both the service and the client library (but is otherwise +not shared and hence located within the service's directory and not +installed by @command{make install}). Each message must start with a +@code{struct GNUNET\_MessageHeader} and must be shorter than 64k. By +convention, all fields in IPC (and P2P) messages must be in big-endian +format (and thus should be read using @code{ntohl} and similar +functions and written using @code{htonl} and similar functions). +Unique message types must be defined for each message struct in the +@file{gnunet\_protocols.h} header (or an extension-specific include +file). + +@menu +* Connecting to the Service:: +* Sending messages:: +* Receiving Replies from the Service:: +@end menu + +@node Connecting to the Service +@subsubsection Connecting to the Service + +Before a client library can implement the application-specific protocol +with the service, a connection must be created: + +@example +@verbatiminclude tutorial-examples/003.c +@end example + +@noindent +As a result a @code{GNUNET\_MQ\_Handle} is returned +which can to used henceforth to transmit messages to the service. +The complete MQ API can be found in @file{gnunet\_mq\_lib.h}. +The @code{hanlders} array in the example above is incomplete. +Here is where you will define which messages you expect to +receive from the service, and which functions handle them. +The @code{error\_cb} is a function that is to be called whenever +there are errors communicating with the service. + +@node Sending messages +@subsubsection Sending messages + +In GNUnet, messages are always sent beginning with a +@code{struct GNUNET\_MessageHeader} in big endian format. +This header defines the size and the type of the +message, the payload follows after this header. + +@example +@verbatiminclude tutorial-examples/004.c +@end example + +@noindent +Existing message types are defined in @file{gnunet\_protocols.h}. +A common way to create a message is with an envelope: + +@example +@verbatiminclude tutorial-examples/005.c +@end example + +@noindent +Exercise: Define a message struct that includes a 32-bit +unsigned integer in addition to the standard GNUnet MessageHeader. +Add a C struct and define a fresh protocol number for your message. +Protocol numbers in gnunet-ext are defined +in @file{gnunet-ext/src/include/gnunet_protocols_ext.h} + +Exercise: Find out how you can determine the number of messages +in a message queue. + +Exercise: Find out how you can determine when a message you +have queued was actually transmitted. + +Exercise: Define a helper function to transmit a 32-bit +unsigned integer (as payload) to a service using some given client +handle. + +@node Receiving Replies from the Service +@subsubsection Receiving Replies from the Service + +Clients can receive messages from the service using the handlers +specified in the @code{handlers} array we specified when connecting +to the service. Entries in the the array are usually created using +one of two macros, depending on whether the message is fixed size +or variable size. Variable size messages are managed using two +callbacks, one to check that the message is well-formed, the other +to actually process the message. Fixed size messages are fully +checked by the MQ-logic, and thus only need to provide the handler +to process the message. Note that the prefixes @code{check\_} +and @code{handle\_} are mandatory. + +@example +@verbatiminclude tutorial-examples/006.c +@end example + +@noindent +Exercise: Expand your helper function to receive a response message +(for example, containing just the @code{struct GNUnet MessageHeader} +without any payload). Upon receiving the service's response, you +should call a callback provided to your helper function's API. + +Exercise: Figure out where you can pass values to the +closures (@code{cls}). + +@node Writing a user interface +@subsection Writing a user interface + +Given a client library, all it takes to access a service now is to +combine calls to the client library with parsing command-line +options. + +Exercise: Call your client API from your @code{run()} method in your +client application to send a request to the service. For example, +send a 32-bit integer value based on a number given at the +command-line to the service. + +@node Writing a Service +@section Writing a Service + +Before you can test the client you've written so far, you'll +need to also implement the corresponding service. + +@menu +* Code Placement:: +* Starting a Service:: +@end menu + +@node Code Placement +@subsection Code Placement + +New services are placed in their own subdirectory under +@file{gnunet/src}. This subdirectory should contain the API +implementation file @file{SERVICE\_api.c}, the description of +the client-service protocol @file{SERVICE.h} and P2P protocol +@file{SERVICE\_protocol.h}, the implementation of the service itself +@file{gnunet-service-SERVICE.h} and several files for tests, +including test code and configuration files. + +@node Starting a Service +@subsection Starting a Service + +The key API definition for creating a service is the +@code{GNUNET\_SERVICE\_MAIN} macro: + +@example +@verbatiminclude tutorial-examples/007.c +@end example + +@noindent +In addition to the service name and flags, the macro takes three +functions, typically called @code{run}, @code{client\_connect\_cb} and +@code{client\_disconnect\_cb} as well as an array of message handlers +that will be called for incoming messages from clients. + +A minimal version of the three central service funtions would look +like this: + +@example +@verbatiminclude tutorial-examples/008.c +@end example + +@noindent +Exercise: Write a stub service that processes no messages at all +in your code. Create a default configuration for it, integrate it +with the build system and start the service from +@command{gnunet-service-arm} using @command{gnunet-arm -i NAME}. + +Exercise: Figure out how to set the closure (@code{cls}) for handlers +of a service. + +Exercise: Figure out how to send messages from the service back to the +client. + +Each handler function in the service @b{must} eventually (possibly in some +asynchronous continuation) call +@code{GNUNET\_SERVICE\_client\_continue()}. Only after this call +additional messages from the same client may +be processed. This way, the service can throttle processing messages +from the same client. + +Exercise: Change the service to ``handle'' the message from your +client (for now, by printing a message). What happens if you +forget to call @code{GNUNET\_SERVICE\_client\_continue()}? + +@node Interacting directly with other Peers using the CORE Service +@section Interacting directly with other Peers using the CORE Service + +FIXME: This section still needs to be updated to the lastest API! + +One of the most important services in GNUnet is the @code{CORE} service +managing connections between peers and handling encryption between peers. + +One of the first things any service that extends the P2P protocol +typically does is connect to the @code{CORE} service using: + +@example +@verbatiminclude tutorial-examples/009.c +@end example + +@menu +* New P2P connections:: +* Receiving P2P Messages:: +* Sending P2P Messages:: +* End of P2P connections:: +@end menu + +@node New P2P connections +@subsection New P2P connections + +Before any traffic with a different peer can be exchanged, the peer must +be known to the service. This is notified by the @code{CORE} +@code{connects} callback, which communicates the identity of the new +peer to the service: + +@example +@verbatiminclude tutorial-examples/010.c +@end example + +@noindent +Note that whatever you return from @code{connects} is given as the +@code{cls} argument to the message handlers for messages from +the respective peer. + +Exercise: Create a service that connects to the @code{CORE}. Then +start (and connect) two peers and print a message once your connect +callback is invoked. + +@node Receiving P2P Messages +@subsection Receiving P2P Messages + +To receive messages from @code{CORE}, you pass the desired +@code{handlers} to the @code{GNUNET\_CORE\_connect()} function, +just as we showed for services. + +It is your responsibility to process messages fast enough or +to implement flow control. If an application does not process +CORE messages fast enough, CORE will randomly drop messages +to not keep a very long queue in memory. + +Exercise: Start one peer with a new service that has a message +handler and start a second peer that only has your ``old'' service +without message handlers. Which ``connect'' handlers are invoked when +the two peers are connected? Why? + +@node Sending P2P Messages +@subsection Sending P2P Messages + +You can transmit messages to other peers using the @code{mq} you were +given during the @code{connect} callback. Note that the @code{mq} +automatically is released upon @code{disconnect} and that you must +not use it afterwards. + +It is your responsibility to not over-fill the message queue, GNUnet +will send the messages roughly in the order given as soon as possible. + +Exercise: Write a service that upon connect sends messages as +fast as possible to the other peer (the other peer should run a +service that ``processes'' those messages). How fast is the +transmission? Count using the STATISTICS service on both ends. Are +messages lost? How can you transmit messages faster? What happens if +you stop the peer that is receiving your messages? + +@node End of P2P connections +@subsection End of P2P connections + +If a message handler returns @code{GNUNET\_SYSERR}, the remote +peer shuts down or there is an unrecoverable network +disconnection, CORE notifies the service that the peer disconnected. +After this notification no more messages will be received from the +peer and the service is no longer allowed to send messages to the peer. +The disconnect callback looks like the following: + +@example +@verbatiminclude tutorial-examples/011.c +@end example + +@noindent +Exercise: Fix your service to handle peer disconnects. + +@node Storing peer-specific data using the PEERSTORE service +@section Storing peer-specific data using the PEERSTORE service + +GNUnet's PEERSTORE service offers a persistorage for arbitrary +peer-specific data. Other GNUnet services can use the PEERSTORE +to store, retrieve and monitor data records. Each data record +stored with PEERSTORE contains the following fields: + +@itemize +@item subsystem: Name of the subsystem responsible for the record. +@item peerid: Identity of the peer this record is related to. +@item key: a key string identifying the record. +@item value: binary record value. +@item expiry: record expiry date. +@end itemize + +The first step is to start a connection to the PEERSTORE service: +@example +@verbatiminclude tutorial-examples/012.c +@end example + +The service handle @code{peerstore_handle} will be needed for +all subsequent PEERSTORE operations. + +@menu +* Storing records:: +* Retrieving records:: +* Monitoring records:: +* Disconnecting from PEERSTORE:: +@end menu + +@node Storing records +@subsection Storing records + +To store a new record, use the following function: + +@example +@verbatiminclude tutorial-examples/013.c +@end example + +@noindent +The @code{options} parameter can either be +@code{GNUNET_PEERSTORE_STOREOPTION_MULTIPLE} which means that multiple +values can be stored under the same key combination +(subsystem, peerid, key), or @code{GNUNET_PEERSTORE_STOREOPTION_REPLACE} +which means that PEERSTORE will replace any existing values under the +given key combination (subsystem, peerid, key) with the new given value. + +The continuation function @code{cont} will be called after the store +request is successfully sent to the PEERSTORE service. This does not +guarantee that the record is successfully stored, only that it was +received by the service. + +The @code{GNUNET_PEERSTORE_store} function returns a handle to the store +operation. This handle can be used to cancel the store operation only +before the continuation function is called: + +@example +@verbatiminclude tutorial-examples/013.1.c +@end example + +@node Retrieving records +@subsection Retrieving records + +To retrieve stored records, use the following function: + +@example +@verbatiminclude tutorial-examples/014.c +@end example + +@noindent +The values of @code{peer} and @code{key} can be @code{NULL}. This +allows the iteration over values stored under any of the following +key combinations: + +@itemize +@item (subsystem) +@item (subsystem, peerid) +@item (subsystem, key) +@item (subsystem, peerid, key) +@end itemize + +The @code{callback} function will be called once with each retrieved +record and once more with a @code{NULL} record to signal the end of +results. + +The @code{GNUNET_PEERSTORE_iterate} function returns a handle to the +iterate operation. This handle can be used to cancel the iterate +operation only before the callback function is called with a +@code{NULL} record. + +@node Monitoring records +@subsection Monitoring records + +PEERSTORE offers the functionality of monitoring for new records +stored under a specific key combination (subsystem, peerid, key). +To start the monitoring, use the following function: + +@example +@verbatiminclude tutorial-examples/015.c +@end example + +@noindent +Whenever a new record is stored under the given key combination, +the @code{callback} function will be called with this new +record. This will continue until the connection to the PEERSTORE +service is broken or the watch operation is canceled: + +@example +@verbatiminclude tutorial-examples/016.c +@end example + +@node Disconnecting from PEERSTORE +@subsection Disconnecting from PEERSTORE + +When the connection to the PEERSTORE service is no longer needed, +disconnect using the following function: + +@example +@verbatiminclude tutorial-examples/017.c +@end example + +@noindent +If the @code{sync_first} flag is set to @code{GNUNET_YES}, +the API will delay the disconnection until all store requests +are received by the PEERSTORE service. Otherwise, it will +disconnect immediately. + +@node Using the DHT +@section Using the DHT + +The DHT allows to store data so other peers in the P2P network can +access it and retrieve data stored by any peers in the network. +This section will explain how to use the DHT. Of course, the first +thing to do is to connect to the DHT service: + +@example +@verbatiminclude tutorial-examples/018.c +@end example + +@noindent +The second parameter indicates how many requests in parallel to expect. +It is not a hard limit, but a good approximation will make the DHT more +efficient. + +@menu +* Storing data in the DHT:: +* Obtaining data from the DHT:: +* Implementing a block plugin:: +* Monitoring the DHT:: +@end menu + +@node Storing data in the DHT +@subsection Storing data in the DHT +Since the DHT is a dynamic environment (peers join and leave frequently) +the data that we put in the DHT does not stay there indefinitely. It is +important to ``refresh'' the data periodically by simply storing it +again, in order to make sure other peers can access it. + +The put API call offers a callback to signal that the PUT request has been +sent. This does not guarantee that the data is accessible to others peers, +or even that is has been stored, only that the service has requested to +a neighboring peer the retransmission of the PUT request towards its final +destination. Currently there is no feedback about whether or not the data +has been sucessfully stored or where it has been stored. In order to +improve the availablilty of the data and to compensate for possible +errors, peers leaving and other unfavorable events, just make several +PUT requests! + +@example +@verbatiminclude tutorial-examples/019.c +@end example + +@noindent +Exercise: Store a value in the DHT periodically to make sure it +is available over time. You might consider using the function +@code{GNUNET\_SCHEDULER\_add\_delayed} and call +@code{GNUNET\_DHT\_put} from inside a helper function. + +@node Obtaining data from the DHT +@subsection Obtaining data from the DHT + +As we saw in the previous example, the DHT works in an asynchronous mode. +Each request to the DHT is executed ``in the background'' and the API +calls return immediately. In order to receive results from the DHT, the +API provides a callback. Once started, the request runs in the service, +the service will try to get as many results as possible (filtering out +duplicates) until the timeout expires or we explicitly stop the request. +It is possible to give a ``forever'' timeout with +@code{GNUNET\_TIME\_UNIT\_FOREVER\_REL}. + +If we give a route option @code{GNUNET\_DHT\_RO\_RECORD\_ROUTE} +the callback will get a list of all the peers the data has travelled, +both on the PUT path and on the GET path. + +@example +@verbatiminclude tutorial-examples/020.c +@end example + +@noindent +Exercise: Store a value in the DHT and after a while retrieve it. +Show the IDs of all the peers the requests have gone through. +In order to convert a peer ID to a string, use the function +@code{GNUNET\_i2s}. Pay attention to the route option parameters +in both calls! + +@node Implementing a block plugin +@subsection Implementing a block plugin + +In order to store data in the DHT, it is necessary to provide a block +plugin. The DHT uses the block plugin to ensure that only well-formed +requests and replies are transmitted over the network. + +The block plugin should be put in a file @file{plugin\_block\_SERVICE.c} +in the service's respective directory. The +mandatory functions that need to be implemented for a block plugin are +described in the following sections. + +@menu +* Validating requests and replies:: +* Deriving a key from a reply:: +* Initialization of the plugin:: +* Shutdown of the plugin:: +* Integration of the plugin with the build system:: +@end menu + +@node Validating requests and replies +@subsubsection Validating requests and replies + +The evaluate function should validate a reply or a request. It returns +a @code{GNUNET\_BLOCK\_EvaluationResult}, which is an enumeration. All +possible answers are in @file{gnunet\_block\_lib.h}. The function will +be called with a @code{reply\_block} argument of @code{NULL} for +requests. Note that depending on how @code{evaluate} is called, only +some of the possible return values are valid. The specific meaning of +the @code{xquery} argument is application-specific. Applications that +do not use an extended query should check that the @code{xquery\_size} +is zero. The block group is typically used to filter duplicate +replies. + +@example +@verbatiminclude tutorial-examples/021.c +@end example + +@noindent +Note that it is mandatory to detect duplicate replies in this function +and return the respective status code. Duplicate detection is +typically done using the Bloom filter block group provided by +@file{libgnunetblockgroup.so}. Failure to do so may cause replies to +circle in the network. + +@node Deriving a key from a reply +@subsubsection Deriving a key from a reply + +The DHT can operate more efficiently if it is possible to derive a key +from the value of the corresponding block. The @code{get\_key} +function is used to obtain the key of a block --- for example, by +means of hashing. If deriving the key is not possible, the function +should simply return @code{GNUNET\_SYSERR} (the DHT will still work +just fine with such blocks). + +@example +@verbatiminclude tutorial-examples/022.c +@end example + +@node Initialization of the plugin +@subsubsection Initialization of the plugin + +The plugin is realized as a shared C library. The library must export +an initialization function which should initialize the plugin. The +initialization function specifies what block types the plugin cares +about and returns a struct with the functions that are to be used for +validation and obtaining keys (the ones just defined above). + +@example +@verbatiminclude tutorial-examples/023.c +@end example + +@node Shutdown of the plugin +@subsubsection Shutdown of the plugin + +Following GNUnet's general plugin API concept, the plugin must +export a second function for cleaning up. It usually does very +little. + +@example +@verbatiminclude tutorial-examples/024.c +@end example + +@node Integration of the plugin with the build system +@subsubsection Integration of the plugin with the build system + +In order to compile the plugin, the @file{Makefile.am} file for the +service SERVICE should contain a rule similar to this: +@c Actually this is a Makefile not C. But the whole structure of examples +@c must be improved. + +@example +@verbatiminclude tutorial-examples/025.c +@end example + +@noindent +Exercise: Write a block plugin that accepts all queries +and all replies but prints information about queries and replies +when the respective validation hooks are called. + +@node Monitoring the DHT +@subsection Monitoring the DHT + +It is possible to monitor the functioning of the local +DHT service. When monitoring the DHT, the service will +alert the monitoring program of any events, both started +locally or received for routing from another peer. +The are three different types of events possible: a +GET request, a PUT request or a response (a reply to a GET). + +Since the different events have different associated data, +the API gets 3 different callbacks (one for each message type) +and optional type and key parameters, to allow for filtering of +messages. When an event happens, the appropiate callback is +called with all the information about the event. + +@example +@verbatiminclude tutorial-examples/026.c +@end example + +@node Debugging with gnunet-arm +@section Debugging with gnunet-arm + +Even if services are managed by @command{gnunet-arm}, you can +start them with @command{gdb} or @command{valgrind}. For +example, you could add the following lines to your +configuration file to start the DHT service in a @command{gdb} +session in a fresh @command{xterm}: + +@example +[dht] +PREFIX=xterm -e gdb --args +@end example + +@noindent +Alternatively, you can stop a service that was started via +ARM and run it manually: + +@example +$ gnunet-arm -k dht +$ gdb --args gnunet-service-dht -L DEBUG +$ valgrind gnunet-service-dht -L DEBUG +@end example + +@noindent +Assuming other services are well-written, they will automatically +re-integrate the restarted service with the peer. + +GNUnet provides a powerful logging mechanism providing log +levels @code{ERROR}, @code{WARNING}, @code{INFO} and @code{DEBUG}. +The current log level is configured using the @code{$GNUNET_FORCE_LOG} +environmental variable. The @code{DEBUG} level is only available if +@command{--enable-logging=verbose} was used when running +@command{configure}. More details about logging can be found under +@uref{https://gnunet.org/logging}. + +You should also probably enable the creation of core files, by setting +@code{ulimit}, and echo'ing @code{1} into +@file{/proc/sys/kernel/core\_uses\_pid}. Then you can investigate the +core dumps with @command{gdb}, which is often the fastest method to +find simple errors. + +Exercise: Add a memory leak to your service and obtain a trace +pointing to the leak using @command{valgrind} while running the service +from @command{gnunet-service-arm}. + +@bye diff --git a/doc/documentation/gnunet.texi b/doc/documentation/gnunet.texi new file mode 100644 index 0000000000..35eed54b6a --- /dev/null +++ b/doc/documentation/gnunet.texi @@ -0,0 +1,236 @@ +\input texinfo +@c -*-texinfo-*- + +@c %**start of header +@setfilename gnunet.info +@documentencoding UTF-8 +@settitle GNUnet Reference Manual +@exampleindent 2 +@urefbreakstyle before +@c %**end of header + +@include version.texi + +@c Set Versions which might be used in more than one place: +@set GNUFTP-URL https://ftp.gnu.org/gnu/gnunet +@set PYPI-URL https://pypi.python.org/packages/source +@set GNURL-VERSION-CURRENT 7.55.1 +@set GNUNET-DIST-URL https://gnunet.org/sites/default/files/ +@c @set OPENPGP-SIGNING-KEY-ID + +@copying +Copyright @copyright{} 2001-2017 GNUnet e.V. + +Permission is granted to copy, distribute and/or modify this document +under the terms of the GNU Free Documentation License, Version 1.3 or +any later version published by the Free Software Foundation; with no +Invariant Sections, no Front-Cover Texts, and no Back-Cover Texts. A +copy of the license is included in the section entitled ``GNU Free +Documentation License''. + +A copy of the license is also available from the Free Software +Foundation Web site at @url{http://www.gnu.org/licenses/fdl.html}. + +Alternately, this document is also available under the General +Public License, version 3 or later, as published by the Free Software +Foundation. A copy of the license is included in the section entitled +``GNU General Public License''. + +A copy of the license is also available from the Free Software +Foundation Web site at @url{http://www.gnu.org/licenses/gpl.html}. +@end copying + +@c TODO: Improve this and improve https://directory.fsf.org/wiki/Gnunet + +@dircategory Networking +@direntry +* GNUnet: (gnunet). Framework for secure peer-to-peer networking +@end direntry + +@titlepage +@title GNUnet Reference Manual +@subtitle Installing, configuring, using and contributing to GNUnet +@author The GNUnet Developers + +@page +@vskip 0pt plus 1filll +Edition @value{EDITION} @* +@value{UPDATED} @* + +@insertcopying +@end titlepage + +@summarycontents +@contents + +@node Top +@top Introduction + +This document is the Reference Manual for GNUnet version @value{VERSION}. + +@menu + +* Philosophy:: About GNUnet +* Vocabulary:: Vocabulary +* GNUnet Installation Handbook:: How to install GNUnet +* Using GNUnet:: Using GNUnet +* Configuration Handbook:: Configuring GNUnet +* GNUnet Contributors Handbook:: Contributing to GNUnet +* GNUnet Developer Handbook:: Developing GNUnet +* GNU Free Documentation License:: The license of this manual +* GNU General Public License:: The license of this manual +* Concept Index:: Concepts +* Programming Index:: Data types, functions, and variables + +@detailmenu + --- The Detailed Node Listing --- + +Philosophy + +* Design Goals:: +* Security and Privacy:: +* Versatility:: +* Practicality:: +* Key Concepts:: +* Authentication:: +* Accounting to Encourage Resource Sharing:: +* Confidentiality:: +* Anonymity:: +* Deniability:: +* Peer Identities:: +* Zones in the GNU Name System (GNS Zones):: +* Egos:: +* Backup of Identities and Egos:: +* Revocation:: + +Vocabulary + +* Definitions abbreviations and acronyms:: +* Words and characters:: +* Technical Assumptions:: + +GNUnet Installation Handbook + +* Dependencies:: +* Pre-installation notes:: +* Generic installation instructions:: +* Build instructions for Ubuntu 12.04 using Git:: +* Build instructions for software builds from source:: +* Build Instructions for Microsoft Windows Platforms:: +* Build instructions for Debian 7.5:: +* Installing GNUnet from Git on Ubuntu 14.4:: +* Build instructions for Debian 8:: +* Outdated build instructions for previous revisions:: +@c * Portable GNUnet:: +* The graphical configuration interface:: +* How to start and stop a GNUnet peer:: + +Using GNUnet + +* Checking the Installation:: +* First steps - File-sharing:: +* First steps - Using the GNU Name System:: +* First steps - Using GNUnet Conversation:: +* First steps - Using the GNUnet VPN:: +* File-sharing:: +* The GNU Name System:: +* Using the Virtual Public Network:: + +Configuration Handbook + +GNUnet Contributors Handbook + +* Contributing to GNUnet:: +* Licenses of contributions:: +* Copyright Assignment:: +* Contributing to the Reference Manual:: + +GNUnet Developer Handbook + +* Developer Introduction:: +* Code overview:: +* System Architecture:: +* Subsystem stability:: +* Naming conventions and coding style guide:: +* Build-system:: +* Developing extensions for GNUnet using the gnunet-ext template:: +* Writing testcases:: +* TESTING library:: +* Performance regression analysis with Gauger:: +* TESTBED Subsystem:: +* libgnunetutil:: +* Automatic Restart Manager (ARM):: +* TRANSPORT Subsystem:: +* NAT library:: +* Distance-Vector plugin:: +* SMTP plugin:: +* Bluetooth plugin:: +* WLAN plugin:: +* ATS Subsystem:: +* CORE Subsystem:: +* CADET Subsystem:: +* NSE Subsystem:: +* HOSTLIST Subsystem:: +* IDENTITY Subsystem:: +* NAMESTORE Subsystem:: +* PEERINFO Subsystem:: +* PEERSTORE Subsystem:: +* SET Subsystem:: +* STATISTICS Subsystem:: +* Distributed Hash Table (DHT):: +* GNU Name System (GNS):: +* GNS Namecache:: +* REVOCATION Subsystem:: +* File-sharing (FS) Subsystem:: +* REGEX Subsystem:: + +@end detailmenu +@end menu + +@c ********************************************************************* +@include chapters/philosophy.texi +@c ********************************************************************* + +@include chapters/vocabulary.texi + +@c ********************************************************************* +@include chapters/installation.texi +@c ********************************************************************* + +@c ********************************************************************* +@include chapters/user.texi +@c ********************************************************************* + +@include chapters/configuration.texi + +@include chapters/contributing.texi + +@c ********************************************************************* +@include chapters/developer.texi +@c @include gnunet-c-tutorial.texi +@c ********************************************************************* + +@c ********************************************************************* +@node GNU Free Documentation License +@appendix GNU Free Documentation License +@cindex license, GNU Free Documentation License +@include fdl-1.3.texi + +@c ********************************************************************* +@node GNU General Public License +@appendix GNU General Public License +@cindex license, GNU General Public License +@include gpl-3.0.texi + +@c ********************************************************************* +@node Concept Index +@unnumbered Concept Index +@printindex cp + +@node Programming Index +@unnumbered Programming Index +@syncodeindex tp fn +@syncodeindex vr fn +@printindex fn + +@bye diff --git a/doc/documentation/gpl-3.0.texi b/doc/documentation/gpl-3.0.texi new file mode 100644 index 0000000000..0e2e212acb --- /dev/null +++ b/doc/documentation/gpl-3.0.texi @@ -0,0 +1,717 @@ +@c The GNU General Public License. +@center Version 3, 29 June 2007 + +@c This file is intended to be included within another document, +@c hence no sectioning command or @node. + +@display +Copyright @copyright{} 2007 Free Software Foundation, Inc. @url{http://fsf.org/} + +Everyone is permitted to copy and distribute verbatim copies of this +license document, but changing it is not allowed. +@end display + +@heading Preamble + +The GNU General Public License is a free, copyleft license for +software and other kinds of works. + +The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom +to share and change all versions of a program---to make sure it remains +free software for all its users. We, the Free Software Foundation, +use the GNU General Public License for most of our software; it +applies also to any other work released this way by its authors. You +can apply it to your programs, too. + +When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + +To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you +have certain responsibilities if you distribute copies of the +software, or if you modify it: responsibilities to respect the freedom +of others. + +For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, +receive or can get the source code. And you must show them these +terms so they know their rights. + +Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + +For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + +Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the +manufacturer can do so. This is fundamentally incompatible with the +aim of protecting users' freedom to change the software. The +systematic pattern of such abuse occurs in the area of products for +individuals to use, which is precisely where it is most unacceptable. +Therefore, we have designed this version of the GPL to prohibit the +practice for those products. If such problems arise substantially in +other domains, we stand ready to extend this provision to those +domains in future versions of the GPL, as needed to protect the +freedom of users. + +Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish +to avoid the special danger that patents applied to a free program +could make it effectively proprietary. To prevent this, the GPL +assures that patents cannot be used to render the program non-free. + +The precise terms and conditions for copying, distribution and +modification follow. + +@heading TERMS AND CONDITIONS + +@enumerate 0 +@item Definitions. + +``This License'' refers to version 3 of the GNU General Public License. + +``Copyright'' also means copyright-like laws that apply to other kinds +of works, such as semiconductor masks. + +``The Program'' refers to any copyrightable work licensed under this +License. Each licensee is addressed as ``you''. ``Licensees'' and +``recipients'' may be individuals or organizations. + +To ``modify'' a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of +an exact copy. The resulting work is called a ``modified version'' of +the earlier work or a work ``based on'' the earlier work. + +A ``covered work'' means either the unmodified Program or a work based +on the Program. + +To ``propagate'' a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + +To ``convey'' a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user +through a computer network, with no transfer of a copy, is not +conveying. + +An interactive user interface displays ``Appropriate Legal Notices'' to +the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + +@item Source Code. + +The ``source code'' for a work means the preferred form of the work for +making modifications to it. ``Object code'' means any non-source form +of a work. + +A ``Standard Interface'' means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + +The ``System Libraries'' of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +``Major Component'', in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + +The ``Corresponding Source'' for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + +The Corresponding Source need not include anything that users can +regenerate automatically from other parts of the Corresponding Source. + +The Corresponding Source for a work in source code form is that same +work. + +@item Basic Permissions. + +All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + +You may make, run and propagate covered works that you do not convey, +without conditions so long as your license otherwise remains in force. +You may convey covered works to others for the sole purpose of having +them make modifications exclusively for you, or provide you with +facilities for running those works, provided that you comply with the +terms of this License in conveying all material for which you do not +control copyright. Those thus making or running the covered works for +you must do so exclusively on your behalf, under your direction and +control, on terms that prohibit them from making any copies of your +copyrighted material outside their relationship with you. + +Conveying under any other circumstances is permitted solely under the +conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + +@item Protecting Users' Legal Rights From Anti-Circumvention Law. + +No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + +When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such +circumvention is effected by exercising rights under this License with +respect to the covered work, and you disclaim any intention to limit +operation or modification of the work as a means of enforcing, against +the work's users, your or third parties' legal rights to forbid +circumvention of technological measures. + +@item Conveying Verbatim Copies. + +You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + +You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + +@item Conveying Modified Source Versions. + +You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these +conditions: + +@enumerate a +@item +The work must carry prominent notices stating that you modified it, +and giving a relevant date. + +@item +The work must carry prominent notices stating that it is released +under this License and any conditions added under section 7. This +requirement modifies the requirement in section 4 to ``keep intact all +notices''. + +@item +You must license the entire work, as a whole, under this License to +anyone who comes into possession of a copy. This License will +therefore apply, along with any applicable section 7 additional terms, +to the whole of the work, and all its parts, regardless of how they +are packaged. This License gives no permission to license the work in +any other way, but it does not invalidate such permission if you have +separately received it. + +@item +If the work has interactive user interfaces, each must display +Appropriate Legal Notices; however, if the Program has interactive +interfaces that do not display Appropriate Legal Notices, your work +need not make them do so. +@end enumerate + +A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +``aggregate'' if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + +@item Conveying Non-Source Forms. + +You may convey a covered work in object code form under the terms of +sections 4 and 5, provided that you also convey the machine-readable +Corresponding Source under the terms of this License, in one of these +ways: + +@enumerate a +@item +Convey the object code in, or embodied in, a physical product +(including a physical distribution medium), accompanied by the +Corresponding Source fixed on a durable physical medium customarily +used for software interchange. + +@item +Convey the object code in, or embodied in, a physical product +(including a physical distribution medium), accompanied by a written +offer, valid for at least three years and valid for as long as you +offer spare parts or customer support for that product model, to give +anyone who possesses the object code either (1) a copy of the +Corresponding Source for all the software in the product that is +covered by this License, on a durable physical medium customarily used +for software interchange, for a price no more than your reasonable +cost of physically performing this conveying of source, or (2) access +to copy the Corresponding Source from a network server at no charge. + +@item +Convey individual copies of the object code with a copy of the written +offer to provide the Corresponding Source. This alternative is +allowed only occasionally and noncommercially, and only if you +received the object code with such an offer, in accord with subsection +6b. + +@item +Convey the object code by offering access from a designated place +(gratis or for a charge), and offer equivalent access to the +Corresponding Source in the same way through the same place at no +further charge. You need not require recipients to copy the +Corresponding Source along with the object code. If the place to copy +the object code is a network server, the Corresponding Source may be +on a different server (operated by you or a third party) that supports +equivalent copying facilities, provided you maintain clear directions +next to the object code saying where to find the Corresponding Source. +Regardless of what server hosts the Corresponding Source, you remain +obligated to ensure that it is available for as long as needed to +satisfy these requirements. + +@item +Convey the object code using peer-to-peer transmission, provided you +inform other peers where the object code and Corresponding Source of +the work are being offered to the general public at no charge under +subsection 6d. + +@end enumerate + +A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + +A ``User Product'' is either (1) a ``consumer product'', which means any +tangible personal property which is normally used for personal, +family, or household purposes, or (2) anything designed or sold for +incorporation into a dwelling. In determining whether a product is a +consumer product, doubtful cases shall be resolved in favor of +coverage. For a particular product received by a particular user, +``normally used'' refers to a typical or common use of that class of +product, regardless of the status of the particular user or of the way +in which the particular user actually uses, or expects or is expected +to use, the product. A product is a consumer product regardless of +whether the product has substantial commercial, industrial or +non-consumer uses, unless such uses represent the only significant +mode of use of the product. + +``Installation Information'' for a User Product means any methods, +procedures, authorization keys, or other information required to +install and execute modified versions of a covered work in that User +Product from a modified version of its Corresponding Source. The +information must suffice to ensure that the continued functioning of +the modified object code is in no case prevented or interfered with +solely because modification has been made. + +If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + +The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or +updates for a work that has been modified or installed by the +recipient, or for the User Product in which it has been modified or +installed. Access to a network may be denied when the modification +itself materially and adversely affects the operation of the network +or violates the rules and protocols for communication across the +network. + +Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + +@item Additional Terms. + +``Additional permissions'' are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + +When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + +Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders +of that material) supplement the terms of this License with terms: + +@enumerate a +@item +Disclaiming warranty or limiting liability differently from the terms +of sections 15 and 16 of this License; or + +@item +Requiring preservation of specified reasonable legal notices or author +attributions in that material or in the Appropriate Legal Notices +displayed by works containing it; or + +@item +Prohibiting misrepresentation of the origin of that material, or +requiring that modified versions of such material be marked in +reasonable ways as different from the original version; or + +@item +Limiting the use for publicity purposes of names of licensors or +authors of the material; or + +@item +Declining to grant rights under trademark law for use of some trade +names, trademarks, or service marks; or + +@item +Requiring indemnification of licensors and authors of that material by +anyone who conveys the material (or modified versions of it) with +contractual assumptions of liability to the recipient, for any +liability that these contractual assumptions directly impose on those +licensors and authors. +@end enumerate + +All other non-permissive additional terms are considered ``further +restrictions'' within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + +If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + +Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; the +above requirements apply either way. + +@item Termination. + +You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + +However, if you cease all violation of this License, then your license +from a particular copyright holder is reinstated (a) provisionally, +unless and until the copyright holder explicitly and finally +terminates your license, and (b) permanently, if the copyright holder +fails to notify you of the violation by some reasonable means prior to +60 days after the cessation. + +Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + +Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + +@item Acceptance Not Required for Having Copies. + +You are not required to accept this License in order to receive or run +a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + +@item Automatic Licensing of Downstream Recipients. + +Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + +An ``entity transaction'' is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + +You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + +@item Patents. + +A ``contributor'' is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's ``contributor version''. + +A contributor's ``essential patent claims'' are all patent claims owned +or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, ``control'' includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + +Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + +In the following three paragraphs, a ``patent license'' is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To ``grant'' such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + +If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. ``Knowingly relying'' means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + +If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + +A patent license is ``discriminatory'' if it does not include within the +scope of its coverage, prohibits the exercise of, or is conditioned on +the non-exercise of one or more of the rights that are specifically +granted under this License. You may not convey a covered work if you +are a party to an arrangement with a third party that is in the +business of distributing software, under which you make payment to the +third party based on the extent of your activity of conveying the +work, and under which the third party grants, to any of the parties +who would receive the covered work from you, a discriminatory patent +license (a) in connection with copies of the covered work conveyed by +you (or copies made from those copies), or (b) primarily for and in +connection with specific products or compilations that contain the +covered work, unless you entered into that arrangement, or that patent +license was granted, prior to 28 March 2007. + +Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + +@item No Surrender of Others' Freedom. + +If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey +a covered work so as to satisfy simultaneously your obligations under +this License and any other pertinent obligations, then as a +consequence you may not convey it at all. For example, if you agree +to terms that obligate you to collect a royalty for further conveying +from those to whom you convey the Program, the only way you could +satisfy both those terms and this License would be to refrain entirely +from conveying the Program. + +@item Use with the GNU Affero General Public License. + +Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + +@item Revised Versions of this License. + +The Free Software Foundation may publish revised and/or new versions +of the GNU General Public License from time to time. Such new +versions will be similar in spirit to the present version, but may +differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies that a certain numbered version of the GNU General Public +License ``or any later version'' applies to it, you have the option of +following the terms and conditions either of that numbered version or +of any later version published by the Free Software Foundation. If +the Program does not specify a version number of the GNU General +Public License, you may choose any version ever published by the Free +Software Foundation. + +If the Program specifies that a proxy can decide which future versions +of the GNU General Public License can be used, that proxy's public +statement of acceptance of a version permanently authorizes you to +choose that version for the Program. + +Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + +@item Disclaimer of Warranty. + +THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW@. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM ``AS IS'' WITHOUT +WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE@. THE ENTIRE RISK AS TO THE QUALITY AND +PERFORMANCE OF THE PROGRAM IS WITH YOU@. SHOULD THE PROGRAM PROVE +DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR +CORRECTION. + +@item Limitation of Liability. + +IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR +CONVEYS THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES +ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT +NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR +LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM +TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER +PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + +@item Interpretation of Sections 15 and 16. + +If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + +@end enumerate + +@heading END OF TERMS AND CONDITIONS + +@heading How to Apply These Terms to Your New Programs + +If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these +terms. + +To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the ``copyright'' line and a pointer to where the full notice is found. + +@smallexample +@var{one line to give the program's name and a brief idea of what it does.} +Copyright (C) @var{year} @var{name of author} + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or (at +your option) any later version. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE@. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see @url{http://www.gnu.org/licenses/}. +@end smallexample + +Also add information on how to contact you by electronic and paper mail. + +If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + +@smallexample +@var{program} Copyright (C) @var{year} @var{name of author} +This program comes with ABSOLUTELY NO WARRANTY; for details type @samp{show w}. +This is free software, and you are welcome to redistribute it +under certain conditions; type @samp{show c} for details. +@end smallexample + +The hypothetical commands @samp{show w} and @samp{show c} should show +the appropriate parts of the General Public License. Of course, your +program's commands might be different; for a GUI interface, you would +use an ``about box''. + +You should also get your employer (if you work as a programmer) or school, +if any, to sign a ``copyright disclaimer'' for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +@url{http://www.gnu.org/licenses/}. + +The GNU General Public License does not permit incorporating your +program into proprietary programs. If your program is a subroutine +library, you may consider it more useful to permit linking proprietary +applications with the library. If this is what you want to do, use +the GNU Lesser General Public License instead of this License. But +first, please read @url{http://www.gnu.org/philosophy/why-not-lgpl.html}. diff --git a/doc/documentation/htmlxref.cnf b/doc/documentation/htmlxref.cnf new file mode 100644 index 0000000000..a4928f6fee --- /dev/null +++ b/doc/documentation/htmlxref.cnf @@ -0,0 +1,668 @@ +# htmlxref.cnf - reference file for free Texinfo manuals on the web. +# Modified by Ludovic Courtès <ludo@gnu.org> for the GNU Guix manual. +# Modified by ng0 <ng0@gnunet.org> for the GNUnet manual. + +htmlxrefversion=2017-10-26.06; # UTC + +# Copyright 2010, 2011, 2012, 2013, 2014, 2015 Free Software Foundation, Inc. +# +# Copying and distribution of this file, with or without modification, +# are permitted in any medium without royalty provided the copyright +# notice and this notice are preserved. +# +# The latest version of this file is available at +# http://ftpmirror.gnu.org/texinfo/htmlxref.cnf. +# Email corrections or additions to bug-texinfo@gnu.org. +# The primary goal is to list all relevant GNU manuals; +# other free manuals are also welcome. +# +# To be included in this list, a manual must: +# +# - have a generic url, e.g., no version numbers; +# - have a unique file name (e.g., manual identifier), i.e., be related to the +# package name. Things like "refman" or "tutorial" don't work. +# - follow the naming convention for nodes described at +# http://www.gnu.org/software/texinfo/manual/texinfo/html_node/HTML-Xref.html +# This is what makeinfo and texi2html implement. +# +# Unless the above criteria are met, it's not possible to generate +# reliable cross-manual references. +# +# For information on automatically generating all the useful formats for +# a manual to put on the web, see +# http://www.gnu.org/prep/maintain/html_node/Manuals-on-Web-Pages.html. + +# For people editing this file: when a manual named foo is related to a +# package named bar, the url should contain a variable reference ${BAR}. +# Otherwise, the gnumaint scripts have no way of knowing they are +# associated, and thus gnu.org/manual can't include them. + +# shorten references to manuals on www.gnu.org. +G = https://www.gnu.org +GS = ${G}/software + +3dldf mono ${GS}/3dldf/manual/user_ref/3DLDF.html +3dldf node ${GS}/3dldf/manual/user_ref/ + +alive mono ${GS}/alive/manual/alive.html +alive node ${GS}/alive/manual/html_node/ + +anubis chapter ${GS}/anubis/manual/html_chapter/ +anubis section ${GS}/anubis/manual/html_section/ +anubis node ${GS}/anubis/manual/html_node/ + +artanis mono ${GS}/artanis/manual/artanis.html +artanis node ${GS}/artanis/manual/html_node/ + +aspell section http://aspell.net/man-html/index.html + +auctex mono ${GS}/auctex/manual/auctex.html +auctex node ${GS}/auctex/manual/auctex/ + +autoconf mono ${GS}/autoconf/manual/autoconf.html +autoconf node ${GS}/autoconf/manual/html_node/ + +autogen mono ${GS}/autogen/manual/html_mono/autogen.html +autogen chapter ${GS}/autogen/manual/html_chapter/ +autogen node ${GS}/autoconf/manual/html_node/ + +automake mono ${GS}/automake/manual/automake.html +automake node ${GS}/automake/manual/html_node/ + +avl node http://www.stanford.edu/~blp/avl/libavl.html/ + +bash mono ${GS}/bash/manual/bash.html +bash node ${GS}/bash/manual/html_node/ + +BINUTILS = http://sourceware.org/binutils/docs +binutils node ${BINUTILS}/binutils/ + as node ${BINUTILS}/as/ + bfd node ${BINUTILS}/bfd/ + gprof node ${BINUTILS}/gprof/ + ld node ${BINUTILS}/ld/ + +bison mono ${GS}/bison/manual/bison.html +bison node ${GS}/bison/manual/html_node/ + +bpel2owfn mono ${GS}/bpel2owfn/manual/2.0.x/bpel2owfn.html + +ccd2cue mono ${GS}/ccd2cue/manual/ccd2cue.html +ccd2cue node ${GS}/ccd2cue/manual/html_node/ + +cflow mono ${GS}/cflow/manual/cflow.html +cflow node ${GS}/cflow/manual/html_node/ + +chess mono ${GS}/chess/manual/gnuchess.html +chess node ${GS}/chess/manual/html_node/ + +combine mono ${GS}/combine/manual/combine.html +combine chapter ${GS}/combine/manual/html_chapter/ +combine section ${GS}/combine/manual/html_section/ +combine node ${GS}/combine/manual/html_node/ + +complexity mono ${GS}/complexity/manual/complexity.html +complexity node ${GS}/complexity/manual/html_node/ + +coreutils mono ${GS}/coreutils/manual/coreutils +coreutils node ${GS}/coreutils/manual/html_node/ + +cpio mono ${GS}/cpio/manual/cpio +cpio node ${GS}/cpio/manual/html_node/ + +cssc node ${GS}/cssc/manual/ + +#cvs cannot be handled here; see http://ximbiot.com/cvs/manual. + +ddd mono ${GS}/ddd/manual/html_mono/ddd.html + +ddrescue mono ${GS}/ddrescue/manual/ddrescue_manual.html + +DICO = http://puszcza.gnu.org.ua/software/dico/manual +dico mono ${DICO}/dico.html +dico chapter ${DICO}/html_chapter/ +dico section ${DICO}/html_section/ +dico node ${DICO}/html_node/ + +diffutils mono ${GS}/diffutils/manual/diffutils +diffutils node ${GS}/diffutils/manual/html_node/ + +ed mono ${GS}/ed/manual/ed_manual.html + +EMACS = ${GS}/emacs/manual +emacs mono ${EMACS}/html_mono/emacs.html +emacs node ${EMACS}/html_node/emacs/ + # + ada-mode mono ${EMACS}/html_mono/ada-mode.html + ada-mode node ${EMACS}/html_node/ada-mode/ + # + autotype mono ${EMACS}/html_mono/autotype.html + autotype node ${EMACS}/html_node/autotype/ + # + ccmode mono ${EMACS}/html_mono/ccmode.html + ccmode node ${EMACS}/html_node/ccmode/ + # + cl mono ${EMACS}/html_mono/cl.html + cl node ${EMACS}/html_node/cl/ + # + ebrowse mono ${EMACS}/html_mono/ebrowse.html + ebrowse node ${EMACS}/html_node/ebrowse/ + # + ediff mono ${EMACS}/html_mono/ediff.html + ediff node ${EMACS}/html_node/ediff/ + # + eieio mono ${EMACS}/html_mono/eieio.html + eieio node ${EMACS}/html_node/eieio/ + # + elisp mono ${EMACS}/html_mono/elisp.html + elisp node ${EMACS}/html_node/elisp/ + # + epa mono ${EMACS}/html_mono/epa.html + epa node ${EMACS}/html_node/epa/ + # + erc mono ${EMACS}/html_mono/erc.html + erc node ${EMACS}/html_node/erc/ + # + dired-x mono ${EMACS}/html_mono/dired-x.html + dired-x node ${EMACS}/html_node/dired-x/ + # + eshell mono ${EMACS}/html_mono/eshell.html + eshell node ${EMACS}/html_node/eshell/ + # + flymake mono ${EMACS}/html_mono/flymake.html + flymake node ${EMACS}/html_node/flymake/ + # + gnus mono ${EMACS}/html_mono/gnus.html + gnus node ${EMACS}/html_node/gnus/ + # + idlwave mono ${EMACS}/html_mono/idlwave.html + idlwave node ${EMACS}/html_node/idlwave/ + # + message mono ${EMACS}/html_mono/message.html + message node ${EMACS}/html_node/message/ + # + mh-e mono ${EMACS}/html_mono/mh-e.html + mh-e node ${EMACS}/html_node/mh-e/ + # + nxml-mode mono ${EMACS}/html_mono/nxml-mode.html + nxml-mode node ${EMACS}/html_node/nxml-mode/ + # + org mono ${EMACS}/html_mono/org.html + org node ${EMACS}/html_node/org/ + # + pcl-cvs mono ${EMACS}/html_mono/pcl-cvs.html + pcl-cvs node ${EMACS}/html_node/pcl-cvs/ + # + rcirc mono ${EMACS}/html_mono/rcirc.html + rcirc node ${EMACS}/html_node/rcirc/ + # + semantic mono ${EMACS}/html_mono/semantic.html + semantic node ${EMACS}/html_node/semantic/ + # + smtp mono ${EMACS}/html_mono/smtpmail.html + smtp node ${EMACS}/html_node/smtpmail/ + # + speedbar mono ${EMACS}/html_mono/speedbar.html + speedbar node ${EMACS}/html_node/speedbar/ + # + tramp mono ${EMACS}/html_mono/tramp.html + tramp node ${EMACS}/html_node/tramp/ + # + vip mono ${EMACS}/html_mono/vip.html + vip node ${EMACS}/html_node/vip/ + # + viper mono ${EMACS}/html_mono/viper.html + viper node ${EMACS}/html_node/viper/ + # + woman mono ${EMACS}/html_mono/woman.html + woman node ${EMACS}/html_node/woman/ + # (end emacs manuals) + +easejs mono ${GS}/easejs/manual/easejs.html +easejs node ${GS}/easejs/manual/ + +EMACS_GUIX = https://alezost.github.io/guix.el/manual/latest +emacs-guix mono ${EMACS_GUIX}/emacs-guix.html +emacs-guix node ${EMACS_GUIX}/html_node/ + +emacs-muse node ${GS}/emacs-muse/manual/muse.html +emacs-muse node ${GS}/emacs-muse/manual/html_node/ + +emms node ${GS}/emms/manual/ + +# The file is called 'find.info' but the package is 'findutils'. +find mono ${GS}/findutils/manual/html_mono/find.html +find node ${GS}/findutils/manual/html_node/find_html +findutils mono ${GS}/findutils/manual/html_mono/find.html +findutils node ${GS}/findutils/manual/html_node/find_html + +FLEX = http://flex.sourceforge.net +flex node ${FLEX}/manual/ + +gama mono ${GS}/gama/manual/gama.html +gama node ${GS}/gama/manual/html_node/ + +GAWK = ${GS}/gawk/manual +gawk mono ${GAWK}/gawk.html +gawk node ${GAWK}/html_node/ + gawkinet mono ${GAWK}/gawkinet/gawkinet.html + gawkinet node ${GAWK}/gawkinet/html_node/ + +gcal mono ${GS}/gcal/manual/gcal.html +gcal node ${GS}/gcal/manual/html_node/ + +GCC = http://gcc.gnu.org/onlinedocs +gcc node ${GCC}/gcc/ + cpp node ${GCC}/cpp/ + gcj node ${GCC}/gcj/ + gfortran node ${GCC}/gfortran/ + gnat_rm node ${GCC}/gnat_rm/ + gnat_ugn_unw node ${GCC}/gnat_ugn_unw/ + libgomp node ${GCC}/libgomp/ + libstdc++ node ${GCC}/libstdc++/ + # + gccint node ${GCC}/gccint/ + cppinternals node ${GCC}/cppinternals/ + gfc-internals node ${GCC}/gfc-internals/ + gnat-style node ${GCC}/gnat-style/ + libiberty node ${GCC}/libiberty/ + +GDB = http://sourceware.org/gdb/current/onlinedocs +gdb node ${GDB}/gdb/ + stabs node ${GDB}/stabs/ + +GDBM = http://www.gnu.org.ua/software/gdbm/manual +gdbm mono ${GDBM}/gdbm.html +gdbm chapter ${GDBM}/html_chapter/ +gdbm section ${GDBM}/html_section/ +gdbm node ${GDBM}/html_node/ + +gettext mono ${GS}/gettext/manual/gettext.html +gettext node ${GS}/gettext/manual/html_node/ + +gforth node http://www.complang.tuwien.ac.at/forth/gforth/Docs-html/ + +global mono ${GS}/global/manual/global.html + +gmediaserver node ${GS}/gmediaserver/manual/ + +gmp node http://www.gmplib.org/manual/ + +gnu-arch node ${GS}/gnu-arch/tutorial/ + +gnu-c-manual mono ${GS}/gnu-c-manual/gnu-c-manual.html + +gnu-crypto node ${GS}/gnu-crypto/manual/ + +gnubg mono ${GS}/gnubg/manual/gnubg.html +gnubg node ${GS}/gnubg/manual/html_node/ + +gnubik mono ${GS}/gnubik/manual/gnubik.html +gnubik node ${GS}/gnubik/manual/html_node/ + +gnulib mono ${GS}/gnulib/manual/gnulib.html +gnulib node ${GS}/gnulib/manual/html_node/ + +GNUN = ${GS}/trans-coord/manual +gnun mono ${GNUN}/gnun/gnun.html +gnun node ${GNUN}/gnun/html_node/ + web-trans mono ${GNUN}/web-trans/web-trans.html + web-trans node ${GNUN}/web-trans/html_node/ + +GNUNET = https://docs.gnunet.org/manuals +gnunet node ${GNUNET}/gnunet/ + gnunet-c-tutorial node ${GNUNET}/gnunet-c-tutorial/ + gnunet-java-tutorial node ${GNUNET}/gnunet-java-tutorial/ + +GNUPG = http://www.gnupg.org/documentation/manuals +gnupg node ${GNUPG}/gnupg/ + dirmngr node ${GNUPG}/dirmngr/ + gcrypt node ${GNUPG}/gcrypt/ + libgcrypt node ${GNUPG}/gcrypt/ + ksba node ${GNUPG}/ksba/ + assuan node ${GNUPG}/assuan/ + gpgme node ${GNUPG}/gpgme/ + +gnuprologjava node ${GS}/gnuprologjava/manual/ + +gnuschool mono ${GS}/gnuschool/gnuschool.html + +GNUSTANDARDS = ${G}/prep + maintain mono ${GNUSTANDARDS}/maintain/maintain.html + maintain node ${GNUSTANDARDS}/maintain/html_node/ + # + standards mono ${GNUSTANDARDS}/standards/standards.html + standards node ${GNUSTANDARDS}/standards/html_node/ + +gnutls mono http://gnutls.org/manual/gnutls.html +gnutls node http://gnutls.org/manual/html_node/ + +gnutls-guile mono http://gnutls.org/manual/gnutls-guile.html +gnutls-guile node http://gnutls.org/manual/gnutls-guile/ + +gperf mono ${GS}/gperf/manual/gperf.html +gperf node ${GS}/gperf/manual/html_node/ + +grep mono ${GS}/grep/manual/grep.html +grep node ${GS}/grep/manual/html_node/ + +groff node ${GS}/groff/manual/html_node/ + +GRUB = ${GS}/grub/manual + grub mono ${GRUB}/grub.html + grub node ${GRUB}/html_node/ + # + multiboot mono ${GRUB}/multiboot/multiboot.html + multiboot node ${GRUB}/multiboot/html_node/ + +gsasl mono ${GS}/gsasl/manual/gsasl.html +gsasl node ${GS}/gsasl/manual/html_node/ + +gsl node ${GS}/gsl/manual/html_node/ + +gsrc mono ${GS}/gsrc/manual/gsrc.html +gsrc node ${GS}/gsrc/manual/html_node/ + +gss mono ${GS}/gss/manual/gss.html +gss node ${GS}/gss/manual/html_node/ + +gtypist mono ${GS}/gtypist/doc/ + +guile mono ${GS}/guile/manual/guile.html +guile node ${GS}/guile/manual/html_node/ + +guile-avahi mono http://nongnu.org/guile-avahi/doc/guile-avahi.html + +GUILE_GNOME = ${GS}/guile-gnome/docs + gobject node ${GUILE_GNOME}/gobject/html/ + glib node ${GUILE_GNOME}/glib/html/ + atk node ${GUILE_GNOME}/atk/html/ + pango node ${GUILE_GNOME}/pango/html/ + pangocairo node ${GUILE_GNOME}/pangocairo/html/ + gdk node ${GUILE_GNOME}/gdk/html/ + gtk node ${GUILE_GNOME}/gtk/html/ + libglade node ${GUILE_GNOME}/libglade/html/ + gnome-vfs node ${GUILE_GNOME}/gnome-vfs/html/ + libgnomecanvas node ${GUILE_GNOME}/libgnomecanvas/html/ + gconf node ${GUILE_GNOME}/gconf/html/ + libgnome node ${GUILE_GNOME}/libgnome/html/ + libgnomeui node ${GUILE_GNOME}/libgnomeui/html/ + corba node ${GUILE_GNOME}/corba/html/ + clutter node ${GUILE_GNOME}/clutter/html/ + clutter-glx node ${GUILE_GNOME}/clutter-glx/html/ + +guile-gtk node ${GS}/guile-gtk/docs/guile-gtk/ + +guile-rpc mono ${GS}/guile-rpc/manual/guile-rpc.html +guile-rpc node ${GS}/guile-rpc/manual/html_node/ + +guix mono ${GS}/guix/manual/guix.html +guix node ${GS}/guix/manual/html_node/ + +gv mono ${GS}/gv/manual/gv.html +gv node ${GS}/gv/manual/html_node/ + +gzip mono ${GS}/gzip/manual/gzip.html +gzip node ${GS}/gzip/manual/html_node/ + +hello mono ${GS}/hello/manual/hello.html +hello node ${GS}/hello/manual/html_node/ + +help2man mono ${GS}/help2man/help2man.html + +idutils mono ${GS}/idutils/manual/idutils.html +idutils node ${GS}/idutils/manual/html_node/ + +inetutils mono ${GS}/inetutils/manual/inetutils.html +inetutils node ${GS}/inetutils/manual/html_node/ + +jwhois mono ${GS}/jwhois/manual/jwhois.html +jwhois node ${GS}/jwhois/manual/html_node/ + +libc mono ${GS}/libc/manual/html_mono/libc.html +libc node ${GS}/libc/manual/html_node/ + +LIBCDIO = ${GS}/libcdio + libcdio mono ${LIBCDIO}/libcdio.html + cd-text mono ${LIBCDIO}/cd-text-format.html + +libextractor mono ${GS}/libextractor/manual/libextractor.html +libextractor node ${GS}/libextractor/manual/html_node/ + +libidn mono ${GS}/libidn/manual/libidn.html +libidn node ${GS}/libidn/manual/html_node/ + +librejs mono ${GS}/librejs/manual/librejs.html +librejs node ${GS}/librejs/manual/html_node/ + +libmatheval mono ${GS}/libmatheval/manual/libmatheval.html + +LIBMICROHTTPD = ${GS}/libmicrohttpd +libmicrohttpd mono ${LIBMICROHTTPD}/manual/libmicrohttpd.html +libmicrohttpd node ${LIBMICROHTTPD}/manual/html_node/ + microhttpd-tutorial mono ${LIBMICROHTTPD}/tutorial.html + +libtasn1 mono ${GS}/libtasn1/manual/libtasn1.html +libtasn1 node ${GS}/libtasn1/manual/html_node/ + +libtool mono ${GS}/libtool/manual/libtool.html +libtool node ${GS}/libtool/manual/html_node/ + +lightning mono ${GS}/lightning/manual/lightning.html +lightning node ${GS}/lightning/manual/html_node/ + +# The stable/ url redirects immediately, but that's ok. +# The .html extension is omitted on their web site, but it works if given. +LILYPOND = http://lilypond.org/doc/stable/Documentation + lilypond-internals node ${LILYPOND}/internals/ + lilypond-learning node ${LILYPOND}/learning/ + lilypond-notation node ${LILYPOND}/notation/ + lilypond-snippets node ${LILYPOND}/snippets/ + lilypond-usage node ${LILYPOND}/usage/ + lilypond-web node ${LILYPOND}/web/ + music-glossary node ${LILYPOND}/music-glossary/ + +liquidwar6 mono ${GS}/liquidwar6/manual/liquidwar6.html +liquidwar6 node ${GS}/liquidwar6/manual/html_node/ + +lispintro mono ${GS}/emacs/emacs-lisp-intro/html_mono/emacs-lisp-intro.html +lispintro node ${GS}/emacs/emacs-lisp-intro/html_node/index.html + +LSH = http://www.lysator.liu.se/~nisse/lsh + lsh mono ${LSH}/lsh.html + +m4 mono ${GS}/m4/manual/m4.html +m4 node ${GS}/m4/manual/html_node/ + +mailutils mono ${GS}/mailutils/manual/mailutils.html +mailutils chapter ${GS}/mailutils/manual/html_chapter/ +mailutils section ${GS}/mailutils/manual/html_section/ +mailutils node ${GS}/mailutils/manual/html_node/ + +make mono ${GS}/make/manual/make.html +make node ${GS}/make/manual/html_node/ + +mcron mono ${GS}/mcron/manual/mcron.html +mcron node ${GS}/mcron/manual/html_node/ + +mdk mono ${GS}/mdk/manual/mdk.html +mdk node ${GS}/mdk/manual/html_node/ + +METAEXCHANGE = http://ftp.gwdg.de/pub/gnu2/iwfmdh/doc/texinfo + iwf_mh node ${METAEXCHANGE}/iwf_mh.html + scantest node ${METAEXCHANGE}/scantest.html + +MIT_SCHEME = ${GS}/mit-scheme/documentation + mit-scheme-ref node ${MIT_SCHEME}/mit-scheme-ref/ + mit-scheme-user node ${MIT_SCHEME}/mit-scheme-user/ + sos node ${MIT_SCHEME}/mit-scheme-sos/ + mit-scheme-imail node ${MIT_SCHEME}/mit-scheme-imail/ + +moe mono ${GS}/moe/manual/moe_manual.html + +motti node ${GS}/motti/manual/ + +mpc node http://www.multiprecision.org/index.php?prog=mpc&page=html + +mpfr mono http://www.mpfr.org/mpfr-current/mpfr.html + +mtools mono ${GS}/mtools/manual/mtools.html + +myserver node http://www.myserverproject.net/documentation/ + +nano mono http://www.nano-editor.org/dist/latest/nano.html + +nettle chapter http://www.lysator.liu.se/~nisse/nettle/nettle.html + +ocrad mono ${GS}/ocrad/manual/ocrad_manual.html + +parted mono ${GS}/parted/manual/parted.html +parted node ${GS}/parted/manual/html_node/ + +pascal mono http://www.gnu-pascal.de/gpc/ + +# can't use pcb since url's contain dates --30nov10 + +perl mono ${GS}/perl/manual/perldoc-all.html + +PIES = http://www.gnu.org.ua/software/pies/manual +pies mono ${PIES}/pies.html +pies chapter ${PIES}/html_chapter/ +pies section ${PIES}/html_section/ +pies node ${PIES}/html_node/ + +plotutils mono ${GS}/plotutils/manual/en/plotutils.html +plotutils node ${GS}/plotutils/manual/en/html_node/ + +proxyknife mono ${GS}/proxyknife/manual/proxyknife.html +proxyknife node ${GS}/proxyknife/manual/html_node/ + +pspp mono ${GS}/pspp/manual/pspp.html +pspp node ${GS}/pspp/manual/html_node/ + +pyconfigure mono ${GS}/pyconfigure/manual/pyconfigure.html +pyconfigure node ${GS}/pyconfigure/manual/html_node/ + +R = http://cran.r-project.org/doc/manuals + R-intro mono ${R}/R-intro.html + R-lang mono ${R}/R-lang.html + R-exts mono ${R}/R-exts.html + R-data mono ${R}/R-data.html + R-admin mono ${R}/R-admin.html + R-ints mono ${R}/R-ints.html + +rcs mono ${GS}/rcs/manual/rcs.html +rcs node ${GS}/rcs/manual/html_node/ + +READLINE = http://cnswww.cns.cwru.edu/php/chet/readline +readline mono ${READLINE}/readline.html + rluserman mono ${READLINE}/rluserman.html + history mono ${READLINE}/history.html + +recode mono http://recode.progiciels-bpi.ca/manual/index.html + +recutils mono ${GS}/recutils/manual/recutils.html +recutils node ${GS}/recutils/manual/html_node/ + +reftex mono ${GS}/auctex/manual/reftex.html +reftex node ${GS}/auctex/manual/reftex/ + +remotecontrol mono ${GS}/remotecontrol/manual/remotecontrol.html +remotecontrol node ${GS}/remotecontrol/manual/html_node/ + +rottlog mono ${GS}/rottlog/manual/rottlog.html +rottlog node ${GS}/rottlog/manual/html_node/ + +RUSH = http://www.gnu.org.ua/software/rush/manual +rush mono ${RUSH}/rush.html +rush chapter ${RUSH}/html_chapter/ +rush section ${RUSH}/html_section/ +rush node ${RUSH}/html_node/ + +screen mono ${GS}/screen/manual/screen.html +screen node ${GS}/screen/manual/html_node/ + +sed mono ${GS}/sed/manual/sed.html +sed node ${GS}/sed/manual/html_node/ + +sharutils mono ${GS}/sharutils/manual/html_mono/sharutils.html +sharutils chapter ${GS}/sharutils/manual/html_chapter/ +sharutils node ${GS}/sharutils/manual/html_node/ + +shepherd mono ${GS}/shepherd/manual/shepherd.html +shepherd node ${GS}/shepherd/manual/html_node/ + +# can't use mono files since they have generic names +SMALLTALK = ${GS}/smalltalk +smalltalk node ${SMALLTALK}/manual/html_node/ + smalltalk-base node ${SMALLTALK}/manual-base/html_node/ + smalltalk-libs node ${SMALLTALK}/manual-libs/html_node/ + +sourceinstall mono ${GS}/sourceinstall/manual/sourceinstall.html +sourceinstall node ${GS}/sourceinstall/manual/html_node/ + +sqltutor mono ${GS}/sqltutor/manual/sqltutor.html +sqltutor node ${GS}/sqltutor/manual/html_node/ + +src-highlite mono ${GS}/src-highlite/source-highlight.html + +swbis mono ${GS}/swbis/manual.html + +tar mono ${GS}/tar/manual/tar.html +tar chapter ${GS}/tar/manual/html_chapter/ +tar section ${GS}/tar/manual/html_section/ +tar node ${GS}/autoconf/manual/html_node/ + +teseq mono ${GS}/teseq/teseq.html +teseq node ${GS}/teseq/html_node/ + +TEXINFO = ${GS}/texinfo/manual +texinfo mono ${TEXINFO}/texinfo/texinfo.html +texinfo node ${TEXINFO}/texinfo/html_node/ + # + info mono ${TEXINFO}/info/info.html + info node ${TEXINFO}/info/html_node/ + # + info-stnd mono ${TEXINFO}/info-stnd/info-stnd.html + info-stnd node ${TEXINFO}/info-stnd/html_node/ + +thales node ${GS}/thales/manual/ + +units mono ${GS}/units/manual/units.html +units node ${GS}/units/manual/html_node/ + +vc-dwim mono ${GS}/vc-dwim/manual/vc-dwim.html +vc-dwim node ${GS}/vc-dwim/manual/html_node/ + +wdiff mono ${GS}/wdiff/manual/wdiff.html +wdiff node ${GS}/wdiff/manual/html_node/ + +websocket4j mono ${GS}/websocket4j/manual/websocket4j.html +websocket4j node ${GS}/websocket4j/manual/html_node/ + +wget mono ${GS}/wget/manual/wget.html +wget node ${GS}/wget/manual/html_node/ + +xboard mono ${GS}/xboard/manual/xboard.html +xboard node ${GS}/xboard/manual/html_node/ + +# emacs-page +# Free TeX-related Texinfo manuals on tug.org. + +T = http://tug.org/texinfohtml + +dvipng mono ${T}/dvipng.html +dvips mono ${T}/dvips.html +eplain mono ${T}/eplain.html +kpathsea mono ${T}/kpathsea.html +latex2e mono ${T}/latex2e.html +tlbuild mono ${T}/tlbuild.html +web2c mono ${T}/web2c.html + + +# Local Variables: +# eval: (add-hook 'write-file-hooks 'time-stamp) +# time-stamp-start: "htmlxrefversion=" +# time-stamp-format: "%:y-%02m-%02d.%02H" +# time-stamp-time-zone: "UTC" +# time-stamp-end: "; # UTC" +# End: diff --git a/doc/documentation/images/daemon_lego_block.png b/doc/documentation/images/daemon_lego_block.png Binary files differnew file mode 100644 index 0000000000..5a088b5321 --- /dev/null +++ b/doc/documentation/images/daemon_lego_block.png diff --git a/doc/documentation/images/daemon_lego_block.svg b/doc/documentation/images/daemon_lego_block.svg new file mode 100644 index 0000000000..38ad90d134 --- /dev/null +++ b/doc/documentation/images/daemon_lego_block.svg @@ -0,0 +1,126 @@ +<?xml version="1.0" encoding="UTF-8" standalone="no"?> +<!-- Created with Inkscape (http://www.inkscape.org/) --> + +<svg + xmlns:dc="http://purl.org/dc/elements/1.1/" + xmlns:cc="http://creativecommons.org/ns#" + xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" + xmlns:svg="http://www.w3.org/2000/svg" + xmlns="http://www.w3.org/2000/svg" + xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd" + xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape" + width="744.09448819" + height="1052.3622047" + id="svg6781" + version="1.1" + inkscape:version="0.48.2 r9819" + sodipodi:docname="New document 58"> + <defs + id="defs6783" /> + <sodipodi:namedview + id="base" + pagecolor="#ffffff" + bordercolor="#666666" + borderopacity="1.0" + inkscape:pageopacity="0.0" + inkscape:pageshadow="2" + inkscape:zoom="0.35" + inkscape:cx="375" + inkscape:cy="520" + inkscape:document-units="px" + inkscape:current-layer="layer1" + showgrid="false" + inkscape:window-width="1366" + inkscape:window-height="721" + inkscape:window-x="-2" + inkscape:window-y="-3" + inkscape:window-maximized="1" /> + <metadata + id="metadata6786"> + <rdf:RDF> + <cc:Work + rdf:about=""> + <dc:format>image/svg+xml</dc:format> + <dc:type + rdf:resource="http://purl.org/dc/dcmitype/StillImage" /> + <dc:title></dc:title> + </cc:Work> + </rdf:RDF> + </metadata> + <g + inkscape:label="Layer 1" + inkscape:groupmode="layer" + id="layer1"> + <g + transform="translate(-4.5298577,-148.04661)" + id="g6746"> + <path + style="fill:#5fd38d;fill-opacity:1;stroke:#faf6a2;stroke-width:1.99014676;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dashoffset:0" + d="m 183.84284,595.7683 350.8064,0 0,202.04036 -350.8064,0 z" + id="path6693" + inkscape:connector-curvature="0" /> + <rect + style="fill:#ffffff;fill-opacity:1;stroke:#faf6a2;stroke-width:2.22747946;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dasharray:none;stroke-dashoffset:0" + id="rect6695" + width="66.670067" + height="75.18058" + x="223.74881" + y="737.19458" /> + <rect + y="737.19458" + x="331.83514" + height="67.323441" + width="66.670067" + id="rect6697" + style="fill:#ffffff;fill-opacity:1;stroke:#faf6a2;stroke-width:2.10787106;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dasharray:none;stroke-dashoffset:0" /> + <rect + style="fill:#ffffff;fill-opacity:1;stroke:#faf6a2;stroke-width:2.06117821;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dasharray:none;stroke-dashoffset:0" + id="rect6699" + width="66.670067" + height="64.373825" + x="434.8707" + y="737.19458" /> + <path + style="fill:#37c871;fill-opacity:1;stroke:#faf6a2;stroke-width:0.98368376;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dashoffset:0" + d="m 223.60976,736.21851 67.23534,0.38374 0,23.63648 -67.23534,37.13818 z" + id="path6701" + inkscape:connector-curvature="0" + sodipodi:nodetypes="ccccc" /> + <path + sodipodi:nodetypes="ccccc" + inkscape:connector-curvature="0" + id="path6703" + d="m 331.69608,736.21851 67.23534,0.3894 0,23.98466 -67.23534,37.68524 z" + style="fill:#37c871;fill-opacity:1;stroke:#faf6a2;stroke-width:0.99090236;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dashoffset:0" /> + <path + style="fill:#37c871;fill-opacity:1;stroke:#faf6a2;stroke-width:0.98368376;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dashoffset:0" + d="m 434.73164,736.21851 67.23534,0.38374 0,23.63648 -67.23534,37.13818 z" + id="path6705" + inkscape:connector-curvature="0" + sodipodi:nodetypes="ccccc" /> + <path + style="fill:#37c871;fill-opacity:1;stroke:#faf6a2;stroke-width:1.92068994;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dashoffset:0" + d="m 533.72659,595.02309 56.12366,-29.34622 -1.01015,190.24271 -55.11351,41.45733 z" + id="path6707" + inkscape:connector-curvature="0" + sodipodi:nodetypes="ccccc" /> + <path + style="fill:#37c871;fill-opacity:1;stroke:#faf6a2;stroke-width:1.99424875;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dashoffset:0" + d="m 245.46708,566.47881 345.46203,-1.01015 -56.56854,31.31472 -349.50264,-1.01014 z" + id="path6709" + inkscape:connector-curvature="0" + sodipodi:nodetypes="ccccc" /> + <text + sodipodi:linespacing="125%" + id="text6715" + y="677.59558" + x="234.35539" + style="font-size:36px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + xml:space="preserve"><tspan + y="677.59558" + x="234.35539" + id="tspan6717" + sodipodi:role="line">User Interface</tspan></text> + </g> + </g> +</svg> diff --git a/doc/documentation/images/gnunet-0-10-peerinfo.png b/doc/documentation/images/gnunet-0-10-peerinfo.png Binary files differnew file mode 100644 index 0000000000..c5e711aff5 --- /dev/null +++ b/doc/documentation/images/gnunet-0-10-peerinfo.png diff --git a/doc/documentation/images/gnunet-fs-gtk-0-10-star-tab.png b/doc/documentation/images/gnunet-fs-gtk-0-10-star-tab.png Binary files differnew file mode 100644 index 0000000000..d7993cc468 --- /dev/null +++ b/doc/documentation/images/gnunet-fs-gtk-0-10-star-tab.png diff --git a/doc/documentation/images/gnunet-gtk-0-10-download-area.png b/doc/documentation/images/gnunet-gtk-0-10-download-area.png Binary files differnew file mode 100644 index 0000000000..8500d46c99 --- /dev/null +++ b/doc/documentation/images/gnunet-gtk-0-10-download-area.png diff --git a/doc/documentation/images/gnunet-gtk-0-10-fs-menu.png b/doc/documentation/images/gnunet-gtk-0-10-fs-menu.png Binary files differnew file mode 100644 index 0000000000..dc20c45a95 --- /dev/null +++ b/doc/documentation/images/gnunet-gtk-0-10-fs-menu.png diff --git a/doc/documentation/images/gnunet-gtk-0-10-fs-publish-editing.png b/doc/documentation/images/gnunet-gtk-0-10-fs-publish-editing.png Binary files differnew file mode 100644 index 0000000000..6f9f75ea6d --- /dev/null +++ b/doc/documentation/images/gnunet-gtk-0-10-fs-publish-editing.png diff --git a/doc/documentation/images/gnunet-gtk-0-10-fs-publish-select.png b/doc/documentation/images/gnunet-gtk-0-10-fs-publish-select.png Binary files differnew file mode 100644 index 0000000000..50672e379a --- /dev/null +++ b/doc/documentation/images/gnunet-gtk-0-10-fs-publish-select.png diff --git a/doc/documentation/images/gnunet-gtk-0-10-fs-publish-with-file.png b/doc/documentation/images/gnunet-gtk-0-10-fs-publish-with-file.png Binary files differnew file mode 100644 index 0000000000..b465425630 --- /dev/null +++ b/doc/documentation/images/gnunet-gtk-0-10-fs-publish-with-file.png diff --git a/doc/documentation/images/gnunet-gtk-0-10-fs-publish-with-file_0.png b/doc/documentation/images/gnunet-gtk-0-10-fs-publish-with-file_0.png Binary files differnew file mode 100644 index 0000000000..b465425630 --- /dev/null +++ b/doc/documentation/images/gnunet-gtk-0-10-fs-publish-with-file_0.png diff --git a/doc/documentation/images/gnunet-gtk-0-10-fs-publish.png b/doc/documentation/images/gnunet-gtk-0-10-fs-publish.png Binary files differnew file mode 100644 index 0000000000..033b38fa5b --- /dev/null +++ b/doc/documentation/images/gnunet-gtk-0-10-fs-publish.png diff --git a/doc/documentation/images/gnunet-gtk-0-10-fs-published.png b/doc/documentation/images/gnunet-gtk-0-10-fs-published.png Binary files differnew file mode 100644 index 0000000000..fbd3dd6a35 --- /dev/null +++ b/doc/documentation/images/gnunet-gtk-0-10-fs-published.png diff --git a/doc/documentation/images/gnunet-gtk-0-10-fs-search.png b/doc/documentation/images/gnunet-gtk-0-10-fs-search.png Binary files differnew file mode 100644 index 0000000000..bb64ab92e8 --- /dev/null +++ b/doc/documentation/images/gnunet-gtk-0-10-fs-search.png diff --git a/doc/documentation/images/gnunet-gtk-0-10-fs.png b/doc/documentation/images/gnunet-gtk-0-10-fs.png Binary files differnew file mode 100644 index 0000000000..c7a2948782 --- /dev/null +++ b/doc/documentation/images/gnunet-gtk-0-10-fs.png diff --git a/doc/documentation/images/gnunet-gtk-0-10-gns-a-done.png b/doc/documentation/images/gnunet-gtk-0-10-gns-a-done.png Binary files differnew file mode 100644 index 0000000000..f8231b3a62 --- /dev/null +++ b/doc/documentation/images/gnunet-gtk-0-10-gns-a-done.png diff --git a/doc/documentation/images/gnunet-gtk-0-10-gns-a.png b/doc/documentation/images/gnunet-gtk-0-10-gns-a.png Binary files differnew file mode 100644 index 0000000000..39858d72c1 --- /dev/null +++ b/doc/documentation/images/gnunet-gtk-0-10-gns-a.png diff --git a/doc/documentation/images/gnunet-gtk-0-10-gns.png b/doc/documentation/images/gnunet-gtk-0-10-gns.png Binary files differnew file mode 100644 index 0000000000..c71a2bd7be --- /dev/null +++ b/doc/documentation/images/gnunet-gtk-0-10-gns.png diff --git a/doc/documentation/images/gnunet-gtk-0-10-identity.png b/doc/documentation/images/gnunet-gtk-0-10-identity.png Binary files differnew file mode 100644 index 0000000000..d0b426098a --- /dev/null +++ b/doc/documentation/images/gnunet-gtk-0-10-identity.png diff --git a/doc/documentation/images/gnunet-gtk-0-10-search-selected.png b/doc/documentation/images/gnunet-gtk-0-10-search-selected.png Binary files differnew file mode 100644 index 0000000000..da1ad4d31e --- /dev/null +++ b/doc/documentation/images/gnunet-gtk-0-10-search-selected.png diff --git a/doc/documentation/images/gnunet-gtk-0-10-traffic.png b/doc/documentation/images/gnunet-gtk-0-10-traffic.png Binary files differnew file mode 100644 index 0000000000..76458f717a --- /dev/null +++ b/doc/documentation/images/gnunet-gtk-0-10-traffic.png diff --git a/doc/documentation/images/gnunet-gtk-0-10.png b/doc/documentation/images/gnunet-gtk-0-10.png Binary files differnew file mode 100644 index 0000000000..3615849a79 --- /dev/null +++ b/doc/documentation/images/gnunet-gtk-0-10.png diff --git a/doc/documentation/images/gnunet-namestore-gtk-phone.png b/doc/documentation/images/gnunet-namestore-gtk-phone.png Binary files differnew file mode 100644 index 0000000000..3bb859629a --- /dev/null +++ b/doc/documentation/images/gnunet-namestore-gtk-phone.png diff --git a/doc/documentation/images/gnunet-namestore-gtk-vpn.png b/doc/documentation/images/gnunet-namestore-gtk-vpn.png Binary files differnew file mode 100644 index 0000000000..c716729ba2 --- /dev/null +++ b/doc/documentation/images/gnunet-namestore-gtk-vpn.png diff --git a/doc/documentation/images/gnunet-setup-exit.png b/doc/documentation/images/gnunet-setup-exit.png Binary files differnew file mode 100644 index 0000000000..66bd972bc0 --- /dev/null +++ b/doc/documentation/images/gnunet-setup-exit.png diff --git a/doc/documentation/images/gnunet-tutorial-service.png b/doc/documentation/images/gnunet-tutorial-service.png Binary files differnew file mode 100644 index 0000000000..6daed2f35f --- /dev/null +++ b/doc/documentation/images/gnunet-tutorial-service.png diff --git a/doc/documentation/images/gnunet-tutorial-system.png b/doc/documentation/images/gnunet-tutorial-system.png Binary files differnew file mode 100644 index 0000000000..8b54e16cf2 --- /dev/null +++ b/doc/documentation/images/gnunet-tutorial-system.png diff --git a/doc/documentation/images/iceweasel-preferences.png b/doc/documentation/images/iceweasel-preferences.png Binary files differnew file mode 100644 index 0000000000..e62c2c4d99 --- /dev/null +++ b/doc/documentation/images/iceweasel-preferences.png diff --git a/doc/documentation/images/iceweasel-proxy.png b/doc/documentation/images/iceweasel-proxy.png Binary files differnew file mode 100644 index 0000000000..9caad4508d --- /dev/null +++ b/doc/documentation/images/iceweasel-proxy.png diff --git a/doc/documentation/images/lego_stack.svg b/doc/documentation/images/lego_stack.svg new file mode 100644 index 0000000000..a0e8017c39 --- /dev/null +++ b/doc/documentation/images/lego_stack.svg @@ -0,0 +1,737 @@ +<?xml version="1.0" encoding="UTF-8" standalone="no"?> +<!-- Created with Inkscape (http://www.inkscape.org/) --> + +<svg + xmlns:osb="http://www.openswatchbook.org/uri/2009/osb" + xmlns:dc="http://purl.org/dc/elements/1.1/" + xmlns:cc="http://creativecommons.org/ns#" + xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" + xmlns:svg="http://www.w3.org/2000/svg" + xmlns="http://www.w3.org/2000/svg" + xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd" + xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape" + width="744.09448819" + height="1052.3622047" + id="svg2" + version="1.1" + inkscape:version="0.48.1 r9760" + sodipodi:docname="System.svg"> + <defs + id="defs4"> + <linearGradient + id="linearGradient6602"> + <stop + style="stop-color:#df8060;stop-opacity:1;" + offset="0" + id="stop6604" /> + <stop + style="stop-color:#df8002;stop-opacity:0;" + offset="1" + id="stop6606" /> + </linearGradient> + <linearGradient + id="linearGradient4392" + osb:paint="solid"> + <stop + style="stop-color:#faf6a6;stop-opacity:1;" + offset="0" + id="stop4394" /> + </linearGradient> + <inkscape:perspective + sodipodi:type="inkscape:persp3d" + inkscape:vp_x="883.99395 : 559.99673 : 1" + inkscape:vp_y="13.319386 : 993.87659 : 0" + inkscape:vp_z="285.3157 : 504.79962 : 1" + inkscape:persp3d-origin="481.39556 : 281.96355 : 1" + id="perspective3070" /> + <inkscape:perspective + sodipodi:type="inkscape:persp3d" + inkscape:vp_x="76.097926 : 349.87282 : 1" + inkscape:vp_y="-13.319386 : 979.366 : 0" + inkscape:vp_z="752.55793 : 376.31441 : 1" + inkscape:persp3d-origin="373.64045 : 350.98006 : 1" + id="perspective3012" /> + </defs> + <sodipodi:namedview + id="base" + pagecolor="#ffffff" + bordercolor="#666666" + borderopacity="1.0" + inkscape:pageopacity="0.0" + inkscape:pageshadow="2" + inkscape:zoom="0.98994949" + inkscape:cx="322.06882" + inkscape:cy="568.82291" + inkscape:document-units="px" + inkscape:current-layer="layer1" + showgrid="false" + inkscape:window-width="846" + inkscape:window-height="963" + inkscape:window-x="59" + inkscape:window-y="0" + inkscape:window-maximized="0" /> + <metadata + id="metadata7"> + <rdf:RDF> + <cc:Work + rdf:about=""> + <dc:format>image/svg+xml</dc:format> + <dc:type + rdf:resource="http://purl.org/dc/dcmitype/StillImage" /> + <dc:title /> + </cc:Work> + </rdf:RDF> + </metadata> + <g + inkscape:label="Layer 1" + inkscape:groupmode="layer" + id="layer1"> + <text + xml:space="preserve" + style="font-size:40px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + x="352.03815" + y="-190.12544" + id="text6623" + sodipodi:linespacing="125%"><tspan + sodipodi:role="line" + id="tspan6625" + x="352.03815" + y="-190.12544" /></text> + <text + xml:space="preserve" + style="font-size:40px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + x="338.40109" + y="-300.73715" + id="text6627" + sodipodi:linespacing="125%"><tspan + sodipodi:role="line" + id="tspan6629" + x="338.40109" + y="-300.73715" /></text> + <g + style="font-size:24px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + id="text6643" /> + <text + xml:space="preserve" + style="font-size:24px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + x="71.720833" + y="95.747719" + id="text6648" + sodipodi:linespacing="125%"><tspan + sodipodi:role="line" + id="tspan6650" + x="71.720833" + y="95.747719" /></text> + <text + xml:space="preserve" + style="font-size:32px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + x="262.63965" + y="666.48389" + id="text6711" + sodipodi:linespacing="125%"><tspan + sodipodi:role="line" + id="tspan6713" + x="262.63965" + y="666.48389" /></text> + <path + inkscape:connector-curvature="0" + id="path3506" + d="m 198.00647,673.76257 236.93358,0 0,158.2919 -236.93358,0 z" + style="fill:#000000;fill-opacity:1;stroke:#faf6a2;stroke-width:1.44768786;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dashoffset:0" /> + <path + sodipodi:nodetypes="ccccc" + inkscape:connector-curvature="0" + id="path3432" + d="m 169.32669,654.90334 464.83332,-2.26992 -33.76593,25.73079 -483.97287,-0.12904 z" + style="fill:#deaa87;fill-opacity:1;stroke:#faf6a2;stroke-width:2.18398547;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dashoffset:0" /> + <rect + style="fill:#000000;fill-opacity:1;stroke:#59000c;stroke-width:1.35822594;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0.48230088;stroke-dasharray:none;stroke-dashoffset:0" + id="rect3416" + width="28.495705" + height="172.2845" + x="464.19418" + y="518.96954" /> + <rect + style="fill:#000000;fill-opacity:1;stroke:#59000c;stroke-width:1.36876941;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0.48230088;stroke-dasharray:none;stroke-dashoffset:0" + id="rect3414" + width="34.729141" + height="170.67587" + x="340.86124" + y="517.93475" /> + <path + style="fill:#deaa87;fill-opacity:1;stroke:#faf6a2;stroke-width:2.04969239;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dashoffset:0" + d="m 246.8138,499.06358 386.50295,-0.94821 -41.88736,26.04231 -413.96081,0 z" + id="rect6568-0" + inkscape:connector-curvature="0" + sodipodi:nodetypes="ccccc" /> + <path + style="fill:#ffdd55;fill-opacity:1;stroke:#faf6a2;stroke-width:1.49989259;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dashoffset:0" + d="m 276.05867,399.52042 323.05541,0 0,124.61741 -323.05541,0 z" + id="rect5973" + inkscape:connector-curvature="0" /> + <path + style="fill:#ffcc00;fill-opacity:1;stroke:#faf6a2;stroke-width:1.19094384;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dashoffset:0" + d="m 599.16863,399.06078 34.35465,-18.10059 0,117.34068 -34.35465,25.57066 z" + id="rect6542" + inkscape:connector-curvature="0" + sodipodi:nodetypes="ccccc" /> + <rect + style="fill:#c87137;fill-opacity:1;stroke:#c48069;stroke-width:1.50087094;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dasharray:none;stroke-dashoffset:0" + id="rect6557" + width="322.88623" + height="30.529778" + x="276.67755" + y="368.99368" /> + <path + style="fill:#ff9955;fill-opacity:1;stroke:#faf6a2;stroke-width:0.50882494;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dashoffset:0" + d="m 598.94047,368.99367 34.58281,-16.82253 0,28.66061 -34.58281,18.06864 z" + id="rect6561" + inkscape:connector-curvature="0" + sodipodi:nodetypes="ccccc" /> + <path + sodipodi:nodetypes="ccccc" + inkscape:connector-curvature="0" + id="path6564" + d="m 598.94047,369.07046 34.30741,-17.12981 0,29.18412 -34.30741,18.39868 z" + style="fill:#c87137;fill-opacity:1;stroke:#faf6a2;stroke-width:0.51140249;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dashoffset:0" + inkscape:transform-center-x="-70.147578" + inkscape:transform-center-y="15.429055" /> + <path + style="fill:#d38d5f;fill-opacity:1;stroke:#faf6a2;stroke-width:1.47079194;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dashoffset:0" + d="m 330.2508,353.23478 302.87005,-0.62306 -38.33414,16.82253 -318.87597,0 z" + id="rect6568" + inkscape:connector-curvature="0" + sodipodi:nodetypes="ccccc" /> + <text + xml:space="preserve" + style="font-size:22.32217598px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + x="391.63083" + y="461.858" + id="text6656" + sodipodi:linespacing="125%" + transform="scale(1.0052948,0.9947331)"><tspan + sodipodi:role="line" + id="tspan6658" + x="391.63083" + y="461.858">Service</tspan></text> + <path + sodipodi:nodetypes="ccccc" + inkscape:connector-curvature="0" + id="path6707" + d="m 598.75503,244.83802 34.98432,-18.10059 0.26082,125.2709 -35.24514,17.64044 z" + style="fill:#37c871;fill-opacity:1;stroke:#faf6a2;stroke-width:1.19094384;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dashoffset:0" /> + <path + sodipodi:nodetypes="ccccc" + inkscape:connector-curvature="0" + id="path6709" + d="m 419.07032,228.1132 214.71185,-1.24611 -34.63196,19.9378 L 381.29,246.18184 z" + style="fill:#37c871;fill-opacity:1;stroke:#faf6a2;stroke-width:1.23655474;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dashoffset:0" /> + <rect + style="fill:#5fd38d;fill-opacity:1;stroke:#c48069;stroke-width:1.23640049;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dasharray:none;stroke-dashoffset:0" + id="rect7224" + width="217.86653" + height="122.74216" + x="381.70358" + y="246.25151" /> + <text + xml:space="preserve" + style="font-size:22.32217598px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + x="409.16376" + y="302.05649" + id="text6715" + sodipodi:linespacing="125%" + transform="scale(1.0052948,0.9947331)"><tspan + sodipodi:role="line" + id="tspan6717" + x="409.16376" + y="302.05649">User Interface</tspan></text> + <g + id="g7219" + transform="matrix(0.62334353,0,0,0.61679464,281.18563,257.70936)"> + <rect + y="119.99139" + x="198.49498" + height="60.609154" + width="66.670067" + id="rect6571" + style="fill:#ff9955;fill-opacity:1;stroke:#faf6a2;stroke-width:2;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dasharray:none;stroke-dashoffset:0" /> + <text + sodipodi:linespacing="125%" + id="text6631" + y="160.39748" + x="206.07112" + style="font-size:32px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + xml:space="preserve"><tspan + y="160.39748" + x="206.07112" + id="tspan6633" + sodipodi:role="line">API</tspan></text> + </g> + <g + transform="matrix(0.62334353,0,0,0.61679464,344.78251,257.70936)" + id="g7226"> + <rect + style="fill:#ff9955;fill-opacity:1;stroke:#faf6a2;stroke-width:2;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dasharray:none;stroke-dashoffset:0" + id="rect7228" + width="66.670067" + height="60.609154" + x="198.49498" + y="119.99139" /> + <text + xml:space="preserve" + style="font-size:32px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + x="206.07112" + y="160.39748" + id="text7230" + sodipodi:linespacing="125%"><tspan + sodipodi:role="line" + id="tspan7232" + x="206.07112" + y="160.39748">API</tspan></text> + </g> + <g + id="g7234" + transform="matrix(0.62334353,0,0,0.61679464,409.3239,257.70936)"> + <rect + y="119.99139" + x="198.49498" + height="60.609154" + width="66.670067" + id="rect7236" + style="fill:#ff9955;fill-opacity:1;stroke:#faf6a2;stroke-width:2;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dasharray:none;stroke-dashoffset:0" /> + <text + sodipodi:linespacing="125%" + id="text7238" + y="160.39748" + x="206.07112" + style="font-size:32px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + xml:space="preserve"><tspan + y="160.39748" + x="206.07112" + id="tspan7240" + sodipodi:role="line">API</tspan></text> + </g> + <g + transform="matrix(0.62334353,0,0,0.61679464,175.75806,412.85048)" + id="g7242"> + <rect + style="fill:#ff9955;fill-opacity:1;stroke:#faf6a2;stroke-width:2;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dasharray:none;stroke-dashoffset:0" + id="rect7244" + width="66.670067" + height="60.609154" + x="198.49498" + y="119.99139" /> + <text + xml:space="preserve" + style="font-size:32px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + x="206.07112" + y="160.39748" + id="text7246" + sodipodi:linespacing="125%"><tspan + sodipodi:role="line" + id="tspan7248" + x="206.07112" + y="160.39748">API</tspan></text> + </g> + <g + id="g7250" + transform="matrix(0.62334353,0,0,0.61679464,240.79871,413.29105)"> + <rect + y="119.99139" + x="198.49498" + height="60.609154" + width="66.670067" + id="rect7252" + style="fill:#ff9955;fill-opacity:1;stroke:#faf6a2;stroke-width:2;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dasharray:none;stroke-dashoffset:0" /> + <text + sodipodi:linespacing="125%" + id="text7254" + y="160.39748" + x="206.07112" + style="font-size:32px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + xml:space="preserve"><tspan + y="160.39748" + x="206.07112" + id="tspan7256" + sodipodi:role="line">API</tspan></text> + </g> + <g + transform="matrix(0.62334353,0,0,0.61679464,303.79756,412.40991)" + id="g7258"> + <rect + style="fill:#ff9955;fill-opacity:1;stroke:#faf6a2;stroke-width:2;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dasharray:none;stroke-dashoffset:0" + id="rect7260" + width="66.670067" + height="60.609154" + x="198.49498" + y="119.99139" /> + <text + xml:space="preserve" + style="font-size:32px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + x="206.07112" + y="160.39748" + id="text7262" + sodipodi:linespacing="125%"><tspan + sodipodi:role="line" + id="tspan7264" + x="206.07112" + y="160.39748">API</tspan></text> + </g> + <g + id="g7266" + transform="matrix(0.62334353,0,0,0.61679464,369.88148,412.40991)"> + <rect + y="119.99139" + x="198.49498" + height="60.609154" + width="66.670067" + id="rect7268" + style="fill:#ff9955;fill-opacity:1;stroke:#faf6a2;stroke-width:2;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dasharray:none;stroke-dashoffset:0" /> + <text + sodipodi:linespacing="125%" + id="text7270" + y="160.39748" + x="206.07112" + style="font-size:32px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + xml:space="preserve"><tspan + y="160.39748" + x="206.07112" + id="tspan7272" + sodipodi:role="line">API</tspan></text> + </g> + <path + style="fill:#ffeeaa;fill-opacity:1;stroke:#faf6a2;stroke-width:0.91879815;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dashoffset:0" + d="m 478.56081,554.09281 121.22633,0 0,124.61741 -121.22633,0 z" + id="rect5973-1" + inkscape:connector-curvature="0" /> + <g + transform="matrix(0.62334353,0,0,0.61679464,422.424,566.60858)" + id="g3474"> + <rect + style="fill:#ff9955;fill-opacity:1;stroke:#faf6a2;stroke-width:2;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dasharray:none;stroke-dashoffset:0" + id="rect3476" + width="66.670067" + height="60.609154" + x="198.49498" + y="119.99139" /> + <text + xml:space="preserve" + style="font-size:32px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + x="206.07112" + y="160.39748" + id="text3478" + sodipodi:linespacing="125%"><tspan + sodipodi:role="line" + id="tspan3480" + x="206.07112" + y="160.39748">API</tspan></text> + </g> + <path + style="fill:#ffe680;fill-opacity:1;stroke:#faf6a2;stroke-width:1.18771458;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dashoffset:0" + d="m 599.60339,554.02055 33.72575,-29.55535 0.88568,128.35487 -34.61143,26.01123 z" + id="rect6542-8" + inkscape:connector-curvature="0" + sodipodi:nodetypes="ccccc" /> + <path + sodipodi:nodetypes="ccccc" + inkscape:connector-curvature="0" + id="path6564-2" + d="m 598.92998,524.03024 34.30741,-25.94116 0,26.5407 -34.30741,29.85344 z" + style="fill:#d38d5f;fill-opacity:1;stroke:#faf6a2;stroke-width:0.51140249;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dashoffset:0" + inkscape:transform-center-x="-70.147578" + inkscape:transform-center-y="15.429055" /> + <path + inkscape:transform-center-y="15.492457" + inkscape:transform-center-x="-70.147578" + style="fill:#c87137;fill-opacity:1;stroke:#faf6a2;stroke-width:0.51245213;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dashoffset:0" + d="m 598.92998,524.13683 34.30741,-26.04775 0,26.64977 -34.30741,29.97611 z" + id="path3402" + inkscape:connector-curvature="0" + sodipodi:nodetypes="ccccc" /> + <path + sodipodi:nodetypes="ccccc" + inkscape:connector-curvature="0" + id="path3404" + d="m 599.82047,678.2289 34.30741,-25.94116 0,26.5407 -34.30741,34.25912 z" + style="fill:#c87137;fill-opacity:1;stroke:#faf6a2;stroke-width:0.51140249;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dashoffset:0" + inkscape:transform-center-x="-70.147578" + inkscape:transform-center-y="15.429055" /> + <path + sodipodi:nodetypes="ccccc" + inkscape:connector-curvature="0" + id="path3406" + d="m 600.04863,707.77865 33.90941,-29.55535 0.89049,128.35487 -34.7999,26.01123 z" + style="fill:#37c837;fill-opacity:1;stroke:#faf6a2;stroke-width:1.19094384;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dashoffset:0" /> + <path + inkscape:connector-curvature="0" + id="path3410" + d="m 356.56358,554.09281 120.92249,0 0,124.19268 -120.92249,0 z" + style="fill:#ffeeaa;fill-opacity:1;stroke:#faf6a2;stroke-width:0.91608089;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dashoffset:0" /> + <path + style="fill:#ffeeaa;fill-opacity:1;stroke:#faf6a2;stroke-width:1.11023378;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dashoffset:0" + d="m 177.52518,554.09281 177.51841,0 0,124.25702 -177.51841,0 z" + id="path3412" + inkscape:connector-curvature="0" /> + <rect + style="fill:#c87137;fill-opacity:1;stroke:#c48069;stroke-width:1.11264122;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dasharray:none;stroke-dashoffset:0" + id="rect6557-7" + width="177.44882" + height="30.529778" + x="177.65657" + y="523.95343" /> + <rect + y="678.1521" + x="116.73995" + height="29.53463" + width="177.54182" + id="rect3408" + style="fill:#c87137;fill-opacity:1;stroke:#c48069;stroke-width:1.09464383;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dasharray:none;stroke-dashoffset:0" /> + <rect + y="523.95343" + x="356.55023" + height="30.529778" + width="120.86897" + id="rect3420" + style="fill:#c87137;fill-opacity:1;stroke:#c48069;stroke-width:0.91828173;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dasharray:none;stroke-dashoffset:0" /> + <rect + style="fill:#c87137;fill-opacity:1;stroke:#c48069;stroke-width:0.91828173;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dasharray:none;stroke-dashoffset:0" + id="rect3422" + width="120.86897" + height="30.529778" + x="478.54919" + y="523.95343" /> + <text + xml:space="preserve" + style="font-size:22.32217598px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + x="372.34232" + y="622.53217" + id="text6656-2" + sodipodi:linespacing="125%" + transform="scale(1.0052948,0.9947331)"><tspan + sodipodi:role="line" + id="tspan6658-2" + x="372.34232" + y="622.53217">Service</tspan></text> + <text + sodipodi:linespacing="125%" + id="text3424" + y="622.53217" + x="220.56013" + style="font-size:22.32217598px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + xml:space="preserve" + transform="scale(1.0052948,0.9947331)"><tspan + y="622.53217" + x="220.56013" + id="tspan3426" + sodipodi:role="line">Service</tspan></text> + <text + xml:space="preserve" + style="font-size:22.32217598px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + x="493.85532" + y="622.54492" + id="text3428" + sodipodi:linespacing="125%" + transform="scale(1.0052948,0.9947331)"><tspan + sodipodi:role="line" + id="tspan3430" + x="493.85532" + y="622.54492">Service</tspan></text> + <g + id="g3434" + transform="matrix(0.62334353,0,0,0.61679464,120.10238,566.60858)"> + <rect + y="119.99139" + x="198.49498" + height="60.609154" + width="66.670067" + id="rect3436" + style="fill:#ff9955;fill-opacity:1;stroke:#faf6a2;stroke-width:2;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dasharray:none;stroke-dashoffset:0" /> + <text + sodipodi:linespacing="125%" + id="text3438" + y="160.39748" + x="206.07112" + style="font-size:32px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + xml:space="preserve"><tspan + y="160.39748" + x="206.07112" + id="tspan3440" + sodipodi:role="line">API</tspan></text> + </g> + <g + transform="matrix(0.62334353,0,0,0.61679464,181.54625,566.60858)" + id="g3442"> + <rect + style="fill:#ff9955;fill-opacity:1;stroke:#faf6a2;stroke-width:2;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dasharray:none;stroke-dashoffset:0" + id="rect3444" + width="66.670067" + height="60.609154" + x="198.49498" + y="119.99139" /> + <text + xml:space="preserve" + style="font-size:32px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + x="206.07112" + y="160.39748" + id="text3446" + sodipodi:linespacing="125%"><tspan + sodipodi:role="line" + id="tspan3448" + x="206.07112" + y="160.39748">API</tspan></text> + </g> + <g + id="g3450" + transform="matrix(0.62334353,0,0,0.61679464,242.09962,566.60858)"> + <rect + y="119.99139" + x="198.49498" + height="60.609154" + width="66.670067" + id="rect3452" + style="fill:#ff9955;fill-opacity:1;stroke:#faf6a2;stroke-width:2;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dasharray:none;stroke-dashoffset:0" /> + <text + sodipodi:linespacing="125%" + id="text3454" + y="160.39748" + x="206.07112" + style="font-size:32px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + xml:space="preserve"><tspan + y="160.39748" + x="206.07112" + id="tspan3456" + sodipodi:role="line">API</tspan></text> + </g> + <g + transform="matrix(0.62334353,0,0,0.61679464,303.54348,566.60858)" + id="g3458"> + <rect + style="fill:#ff9955;fill-opacity:1;stroke:#faf6a2;stroke-width:2;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dasharray:none;stroke-dashoffset:0" + id="rect3460" + width="66.670067" + height="60.609154" + x="198.49498" + y="119.99139" /> + <text + xml:space="preserve" + style="font-size:32px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + x="206.07112" + y="160.39748" + id="text3462" + sodipodi:linespacing="125%"><tspan + sodipodi:role="line" + id="tspan3464" + x="206.07112" + y="160.39748">API</tspan></text> + </g> + <g + id="g3466" + transform="matrix(0.62334353,0,0,0.61679464,362.76112,566.60858)"> + <rect + y="119.99139" + x="198.49498" + height="60.609154" + width="66.670067" + id="rect3468" + style="fill:#ff9955;fill-opacity:1;stroke:#faf6a2;stroke-width:2;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dasharray:none;stroke-dashoffset:0" /> + <text + sodipodi:linespacing="125%" + id="text3470" + y="160.39748" + x="206.07112" + style="font-size:32px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + xml:space="preserve"><tspan + y="160.39748" + x="206.07112" + id="tspan3472" + sodipodi:role="line">API</tspan></text> + </g> + <path + style="fill:#5fd35f;fill-opacity:1;stroke:#faf6a2;stroke-width:1.11993289;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dashoffset:0" + d="m 420.2626,707.53388 180.11119,0 0,124.61741 -180.11119,0 z" + id="path3490" + inkscape:connector-curvature="0" /> + <g + transform="matrix(0.62334353,0,0,0.61679464,62.665728,566.60858)" + id="g3492"> + <rect + style="fill:#ff9955;fill-opacity:1;stroke:#faf6a2;stroke-width:2;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dasharray:none;stroke-dashoffset:0" + id="rect3494" + width="66.670067" + height="60.609154" + x="198.49498" + y="119.99139" /> + <text + xml:space="preserve" + style="font-size:32px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + x="206.07112" + y="160.39748" + id="text3496" + sodipodi:linespacing="125%"><tspan + sodipodi:role="line" + id="tspan3498" + x="206.07112" + y="160.39748">API</tspan></text> + </g> + <path + inkscape:connector-curvature="0" + id="path3500" + d="m 116.52597,707.54132 177.63643,0 0,124.61741 -177.63643,0 z" + style="fill:#5fd35f;fill-opacity:1;stroke:#faf6a2;stroke-width:1.11221218;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dashoffset:0" /> + <path + style="fill:#5fd35f;fill-opacity:1;stroke:#faf6a2;stroke-width:0.92545629;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dashoffset:0" + d="m 295.65636,707.63061 122.98965,0 0,124.61741 -122.98965,0 z" + id="path3502" + inkscape:connector-curvature="0" /> + <text + xml:space="preserve" + style="font-size:22.32217598px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + x="162.54019" + y="779.76184" + id="text3508" + sodipodi:linespacing="125%" + transform="scale(1.0052948,0.9947331)"><tspan + sodipodi:role="line" + id="tspan3510" + x="162.54019" + y="779.76184">Service</tspan></text> + <text + sodipodi:linespacing="125%" + id="text3512" + y="779.7619" + x="313.56918" + style="font-size:22.32217598px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + xml:space="preserve" + transform="scale(1.0052948,0.9947331)"><tspan + y="779.7619" + x="313.56918" + id="tspan3514" + sodipodi:role="line">Service</tspan></text> + <text + xml:space="preserve" + style="font-size:22.32217598px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + x="465.48401" + y="779.7619" + id="text3516" + sodipodi:linespacing="125%" + transform="scale(1.0052948,0.9947331)"><tspan + sodipodi:role="line" + id="tspan3518" + x="465.48401" + y="779.7619">Service</tspan></text> + <rect + style="fill:#c87137;fill-opacity:1;stroke:#c48069;stroke-width:0.91063529;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dasharray:none;stroke-dashoffset:0" + id="rect3520" + width="122.86946" + height="29.53463" + x="295.75125" + y="678.1521" /> + <rect + y="678.1521" + x="420.27423" + height="29.53463" + width="179.80205" + id="rect3522" + style="fill:#c87137;fill-opacity:1;stroke:#c48069;stroke-width:1.10158956;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dasharray:none;stroke-dashoffset:0" /> + </g> +</svg> diff --git a/doc/documentation/images/service_lego_block.png b/doc/documentation/images/service_lego_block.png Binary files differnew file mode 100644 index 0000000000..56caf6b9cc --- /dev/null +++ b/doc/documentation/images/service_lego_block.png diff --git a/doc/documentation/images/service_lego_block.svg b/doc/documentation/images/service_lego_block.svg new file mode 100644 index 0000000000..ef0d0234fd --- /dev/null +++ b/doc/documentation/images/service_lego_block.svg @@ -0,0 +1,345 @@ +<?xml version="1.0" encoding="UTF-8" standalone="no"?> +<!-- Created with Inkscape (http://www.inkscape.org/) --> + +<svg + xmlns:osb="http://www.openswatchbook.org/uri/2009/osb" + xmlns:dc="http://purl.org/dc/elements/1.1/" + xmlns:cc="http://creativecommons.org/ns#" + xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" + xmlns:svg="http://www.w3.org/2000/svg" + xmlns="http://www.w3.org/2000/svg" + xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd" + xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape" + width="744.09448819" + height="1052.3622047" + id="svg2" + version="1.1" + inkscape:version="0.48.2 r9819" + sodipodi:docname="Lego block 3.svg"> + <defs + id="defs4"> + <linearGradient + id="linearGradient6602"> + <stop + style="stop-color:#df8060;stop-opacity:1;" + offset="0" + id="stop6604" /> + <stop + style="stop-color:#df8002;stop-opacity:0;" + offset="1" + id="stop6606" /> + </linearGradient> + <linearGradient + id="linearGradient4392" + osb:paint="solid"> + <stop + style="stop-color:#faf6a6;stop-opacity:1;" + offset="0" + id="stop4394" /> + </linearGradient> + <inkscape:perspective + sodipodi:type="inkscape:persp3d" + inkscape:vp_x="883.99395 : 559.99673 : 1" + inkscape:vp_y="13.319386 : 993.87659 : 0" + inkscape:vp_z="285.3157 : 504.79962 : 1" + inkscape:persp3d-origin="481.39556 : 281.96355 : 1" + id="perspective3070" /> + <inkscape:perspective + sodipodi:type="inkscape:persp3d" + inkscape:vp_x="76.097926 : 349.87282 : 1" + inkscape:vp_y="-13.319386 : 979.366 : 0" + inkscape:vp_z="752.55793 : 376.31441 : 1" + inkscape:persp3d-origin="373.64045 : 350.98006 : 1" + id="perspective3012" /> + </defs> + <sodipodi:namedview + id="base" + pagecolor="#ffffff" + bordercolor="#666666" + borderopacity="1.0" + inkscape:pageopacity="0.0" + inkscape:pageshadow="2" + inkscape:zoom="0.49497475" + inkscape:cx="385.59974" + inkscape:cy="826.03166" + inkscape:document-units="px" + inkscape:current-layer="layer1" + showgrid="false" + inkscape:window-width="1366" + inkscape:window-height="721" + inkscape:window-x="-2" + inkscape:window-y="-3" + inkscape:window-maximized="1" /> + <metadata + id="metadata7"> + <rdf:RDF> + <cc:Work + rdf:about=""> + <dc:format>image/svg+xml</dc:format> + <dc:type + rdf:resource="http://purl.org/dc/dcmitype/StillImage" /> + <dc:title></dc:title> + </cc:Work> + </rdf:RDF> + </metadata> + <g + inkscape:label="Layer 1" + inkscape:groupmode="layer" + id="layer1"> + <path + style="fill:#ffdd55;fill-opacity:1;stroke:#faf6a2;stroke-width:2.26315212;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dashoffset:0" + d="m 74.934278,230.09308 453.654042,0 0,202.04036 -453.654042,0 z" + id="rect5973" + inkscape:connector-curvature="0" /> + <path + style="fill:#ffcc00;fill-opacity:1;stroke:#faf6a2;stroke-width:1.92068994;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dashoffset:0" + d="m 528.67583,229.34787 55.11351,-29.34622 0,190.24271 -55.11351,41.45733 z" + id="rect6542" + inkscape:connector-curvature="0" + sodipodi:nodetypes="ccccc" /> + <rect + style="fill:#d38d5f;fill-opacity:1;stroke:#c48069;stroke-width:2.2674458;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dasharray:none;stroke-dashoffset:0" + id="rect6557" + width="454.54535" + height="49.497475" + x="74.764442" + y="180.60052" /> + <path + style="fill:#ff9955;fill-opacity:1;stroke:#faf6a2;stroke-width:0.8206054;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dashoffset:0" + d="m 528.30981,180.60052 55.47954,-27.27412 0,46.46702 -55.47954,29.29442 z" + id="rect6561" + inkscape:connector-curvature="0" + sodipodi:nodetypes="ccccc" /> + <path + sodipodi:nodetypes="ccccc" + inkscape:connector-curvature="0" + id="path6564" + d="m 528.30981,180.72501 55.03773,-27.7723 0,47.31578 -55.03773,29.8295 z" + style="fill:#d38d5f;fill-opacity:1;stroke:#faf6a2;stroke-width:0.82476228;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dashoffset:0" + inkscape:transform-center-x="-70.147578" + inkscape:transform-center-y="15.429055" /> + <path + style="fill:#deaa87;fill-opacity:1;stroke:#faf6a2;stroke-width:2.23265362;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dashoffset:0" + d="m 153.39374,154.33657 430.4643,-1.01015 -54.4837,27.27411 -453.213248,0 z" + id="rect6568" + inkscape:connector-curvature="0" + sodipodi:nodetypes="ccccc" /> + <text + xml:space="preserve" + style="font-size:40px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + x="352.03815" + y="-190.12544" + id="text6623" + sodipodi:linespacing="125%"><tspan + sodipodi:role="line" + id="tspan6625" + x="352.03815" + y="-190.12544" /></text> + <text + xml:space="preserve" + style="font-size:40px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + x="338.40109" + y="-300.73715" + id="text6627" + sodipodi:linespacing="125%"><tspan + sodipodi:role="line" + id="tspan6629" + x="338.40109" + y="-300.73715" /></text> + <rect + style="fill:#ff9955;fill-opacity:1;stroke:#faf6a2;stroke-width:2;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dasharray:none;stroke-dashoffset:0" + id="rect6571" + width="66.670067" + height="60.609154" + x="198.49498" + y="119.99139" /> + <path + style="fill:#ff6600;fill-opacity:1;stroke:#faf6a2;stroke-width:1.98413372;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dashoffset:0" + d="m 265.16503,119.45792 44.95179,-22.465406 0,57.057986 -44.95179,26.55003 z" + id="path6600" + inkscape:connector-curvature="0" + sodipodi:nodetypes="ccccc" /> + <path + style="fill:#ff6600;fill-opacity:1;stroke:#c48069;stroke-width:1.99687159;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dashoffset:0" + d="m 243.06977,97.26295 66.86223,10e-7 -45.08135,22.728439 -66.3557,0 z" + id="path6617" + inkscape:connector-curvature="0" + sodipodi:nodetypes="ccccc" /> + <text + xml:space="preserve" + style="font-size:32px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + x="206.07112" + y="160.39748" + id="text6631" + sodipodi:linespacing="125%"><tspan + sodipodi:role="line" + id="tspan6633" + x="206.07112" + y="160.39748">API</tspan></text> + <rect + y="119.99139" + x="313.65237" + height="60.609154" + width="66.670067" + id="rect6573" + style="fill:#ff9955;fill-opacity:1;stroke:#faf6a2;stroke-width:2;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dasharray:none;stroke-dashoffset:0" /> + <path + sodipodi:nodetypes="ccccc" + inkscape:connector-curvature="0" + id="path6598" + d="m 379.81735,119.56755 44.95179,-22.425126 0,56.955676 -44.95179,26.50243 z" + style="fill:#ff6600;fill-opacity:1;stroke:#faf6a2;stroke-width:1.98235416;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dashoffset:0" /> + <path + sodipodi:nodetypes="ccccc" + inkscape:connector-curvature="0" + id="path6615" + d="m 358.25117,97.26295 66.89824,10e-7 -45.10563,22.728439 -66.39143,0 z" + style="fill:#ff6600;fill-opacity:1;stroke:#c48069;stroke-width:1.99740911;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dashoffset:0" /> + <text + sodipodi:linespacing="125%" + id="text6635" + y="160.39748" + x="322.23865" + style="font-size:32px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + xml:space="preserve"><tspan + y="160.39748" + x="322.23865" + id="tspan6637" + sodipodi:role="line">API</tspan></text> + <rect + style="fill:#ff9955;fill-opacity:1;stroke:#faf6a2;stroke-width:2;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dasharray:none;stroke-dashoffset:0" + id="rect6575" + width="66.670067" + height="60.609154" + x="428.80978" + y="119.99139" /> + <path + style="fill:#ff6600;fill-opacity:1;stroke:#faf6a2;stroke-width:1.98960423;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dashoffset:0" + d="m 494.97474,119.62537 44.95179,-22.589454 0,57.373054 -44.95179,26.69664 z" + id="rect6595" + inkscape:connector-curvature="0" + sodipodi:nodetypes="ccccc" /> + <path + style="fill:#ff6600;fill-opacity:1;stroke:#c48069;stroke-width:1.99399996;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dashoffset:0" + d="m 473.25645,97.26295 66.67007,10e-7 -44.95179,22.728439 -66.16499,0 z" + id="rect6612" + inkscape:connector-curvature="0" + sodipodi:nodetypes="ccccc" /> + <text + xml:space="preserve" + style="font-size:32px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + x="439.41635" + y="159.89241" + id="text6639" + sodipodi:linespacing="125%"><tspan + sodipodi:role="line" + id="tspan6641" + x="439.41635" + y="159.89241">API</tspan></text> + <g + style="font-size:24px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + id="text6643" /> + <text + xml:space="preserve" + style="font-size:24px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + x="71.720833" + y="95.747719" + id="text6648" + sodipodi:linespacing="125%"><tspan + sodipodi:role="line" + id="tspan6650" + x="71.720833" + y="95.747719" /></text> + <text + xml:space="preserve" + style="font-size:32px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + x="176.77669" + y="216.96603" + id="text6652" + sodipodi:linespacing="125%"><tspan + sodipodi:role="line" + id="tspan6654" + x="176.77669" + y="216.96603">Network Protocol</tspan></text> + <text + xml:space="preserve" + style="font-size:36px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + x="233.34526" + y="312.93051" + id="text6656" + sodipodi:linespacing="125%"><tspan + sodipodi:role="line" + id="tspan6658" + x="233.34526" + y="312.93051">Service</tspan></text> + <rect + style="fill:#ffffff;fill-opacity:1;stroke:#faf6a2;stroke-width:2.09665918;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dasharray:none;stroke-dashoffset:0" + id="rect6660" + width="66.670067" + height="66.609154" + x="216.67773" + y="371.51938" /> + <rect + y="371.51938" + x="322.74374" + height="64.373825" + width="66.670067" + id="rect6662" + style="fill:#ffffff;fill-opacity:1;stroke:#faf6a2;stroke-width:2.06117821;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dasharray:none;stroke-dashoffset:0" /> + <rect + style="fill:#ffffff;fill-opacity:1;stroke:#faf6a2;stroke-width:2;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dasharray:none;stroke-dashoffset:0" + id="rect6664" + width="66.670067" + height="60.609154" + x="423.75903" + y="372.52951" /> + <path + sodipodi:nodetypes="ccccc" + inkscape:connector-curvature="0" + id="path6666" + d="m 423.61996,372.56359 67.23534,-0.62641 0,19.59587 -68.24549,41.17879 z" + style="fill:#ffcc00;fill-opacity:1;stroke:#faf6a2;stroke-width:0.98368376;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dashoffset:0" /> + <path + style="fill:#ffcc00;fill-opacity:1;stroke:#faf6a2;stroke-width:0.98368376;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dashoffset:0" + d="m 322.60471,371.55344 67.23534,0.38374 0,23.63648 -67.23534,37.13818 z" + id="path6668" + inkscape:connector-curvature="0" + sodipodi:nodetypes="ccccc" /> + <path + sodipodi:nodetypes="ccccc" + inkscape:connector-curvature="0" + id="path6670" + d="m 322.60471,371.55344 67.23534,0.38374 0,23.63648 -67.23534,37.13818 z" + style="fill:#ffcc00;fill-opacity:1;stroke:#faf6a2;stroke-width:0.98368376;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dashoffset:0" /> + <path + style="fill:#ffcc00;fill-opacity:1;stroke:#faf6a2;stroke-width:0.98368376;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dashoffset:0" + d="m 216.53869,371.55344 67.23534,0.38374 0,23.63648 -67.23534,37.13818 z" + id="path6672" + inkscape:connector-curvature="0" + sodipodi:nodetypes="ccccc" /> + <text + xml:space="preserve" + style="font-size:32px;font-style:normal;font-weight:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans" + x="262.63965" + y="666.48389" + id="text6711" + sodipodi:linespacing="125%"><tspan + sodipodi:role="line" + id="tspan6713" + x="262.63965" + y="666.48389" /></text> + <rect + y="371.51938" + x="111.62187" + height="70.798637" + width="66.670067" + id="rect6721" + style="fill:#ffffff;fill-opacity:1;stroke:#faf6a2;stroke-width:2.1615901;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dasharray:none;stroke-dashoffset:0" /> + <path + sodipodi:nodetypes="ccccc" + inkscape:connector-curvature="0" + id="path6723" + d="m 111.48283,370.54329 67.23534,0.38374 0,23.63648 -67.23534,37.13818 z" + style="fill:#ffcc00;fill-opacity:1;stroke:#faf6a2;stroke-width:0.98368376;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-opacity:0;stroke-dashoffset:0" /> + </g> +</svg> diff --git a/doc/documentation/images/service_stack.png b/doc/documentation/images/service_stack.png Binary files differnew file mode 100644 index 0000000000..747d087b20 --- /dev/null +++ b/doc/documentation/images/service_stack.png diff --git a/doc/documentation/images/structure.dot b/doc/documentation/images/structure.dot new file mode 100644 index 0000000000..a53db90b80 --- /dev/null +++ b/doc/documentation/images/structure.dot @@ -0,0 +1,124 @@ +// house = application +// circle (default) = service +// box = daemon +// diamond = library +// black line = dependency +// blue line = extension via plugin +// red line = possibly useful +// dashed = in planning + +// this is what we have...o +digraph dependencies { +splines = true; + + voting [shape=house]; + voting -> consensus; + voting -> identity; + voting -> cadet; + voting -> secretsharing; + secretsharing -> consensus; + + fs [shape=house]; + fs -> dht; + fs -> core; + fs -> datastore; + fs -> cadet; + fs -> ats; + fs -> block [style=dotted,color=blue]; + fs -> identity; + exit [shape=box]; + exit -> cadet; + exit -> tun; + exit -> dnsstub; + vpn -> cadet; + vpn -> regex; + vpn -> tun; + pt [shape=house]; + pt -> cadet; + pt -> vpn; + pt -> dns; + pt -> dnsparser; + dns -> tun; + dns -> dnsstub; + zonemaster [shape=house]; + zonemaster -> namestore; + zonemaster -> dht; + gns -> dns; + gns -> dht; + gns -> block [style=dotted,color=blue]; + gns -> revocation; + gns -> vpn; + gns -> dnsparser; + gns -> dnsstub; + gns -> identity; + revocation -> core; + revocation -> set; + namestore -> identity; + namestore -> gnsrecord; + dnsparser -> gnsrecord [style=dotted,color=blue]; + conversation -> gnsrecord [style=dotted,color=blue]; + gns -> gnsrecord; + dht -> core; + dht -> nse; + dht -> block; + dht -> datacache; + dht -> peerinfo; + dht -> hello; + nse -> core; + regex -> block [style=dotted,color=blue]; + block [shape=diamond]; + datacache [shape=diamond]; + cadet -> core [weight=2]; + cadet -> dht; + cadet -> block [style=dotted,color=blue]; + conversation [shape=house]; + conversation -> cadet; + conversation -> gns; + conversation -> speaker; + conversation -> microphone; + speaker [shape=diamond]; + microphone [shape=diamond]; + regex -> dht; + core -> transport; + topology [shape=box]; + topology -> peerinfo; + topology -> transport; + topology -> core; + topology -> hello; + hostlist [shape=box]; + hostlist -> core; + hostlist -> peerinfo; + hostlist -> hello; + transport -> ats; + transport -> hello; + transport -> peerinfo; + transport -> nat; + transport -> fragmentation; + consensus -> set; + consensus -> cadet; + scalarproduct -> set; + scalarproduct -> cadet; + set -> cadet; + peerinfo -> hello; + fragmentation [shape=diamond]; + hello [shape=diamond]; + nat [shape=diamond]; + tun [shape=diamond]; + dnsparser [shape=diamond]; + dnsstub [shape=diamond]; + + secushare [shape=house]; + multicast; + psyc; + social -> psyc; + social -> gns; + psyc -> psycstore; + psycstore; + social; + secushare -> social; + psyc -> multicast; + multicast -> cadet; + + rps; + rps -> core; +} diff --git a/doc/documentation/index.html b/doc/documentation/index.html new file mode 100644 index 0000000000..0c3b04e9db --- /dev/null +++ b/doc/documentation/index.html @@ -0,0 +1,35 @@ +<title>GNUnet - GNUnet Manuals and Handbooks</title> +<h2>GNUnet - GNUnet Manuals and Handbooks</h2> + +<address>GNUnet e.V.</address> +<address>Fakultät für Informatik -- I8</address> +<address>Technische Universität München</address> +<address>Boltzmannstraße 3</address> +<address>85748 Garching</address> +<address>GERMANY</address> + +<p>The following handbooks and manuals are available:</p> + +<ul> +<li><a href="gnunet/index.html">GNUnet Reference Manual</li> +<li><a href="gnunet-c-tutorial/index.html">GNUnet C Tutorial</li> +</ul> + +<div id="footer"> +<div class="unprintable"> + +<p>Please send general FSF & GNU inquiries to +<a href="mailto:gnu@gnu.org"><gnu@gnu.org></a>. +There are also <a href="/contact/">other ways to contact</a> +the FSF. Broken links and other corrections or suggestions can be sent +to <a href="mailto:gnunet-developers@gnu.org"><gnunet-developers@gnu.org></a>.</p> +</div> + +<p>Copyright © 2001 - 2017 GNUnet e.V.</p> + +<p>This page is licensed under a FIXME License.</p> + +</div> +</div> +</body> +</html> diff --git a/doc/documentation/run-gendocs.sh b/doc/documentation/run-gendocs.sh new file mode 100755 index 0000000000..d025701772 --- /dev/null +++ b/doc/documentation/run-gendocs.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +make version.texi +make version2.texi +./gendocs.sh --email gnunet-developers@gnu.org gnunet-c-tutorial "GNUnet C Tutorial" -o "manual/gnunet-c-tutorial" +#cd manual +#mkdir gnunet-c-tutorial +#mv * gnunet-c-tutorial/ +#cd .. +./gendocs.sh --email gnunet-developers@gnu.org gnunet "GNUnet Reference Manual" -o "manual/gnunet" +#cd manual +#mkdir handbook +#mkdir ../tmp-gnunet +#mv gnunet ../tmp-gnunet +#mv * handbook/ +#mv ../tmp-gnunet gnunet +cp "index.html" manual/ +printf "Success" diff --git a/doc/documentation/testbed_test.c b/doc/documentation/testbed_test.c new file mode 100644 index 0000000000..1696234b05 --- /dev/null +++ b/doc/documentation/testbed_test.c @@ -0,0 +1,99 @@ +#include <unistd.h> +#include <gnunet/platform.h> +#include <gnunet/gnunet_util_lib.h> +#include <gnunet/gnunet_testbed_service.h> +#include <gnunet/gnunet_dht_service.h> + +#define NUM_PEERS 20 + +static struct GNUNET_TESTBED_Operation *dht_op; + +static struct GNUNET_DHT_Handle *dht_handle; + + +struct MyContext +{ + int ht_len; +} ctxt; + + +static int result; + + +static void +shutdown_task (void *cls) +{ + if (NULL != dht_op) + { + GNUNET_TESTBED_operation_done (dht_op); + dht_op = NULL; + dht_handle = NULL; + } + result = GNUNET_OK; +} + + +static void +service_connect_comp (void *cls, + struct GNUNET_TESTBED_Operation *op, + void *ca_result, + const char *emsg) +{ + GNUNET_assert (op == dht_op); + dht_handle = ca_result; + // Do work here... + GNUNET_SCHEDULER_shutdown (); +} + + +static void * +dht_ca (void *cls, const struct GNUNET_CONFIGURATION_Handle *cfg) +{ + struct MyContext *ctxt = cls; + + dht_handle = GNUNET_DHT_connect (cfg, ctxt->ht_len); + return dht_handle; +} + + +static void +dht_da (void *cls, void *op_result) +{ + struct MyContext *ctxt = cls; + + GNUNET_DHT_disconnect ((struct GNUNET_DHT_Handle *) op_result); + dht_handle = NULL; +} + + +static void +test_master (void *cls, + struct GNUNET_TESTBED_RunHandle *h, + unsigned int num_peers, + struct GNUNET_TESTBED_Peer **peers, + unsigned int links_succeeded, + unsigned int links_failed) +{ + ctxt.ht_len = 10; + dht_op = GNUNET_TESTBED_service_connect + (NULL, peers[0], "dht", + &service_connect_comp, NULL, + &dht_ca, &dht_da, &ctxt); + GNUNET_SCHEDULER_add_shutdown (&shutdown_task, NULL); +} + + +int +main (int argc, char **argv) +{ + int ret; + + result = GNUNET_SYSERR; + ret = GNUNET_TESTBED_test_run + ("awesome-test", "template.conf", + NUM_PEERS, 0LL, + NULL, NULL, &test_master, NULL); + if ( (GNUNET_OK != ret) || (GNUNET_OK != result) ) + return 1; + return 0; +} diff --git a/doc/documentation/tutorial-examples/001.c b/doc/documentation/tutorial-examples/001.c new file mode 100644 index 0000000000..7f6699dd22 --- /dev/null +++ b/doc/documentation/tutorial-examples/001.c @@ -0,0 +1,29 @@ +#include <gnunet/platform.h> +#include <gnunet/gnunet_util_lib.h> + +static int ret; + +static void +run (void *cls, + char *const *args, + const char *cfgfile, + const struct GNUNET_CONFIGURATION_Handle *cfg) +{ + // main code here + ret = 0; +} + +int +main (int argc, char *const *argv) +{ + struct GNUNET_GETOPT_CommandLineOption options[] = { + GNUNET_GETOPT_OPTION_END + }; + return (GNUNET_OK == + GNUNET_PROGRAM_run (argc, + argv, + "binary-name", + gettext_noop ("binary description text"), + options, &run, NULL)) ? ret : 1; +} + diff --git a/doc/documentation/tutorial-examples/002.c b/doc/documentation/tutorial-examples/002.c new file mode 100644 index 0000000000..02233fd619 --- /dev/null +++ b/doc/documentation/tutorial-examples/002.c @@ -0,0 +1,17 @@ +static char *string_option; +static int a_flag; + +// ... + struct GNUNET_GETOPT_CommandLineOption options[] = { + GNUNET_GETOPT_option_string ('s', "name", "SOMESTRING", + gettext_noop ("text describing the string_option NAME"), + &string_option}, + GNUNET_GETOPT_option_flag ('f', "flag", + gettext_noop ("text describing the flag option"), + &a_flag), + GNUNET_GETOPT_OPTION_END + }; + string_option = NULL; + a_flag = GNUNET_SYSERR; +// ... + diff --git a/doc/documentation/tutorial-examples/003.c b/doc/documentation/tutorial-examples/003.c new file mode 100644 index 0000000000..d158d7e756 --- /dev/null +++ b/doc/documentation/tutorial-examples/003.c @@ -0,0 +1,11 @@ +struct GNUNET_MQ_MessageHandlers handlers[] = { + // ... + GNUNET_MQ_handler_end () +}; +struct GNUNET_MQ_Handle *mq; + +mq = GNUNET_CLIENT_connect (cfg, + "service-name", + handlers, + &error_cb, + NULL); diff --git a/doc/documentation/tutorial-examples/004.c b/doc/documentation/tutorial-examples/004.c new file mode 100644 index 0000000000..0ef0079070 --- /dev/null +++ b/doc/documentation/tutorial-examples/004.c @@ -0,0 +1,5 @@ +struct GNUNET_MessageHeader +{ + uint16_t size GNUNET_PACKED; + uint16_t type GNUNET_PACKED; +}; diff --git a/doc/documentation/tutorial-examples/005.c b/doc/documentation/tutorial-examples/005.c new file mode 100644 index 0000000000..0c459f509d --- /dev/null +++ b/doc/documentation/tutorial-examples/005.c @@ -0,0 +1,8 @@ +struct GNUNET_MQ_Envelope *env; +struct GNUNET_MessageHeader *msg; + +env = GNUNET_MQ_msg_extra (msg, payload_size, GNUNET_MY_MESSAGE_TYPE); +memcpy (&msg[1], &payload, payload_size); +// Send message via message queue 'mq' +GNUNET_mq_send (mq, env); + diff --git a/doc/documentation/tutorial-examples/006.c b/doc/documentation/tutorial-examples/006.c new file mode 100644 index 0000000000..944d2b18c7 --- /dev/null +++ b/doc/documentation/tutorial-examples/006.c @@ -0,0 +1,31 @@ +static void +handle_fix (void *cls, const struct MyMessage *msg) +{ + // process 'msg' +} + +static int +check_var (void *cls, const struct MyVarMessage *msg) +{ + // check 'msg' is well-formed + return GNUNET_OK; +} + +static void +handle_var (void *cls, const struct MyVarMessage *msg) +{ + // process 'msg' +} + +struct GNUNET_MQ_MessageHandler handlers[] = { + GNUNET_MQ_hd_fixed_size (fix, + GNUNET_MESSAGE_TYPE_MY_FIX, + struct MyMessage, + NULL), + GNUNET_MQ_hd_fixed_size (var, + GNUNET_MESSAGE_TYPE_MY_VAR, + struct MyVarMessage, + NULL), + + GNUNET_MQ_handler_end () +}; diff --git a/doc/documentation/tutorial-examples/007.c b/doc/documentation/tutorial-examples/007.c new file mode 100644 index 0000000000..096539e432 --- /dev/null +++ b/doc/documentation/tutorial-examples/007.c @@ -0,0 +1,10 @@ +GNUNET_SERVICE_MAIN +("service-name", + GNUNET_SERVICE_OPTION_NONE, + &run, + &client_connect_cb, + &client_disconnect_cb, + NULL, + GNUNET_MQ_hd_fixed_size (...), + GNUNET_MQ_hd_var_size (...), + GNUNET_MQ_handler_end ()); diff --git a/doc/documentation/tutorial-examples/008.c b/doc/documentation/tutorial-examples/008.c new file mode 100644 index 0000000000..2dffe2cf91 --- /dev/null +++ b/doc/documentation/tutorial-examples/008.c @@ -0,0 +1,22 @@ +static void +run (void *cls, + const struct GNUNET_CONFIGURATION_Handle *c, + struct GNUNET_SERVICE_Handle *service) +{ +} + +static void * +client_connect_cb (void *cls, + struct GNUNET_SERVICE_Client *c, + struct GNUNET_MQ_Handle *mq) +{ + return c; +} + +static void +client_disconnect_cb (void *cls, + struct GNUNET_SERVICE_Client *c, + void *internal_cls) +{ + GNUNET_assert (c == internal_cls); +} diff --git a/doc/documentation/tutorial-examples/009.c b/doc/documentation/tutorial-examples/009.c new file mode 100644 index 0000000000..26d918fb02 --- /dev/null +++ b/doc/documentation/tutorial-examples/009.c @@ -0,0 +1,9 @@ +#include <gnunet/gnunet_core_service.h> + +struct GNUNET_CORE_Handle * +GNUNET_CORE_connect (const struct GNUNET_CONFIGURATION_Handle *cfg, + void *cls, + GNUNET_CORE_StartupCallback init, + GNUNET_CORE_ConnectEventHandler connects, + GNUNET_CORE_DisconnectEventHandler disconnects, + const struct GNUNET_MQ_MessageHandler *handlers); diff --git a/doc/documentation/tutorial-examples/010.c b/doc/documentation/tutorial-examples/010.c new file mode 100644 index 0000000000..33494490db --- /dev/null +++ b/doc/documentation/tutorial-examples/010.c @@ -0,0 +1,8 @@ +void * +connects (void *cls, + const struct GNUNET_PeerIdentity *peer, + struct GNUNET_MQ_Handle *mq) +{ + return mq; +} + diff --git a/doc/documentation/tutorial-examples/011.c b/doc/documentation/tutorial-examples/011.c new file mode 100644 index 0000000000..23bc051de1 --- /dev/null +++ b/doc/documentation/tutorial-examples/011.c @@ -0,0 +1,8 @@ +void +disconnects (void *cls, + const struct GNUNET_PeerIdentity * peer) +{ + /* Remove peer's identity from known peers */ + /* Make sure no messages are sent to peer from now on */ +} + diff --git a/doc/documentation/tutorial-examples/012.c b/doc/documentation/tutorial-examples/012.c new file mode 100644 index 0000000000..cb21d78ab2 --- /dev/null +++ b/doc/documentation/tutorial-examples/012.c @@ -0,0 +1,4 @@ +#include "gnunet_peerstore_service.h" + +peerstore_handle = GNUNET_PEERSTORE_connect (cfg); + diff --git a/doc/documentation/tutorial-examples/013.1.c b/doc/documentation/tutorial-examples/013.1.c new file mode 100644 index 0000000000..fa52128685 --- /dev/null +++ b/doc/documentation/tutorial-examples/013.1.c @@ -0,0 +1,3 @@ +void +GNUNET_PEERSTORE_store_cancel (struct GNUNET_PEERSTORE_StoreContext + *sc); diff --git a/doc/documentation/tutorial-examples/013.c b/doc/documentation/tutorial-examples/013.c new file mode 100644 index 0000000000..6792417e19 --- /dev/null +++ b/doc/documentation/tutorial-examples/013.c @@ -0,0 +1,12 @@ +struct GNUNET_PEERSTORE_StoreContext * +GNUNET_PEERSTORE_store (struct GNUNET_PEERSTORE_Handle *h, + const char *sub_system, + const struct GNUNET_PeerIdentity *peer, + const char *key, + const void *value, + size_t size, + struct GNUNET_TIME_Absolute expiry, + enum GNUNET_PEERSTORE_StoreOption options, + GNUNET_PEERSTORE_Continuation cont, + void *cont_cls); + diff --git a/doc/documentation/tutorial-examples/014.c b/doc/documentation/tutorial-examples/014.c new file mode 100644 index 0000000000..ce204f7956 --- /dev/null +++ b/doc/documentation/tutorial-examples/014.c @@ -0,0 +1,9 @@ +struct GNUNET_PEERSTORE_IterateContext * +GNUNET_PEERSTORE_iterate (struct GNUNET_PEERSTORE_Handle *h, + const char *sub_system, + const struct GNUNET_PeerIdentity *peer, + const char *key, + struct GNUNET_TIME_Relative timeout, + GNUNET_PEERSTORE_Processor callback, + void *callback_cls); + diff --git a/doc/documentation/tutorial-examples/015.c b/doc/documentation/tutorial-examples/015.c new file mode 100644 index 0000000000..0dd267e8e3 --- /dev/null +++ b/doc/documentation/tutorial-examples/015.c @@ -0,0 +1,8 @@ +struct GNUNET_PEERSTORE_WatchContext * +GNUNET_PEERSTORE_watch (struct GNUNET_PEERSTORE_Handle *h, + const char *sub_system, + const struct GNUNET_PeerIdentity *peer, + const char *key, + GNUNET_PEERSTORE_Processor callback, + void *callback_cls); + diff --git a/doc/documentation/tutorial-examples/016.c b/doc/documentation/tutorial-examples/016.c new file mode 100644 index 0000000000..d169da16d0 --- /dev/null +++ b/doc/documentation/tutorial-examples/016.c @@ -0,0 +1,4 @@ +void +GNUNET_PEERSTORE_watch_cancel (struct GNUNET_PEERSTORE_WatchContext + *wc); + diff --git a/doc/documentation/tutorial-examples/017.c b/doc/documentation/tutorial-examples/017.c new file mode 100644 index 0000000000..c86fbcd1ff --- /dev/null +++ b/doc/documentation/tutorial-examples/017.c @@ -0,0 +1,4 @@ +void +GNUNET_PEERSTORE_disconnect (struct GNUNET_PEERSTORE_Handle *h, + int sync_first); + diff --git a/doc/documentation/tutorial-examples/018.c b/doc/documentation/tutorial-examples/018.c new file mode 100644 index 0000000000..3fc22584c8 --- /dev/null +++ b/doc/documentation/tutorial-examples/018.c @@ -0,0 +1,2 @@ +dht_handle = GNUNET_DHT_connect (cfg, parallel_requests); + diff --git a/doc/documentation/tutorial-examples/019.c b/doc/documentation/tutorial-examples/019.c new file mode 100644 index 0000000000..aaf0015169 --- /dev/null +++ b/doc/documentation/tutorial-examples/019.c @@ -0,0 +1,18 @@ +message_sent_cont (void *cls, + const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + // Request has left local node +} + +struct GNUNET_DHT_PutHandle * +GNUNET_DHT_put (struct GNUNET_DHT_Handle *handle, + const struct GNUNET_HashCode *key, + uint32_t desired_replication_level, + enum GNUNET_DHT_RouteOption options, + enum GNUNET_BLOCK_Type type, + size_t size, + const void *data, + struct GNUNET_TIME_Absolute exp, + struct GNUNET_TIME_Relative timeout, + GNUNET_DHT_PutContinuation cont, void *cont_cls) + diff --git a/doc/documentation/tutorial-examples/020.c b/doc/documentation/tutorial-examples/020.c new file mode 100644 index 0000000000..596db30693 --- /dev/null +++ b/doc/documentation/tutorial-examples/020.c @@ -0,0 +1,25 @@ +static void +get_result_iterator (void *cls, struct GNUNET_TIME_Absolute expiration, + const struct GNUNET_HashCode *key, + const struct GNUNET_PeerIdentity *get_path, + unsigned int get_path_length, + const struct GNUNET_PeerIdentity *put_path, + unsigned int put_path_length, + enum GNUNET_BLOCK_Type type, size_t size, + const void *data) +{ + // Optionally: + GNUNET_DHT_get_stop (get_handle); +} + +get_handle = + GNUNET_DHT_get_start (dht_handle, + block_type, + &key, + replication, + GNUNET_DHT_RO_NONE, + NULL, + 0, + &get_result_iterator, + cls) + diff --git a/doc/documentation/tutorial-examples/021.c b/doc/documentation/tutorial-examples/021.c new file mode 100644 index 0000000000..688a31fe0e --- /dev/null +++ b/doc/documentation/tutorial-examples/021.c @@ -0,0 +1,13 @@ +static enum GNUNET_BLOCK_EvaluationResult +block_plugin_SERVICE_evaluate (void *cls, + enum GNUNET_BLOCK_Type type, + struct GNUNET_BlockGroup *bg, + const GNUNET_HashCode *query, + const void *xquery, + size_t xquery_size, + const void *reply_block, + size_t reply_block_size) +{ + // Verify type, block and bg +} + diff --git a/doc/documentation/tutorial-examples/022.c b/doc/documentation/tutorial-examples/022.c new file mode 100644 index 0000000000..a373619bd3 --- /dev/null +++ b/doc/documentation/tutorial-examples/022.c @@ -0,0 +1,8 @@ +static int +block_plugin_SERVICE_get_key (void *cls, enum GNUNET_BLOCK_Type type, + const void *block, size_t block_size, + struct GNUNET_HashCode *key) +{ + // Store the key in the key argument, return GNUNET_OK on success. +} + diff --git a/doc/documentation/tutorial-examples/023.c b/doc/documentation/tutorial-examples/023.c new file mode 100644 index 0000000000..820c38b10a --- /dev/null +++ b/doc/documentation/tutorial-examples/023.c @@ -0,0 +1,17 @@ +void * +libgnunet_plugin_block_SERVICE_init (void *cls) +{ + static enum GNUNET_BLOCK_Type types[] = + { + GNUNET_BLOCK_TYPE_SERVICE_BLOCKYPE, + GNUNET_BLOCK_TYPE_ANY + }; + struct GNUNET_BLOCK_PluginFunctions *api; + + api = GNUNET_new (struct GNUNET_BLOCK_PluginFunctions); + api->evaluate = &block_plugin_SERICE_evaluate; + api->get_key = &block_plugin_SERVICE_get_key; + api->types = types; + return api; +} + diff --git a/doc/documentation/tutorial-examples/024.c b/doc/documentation/tutorial-examples/024.c new file mode 100644 index 0000000000..2e84b5905c --- /dev/null +++ b/doc/documentation/tutorial-examples/024.c @@ -0,0 +1,9 @@ +void * +libgnunet_plugin_block_SERVICE_done (void *cls) +{ + struct GNUNET_TRANSPORT_PluginFunctions *api = cls; + + GNUNET_free (api); + return NULL; +} + diff --git a/doc/documentation/tutorial-examples/025.c b/doc/documentation/tutorial-examples/025.c new file mode 100644 index 0000000000..66d4f80eca --- /dev/null +++ b/doc/documentation/tutorial-examples/025.c @@ -0,0 +1,15 @@ + plugindir = $(libdir)/gnunet + + plugin_LTLIBRARIES = \ + libgnunet_plugin_block_ext.la + libgnunet_plugin_block_ext_la_SOURCES = \ + plugin_block_ext.c + libgnunet_plugin_block_ext_la_LIBADD = \ + $(prefix)/lib/libgnunethello.la \ + $(prefix)/lib/libgnunetblock.la \ + $(prefix)/lib/libgnunetutil.la + libgnunet_plugin_block_ext_la_LDFLAGS = \ + $(GN_PLUGIN_LDFLAGS) + libgnunet_plugin_block_ext_la_DEPENDENCIES = \ + $(prefix)/lib/libgnunetblock.la + diff --git a/doc/documentation/tutorial-examples/026.c b/doc/documentation/tutorial-examples/026.c new file mode 100644 index 0000000000..264e0b6b97 --- /dev/null +++ b/doc/documentation/tutorial-examples/026.c @@ -0,0 +1,52 @@ +static void +get_callback (void *cls, + enum GNUNET_DHT_RouteOption options, + enum GNUNET_BLOCK_Type type, + uint32_t hop_count, + uint32_t desired_replication_level, + unsigned int path_length, + const struct GNUNET_PeerIdentity *path, + const struct GNUNET_HashCode * key) +{ +} + + +static void +get_resp_callback (void *cls, + enum GNUNET_BLOCK_Type type, + const struct GNUNET_PeerIdentity *get_path, + unsigned int get_path_length, + const struct GNUNET_PeerIdentity *put_path, + unsigned int put_path_length, + struct GNUNET_TIME_Absolute exp, + const struct GNUNET_HashCode * key, + const void *data, + size_t size) +{ +} + + +static void +put_callback (void *cls, + enum GNUNET_DHT_RouteOption options, + enum GNUNET_BLOCK_Type type, + uint32_t hop_count, + uint32_t desired_replication_level, + unsigned int path_length, + const struct GNUNET_PeerIdentity *path, + struct GNUNET_TIME_Absolute exp, + const struct GNUNET_HashCode * key, + const void *data, + size_t size) +{ +} + + +monitor_handle = GNUNET_DHT_monitor_start (dht_handle, + block_type, + key, + &get_callback, + &get_resp_callback, + &put_callback, + cls); + |
