diff options
Diffstat (limited to 'src')
122 files changed, 30234 insertions, 0 deletions
diff --git a/src/boost/assert.hpp b/src/boost/assert.hpp new file mode 100644 index 000000000000..754ebb954bce --- /dev/null +++ b/src/boost/assert.hpp @@ -0,0 +1,38 @@ +// +// boost/assert.hpp - BOOST_ASSERT(expr) +// +// Copyright (c) 2001, 2002 Peter Dimov and Multi Media Ltd. +// +// Permission to copy, use, modify, sell and distribute this software +// is granted provided this copyright notice appears in all copies. +// This software is provided "as is" without express or implied +// warranty, and with no claim as to its suitability for any purpose. +// +// Note: There are no include guards. This is intentional. +// +// See http://www.boost.org/libs/utility/assert.html for documentation. +// + +#undef BOOST_ASSERT + +#if defined(BOOST_DISABLE_ASSERTS) + +# define BOOST_ASSERT(expr) ((void)0) + +#elif defined(BOOST_ENABLE_ASSERT_HANDLER) + +#include <boost/current_function.hpp> + +namespace boost +{ + +void assertion_failed(char const * expr, char const * function, char const * file, long line); // user defined + +} // namespace boost + +#define BOOST_ASSERT(expr) ((expr)? ((void)0): ::boost::assertion_failed(#expr, BOOST_CURRENT_FUNCTION, __FILE__, __LINE__)) + +#else +# include <assert.h> +# define BOOST_ASSERT(expr) assert(expr) +#endif diff --git a/src/boost/checked_delete.hpp b/src/boost/checked_delete.hpp new file mode 100644 index 000000000000..d7f52122de73 --- /dev/null +++ b/src/boost/checked_delete.hpp @@ -0,0 +1,71 @@ +#ifndef BOOST_CHECKED_DELETE_HPP_INCLUDED +#define BOOST_CHECKED_DELETE_HPP_INCLUDED + +// MS compatible compilers support #pragma once + +#if defined(_MSC_VER) && (_MSC_VER >= 1020) +# pragma once +#endif + +// +// boost/checked_delete.hpp +// +// Copyright (c) 1999, 2000, 2001, 2002 boost.org +// Copyright (c) 2002, 2003 Peter Dimov +// Copyright (c) 2003 Daniel Frey +// Copyright (c) 2003 Howard Hinnant +// +// Permission to copy, use, modify, sell and distribute this software +// is granted provided this copyright notice appears in all copies. +// This software is provided "as is" without express or implied +// warranty, and with no claim as to its suitability for any purpose. +// +// See http://www.boost.org/libs/utility/checked_delete.html for documentation. +// + +namespace boost +{ + +// verify that types are complete for increased safety + +template<class T> inline void checked_delete(T * x) +{ + // intentionally complex - simplification causes regressions + typedef char type_must_be_complete[ sizeof(T)? 1: -1 ]; + (void) sizeof(type_must_be_complete); + delete x; +} + +template<class T> inline void checked_array_delete(T * x) +{ + typedef char type_must_be_complete[ sizeof(T)? 1: -1 ]; + (void) sizeof(type_must_be_complete); + delete [] x; +} + +template<class T> struct checked_deleter +{ + typedef void result_type; + typedef T * argument_type; + + void operator()(T * x) const + { + // boost:: disables ADL + boost::checked_delete(x); + } +}; + +template<class T> struct checked_array_deleter +{ + typedef void result_type; + typedef T * argument_type; + + void operator()(T * x) const + { + boost::checked_array_delete(x); + } +}; + +} // namespace boost + +#endif // #ifndef BOOST_CHECKED_DELETE_HPP_INCLUDED diff --git a/src/boost/detail/shared_count.hpp b/src/boost/detail/shared_count.hpp new file mode 100644 index 000000000000..23e78413f721 --- /dev/null +++ b/src/boost/detail/shared_count.hpp @@ -0,0 +1,555 @@ +#ifndef BOOST_DETAIL_SHARED_COUNT_HPP_INCLUDED +#define BOOST_DETAIL_SHARED_COUNT_HPP_INCLUDED + +// MS compatible compilers support #pragma once + +#if defined(_MSC_VER) && (_MSC_VER >= 1020) +# pragma once +#endif + +// +// detail/shared_count.hpp +// +// Copyright (c) 2001, 2002, 2003 Peter Dimov and Multi Media Ltd. +// +// Permission to copy, use, modify, sell and distribute this software +// is granted provided this copyright notice appears in all copies. +// This software is provided "as is" without express or implied +// warranty, and with no claim as to its suitability for any purpose. +// + +//#include <boost/config.hpp> + +#if defined(BOOST_SP_USE_STD_ALLOCATOR) && defined(BOOST_SP_USE_QUICK_ALLOCATOR) +# error BOOST_SP_USE_STD_ALLOCATOR and BOOST_SP_USE_QUICK_ALLOCATOR are incompatible. +#endif + +#include <boost/checked_delete.hpp> +#include <boost/throw_exception.hpp> +//#include <boost/detail/lightweight_mutex.hpp> + +#if defined(BOOST_SP_USE_QUICK_ALLOCATOR) +#include <boost/detail/quick_allocator.hpp> +#endif + +#include <memory> // std::auto_ptr, std::allocator +#include <functional> // std::less +#include <exception> // std::exception +#include <new> // std::bad_alloc +#include <typeinfo> // std::type_info in get_deleter +#include <cstddef> // std::size_t + +#ifdef __BORLANDC__ +# pragma warn -8026 // Functions with excep. spec. are not expanded inline +# pragma warn -8027 // Functions containing try are not expanded inline +#endif + +namespace boost +{ + +// Debug hooks + +#if defined(BOOST_SP_ENABLE_DEBUG_HOOKS) + +void sp_scalar_constructor_hook(void * px, std::size_t size, void * pn); +void sp_array_constructor_hook(void * px); +void sp_scalar_destructor_hook(void * px, std::size_t size, void * pn); +void sp_array_destructor_hook(void * px); + +#endif + + +// The standard library that comes with Borland C++ 5.5.1 +// defines std::exception and its members as having C calling +// convention (-pc). When the definition of bad_weak_ptr +// is compiled with -ps, the compiler issues an error. +// Hence, the temporary #pragma option -pc below. The version +// check is deliberately conservative. + +#if defined(__BORLANDC__) && __BORLANDC__ == 0x551 +# pragma option push -pc +#endif + +class bad_weak_ptr: public std::exception +{ +public: + + virtual char const * what() const throw() + { + return "boost::bad_weak_ptr"; + } +}; + +#if defined(__BORLANDC__) && __BORLANDC__ == 0x551 +# pragma option pop +#endif + +namespace detail +{ + +class sp_counted_base +{ +private: + + // typedef detail::lightweight_mutex mutex_type; + +public: + + sp_counted_base(): use_count_(1), weak_count_(1) + { + } + + virtual ~sp_counted_base() // nothrow + { + } + + // dispose() is called when use_count_ drops to zero, to release + // the resources managed by *this. + + virtual void dispose() = 0; // nothrow + + // destruct() is called when weak_count_ drops to zero. + + virtual void destruct() // nothrow + { + delete this; + } + + virtual void * get_deleter(std::type_info const & ti) = 0; + + void add_ref_copy() + { +#if defined(BOOST_HAS_THREADS) + mutex_type::scoped_lock lock(mtx_); +#endif + ++use_count_; + } + + void add_ref_lock() + { +#if defined(BOOST_HAS_THREADS) + mutex_type::scoped_lock lock(mtx_); +#endif + if(use_count_ == 0) boost::throw_exception(boost::bad_weak_ptr()); + ++use_count_; + } + + void release() // nothrow + { + { +#if defined(BOOST_HAS_THREADS) + mutex_type::scoped_lock lock(mtx_); +#endif + long new_use_count = --use_count_; + + if(new_use_count != 0) return; + } + + dispose(); + weak_release(); + } + + void weak_add_ref() // nothrow + { +#if defined(BOOST_HAS_THREADS) + mutex_type::scoped_lock lock(mtx_); +#endif + ++weak_count_; + } + + void weak_release() // nothrow + { + long new_weak_count; + + { +#if defined(BOOST_HAS_THREADS) + mutex_type::scoped_lock lock(mtx_); +#endif + new_weak_count = --weak_count_; + } + + if(new_weak_count == 0) + { + destruct(); + } + } + + long use_count() const // nothrow + { +#if defined(BOOST_HAS_THREADS) + mutex_type::scoped_lock lock(mtx_); +#endif + return use_count_; + } + +private: + + sp_counted_base(sp_counted_base const &); + sp_counted_base & operator= (sp_counted_base const &); + + long use_count_; // #shared + long weak_count_; // #weak + (#shared != 0) + +#if defined(BOOST_HAS_THREADS) || defined(BOOST_LWM_WIN32) + mutable mutex_type mtx_; +#endif +}; + +#if defined(BOOST_SP_ENABLE_DEBUG_HOOKS) + +template<class T> void cbi_call_constructor_hook(sp_counted_base * pn, T * px, checked_deleter<T> const &, int) +{ + boost::sp_scalar_constructor_hook(px, sizeof(T), pn); +} + +template<class T> void cbi_call_constructor_hook(sp_counted_base *, T * px, checked_array_deleter<T> const &, int) +{ + boost::sp_array_constructor_hook(px); +} + +template<class P, class D> void cbi_call_constructor_hook(sp_counted_base *, P const &, D const &, long) +{ +} + +template<class T> void cbi_call_destructor_hook(sp_counted_base * pn, T * px, checked_deleter<T> const &, int) +{ + boost::sp_scalar_destructor_hook(px, sizeof(T), pn); +} + +template<class T> void cbi_call_destructor_hook(sp_counted_base *, T * px, checked_array_deleter<T> const &, int) +{ + boost::sp_array_destructor_hook(px); +} + +template<class P, class D> void cbi_call_destructor_hook(sp_counted_base *, P const &, D const &, long) +{ +} + +#endif + +// +// Borland's Codeguard trips up over the -Vx- option here: +// +#ifdef __CODEGUARD__ +# pragma option push -Vx- +#endif + +template<class P, class D> class sp_counted_base_impl: public sp_counted_base +{ +private: + + P ptr; // copy constructor must not throw + D del; // copy constructor must not throw + + sp_counted_base_impl(sp_counted_base_impl const &); + sp_counted_base_impl & operator= (sp_counted_base_impl const &); + + typedef sp_counted_base_impl<P, D> this_type; + +public: + + // pre: initial_use_count <= initial_weak_count, d(p) must not throw + + sp_counted_base_impl(P p, D d): ptr(p), del(d) + { +#if defined(BOOST_SP_ENABLE_DEBUG_HOOKS) + detail::cbi_call_constructor_hook(this, p, d, 0); +#endif + } + + virtual void dispose() // nothrow + { +#if defined(BOOST_SP_ENABLE_DEBUG_HOOKS) + detail::cbi_call_destructor_hook(this, ptr, del, 0); +#endif + del(ptr); + } + + virtual void * get_deleter(std::type_info const & ti) + { + return ti == typeid(D)? &del: 0; + } + +#if defined(BOOST_SP_USE_STD_ALLOCATOR) + + void * operator new(std::size_t) + { + return std::allocator<this_type>().allocate(1, static_cast<this_type *>(0)); + } + + void operator delete(void * p) + { + std::allocator<this_type>().deallocate(static_cast<this_type *>(p), 1); + } + +#endif + +#if defined(BOOST_SP_USE_QUICK_ALLOCATOR) + + void * operator new(std::size_t) + { + return quick_allocator<this_type>::alloc(); + } + + void operator delete(void * p) + { + quick_allocator<this_type>::dealloc(p); + } + +#endif +}; + +#if defined(BOOST_SP_ENABLE_DEBUG_HOOKS) + +int const shared_count_id = 0x2C35F101; +int const weak_count_id = 0x298C38A4; + +#endif + +class weak_count; + +class shared_count +{ +private: + + sp_counted_base * pi_; + +#if defined(BOOST_SP_ENABLE_DEBUG_HOOKS) + int id_; +#endif + + friend class weak_count; + +public: + + shared_count(): pi_(0) // nothrow +#if defined(BOOST_SP_ENABLE_DEBUG_HOOKS) + , id_(shared_count_id) +#endif + { + } + + template<class P, class D> shared_count(P p, D d): pi_(0) +#if defined(BOOST_SP_ENABLE_DEBUG_HOOKS) + , id_(shared_count_id) +#endif + { +#ifndef BOOST_NO_EXCEPTIONS + + try + { + pi_ = new sp_counted_base_impl<P, D>(p, d); + } + catch(...) + { + d(p); // delete p + throw; + } + +#else + + pi_ = new sp_counted_base_impl<P, D>(p, d); + + if(pi_ == 0) + { + d(p); // delete p + boost::throw_exception(std::bad_alloc()); + } + +#endif + } + +#ifndef BOOST_NO_AUTO_PTR + + // auto_ptr<Y> is special cased to provide the strong guarantee + + template<class Y> + explicit shared_count(std::auto_ptr<Y> & r): pi_(new sp_counted_base_impl< Y *, checked_deleter<Y> >(r.get(), checked_deleter<Y>())) +#if defined(BOOST_SP_ENABLE_DEBUG_HOOKS) + , id_(shared_count_id) +#endif + { + r.release(); + } + +#endif + + ~shared_count() // nothrow + { + if(pi_ != 0) pi_->release(); +#if defined(BOOST_SP_ENABLE_DEBUG_HOOKS) + id_ = 0; +#endif + } + + shared_count(shared_count const & r): pi_(r.pi_) // nothrow +#if defined(BOOST_SP_ENABLE_DEBUG_HOOKS) + , id_(shared_count_id) +#endif + { + if(pi_ != 0) pi_->add_ref_copy(); + } + + explicit shared_count(weak_count const & r); // throws bad_weak_ptr when r.use_count() == 0 + + shared_count & operator= (shared_count const & r) // nothrow + { + sp_counted_base * tmp = r.pi_; + if(tmp != 0) tmp->add_ref_copy(); + if(pi_ != 0) pi_->release(); + pi_ = tmp; + + return *this; + } + + void swap(shared_count & r) // nothrow + { + sp_counted_base * tmp = r.pi_; + r.pi_ = pi_; + pi_ = tmp; + } + + long use_count() const // nothrow + { + return pi_ != 0? pi_->use_count(): 0; + } + + bool unique() const // nothrow + { + return use_count() == 1; + } + + friend inline bool operator==(shared_count const & a, shared_count const & b) + { + return a.pi_ == b.pi_; + } + + friend inline bool operator<(shared_count const & a, shared_count const & b) + { + return std::less<sp_counted_base *>()(a.pi_, b.pi_); + } + + void * get_deleter(std::type_info const & ti) const + { + return pi_? pi_->get_deleter(ti): 0; + } +}; + +#ifdef __CODEGUARD__ +# pragma option pop +#endif + + +class weak_count +{ +private: + + sp_counted_base * pi_; + +#if defined(BOOST_SP_ENABLE_DEBUG_HOOKS) + int id_; +#endif + + friend class shared_count; + +public: + + weak_count(): pi_(0) // nothrow +#if defined(BOOST_SP_ENABLE_DEBUG_HOOKS) + , id_(weak_count_id) +#endif + { + } + + weak_count(shared_count const & r): pi_(r.pi_) // nothrow +#if defined(BOOST_SP_ENABLE_DEBUG_HOOKS) + , id_(shared_count_id) +#endif + { + if(pi_ != 0) pi_->weak_add_ref(); + } + + weak_count(weak_count const & r): pi_(r.pi_) // nothrow +#if defined(BOOST_SP_ENABLE_DEBUG_HOOKS) + , id_(shared_count_id) +#endif + { + if(pi_ != 0) pi_->weak_add_ref(); + } + + ~weak_count() // nothrow + { + if(pi_ != 0) pi_->weak_release(); +#if defined(BOOST_SP_ENABLE_DEBUG_HOOKS) + id_ = 0; +#endif + } + + weak_count & operator= (shared_count const & r) // nothrow + { + sp_counted_base * tmp = r.pi_; + if(tmp != 0) tmp->weak_add_ref(); + if(pi_ != 0) pi_->weak_release(); + pi_ = tmp; + + return *this; + } + + weak_count & operator= (weak_count const & r) // nothrow + { + sp_counted_base * tmp = r.pi_; + if(tmp != 0) tmp->weak_add_ref(); + if(pi_ != 0) pi_->weak_release(); + pi_ = tmp; + + return *this; + } + + void swap(weak_count & r) // nothrow + { + sp_counted_base * tmp = r.pi_; + r.pi_ = pi_; + pi_ = tmp; + } + + long use_count() const // nothrow + { + return pi_ != 0? pi_->use_count(): 0; + } + + friend inline bool operator==(weak_count const & a, weak_count const & b) + { + return a.pi_ == b.pi_; + } + + friend inline bool operator<(weak_count const & a, weak_count const & b) + { + return std::less<sp_counted_base *>()(a.pi_, b.pi_); + } +}; + +inline shared_count::shared_count(weak_count const & r): pi_(r.pi_) +#if defined(BOOST_SP_ENABLE_DEBUG_HOOKS) + , id_(shared_count_id) +#endif +{ + if(pi_ != 0) + { + pi_->add_ref_lock(); + } + else + { + boost::throw_exception(boost::bad_weak_ptr()); + } +} + +} // namespace detail + +} // namespace boost + +#ifdef __BORLANDC__ +# pragma warn .8027 // Functions containing try are not expanded inline +# pragma warn .8026 // Functions with excep. spec. are not expanded inline +#endif + +#endif // #ifndef BOOST_DETAIL_SHARED_COUNT_HPP_INCLUDED diff --git a/src/boost/detail/workaround.hpp b/src/boost/detail/workaround.hpp new file mode 100644 index 000000000000..0d5a2b2bcbe9 --- /dev/null +++ b/src/boost/detail/workaround.hpp @@ -0,0 +1,74 @@ +// Copyright David Abrahams 2002. Permission to copy, use, +// modify, sell and distribute this software is granted provided this +// copyright notice appears in all copies. This software is provided +// "as is" without express or implied warranty, and with no claim as +// to its suitability for any purpose. +#ifndef WORKAROUND_DWA2002126_HPP +# define WORKAROUND_DWA2002126_HPP + +// Compiler/library version workaround macro +// +// Usage: +// +// #if BOOST_WORKAROUND(BOOST_MSVC, <= 1200) +// ... // workaround code here +// #endif +// +// When BOOST_STRICT_CONFIG is defined, expands to 0. Otherwise, the +// first argument must be undefined or expand to a numeric +// value. The above expands to: +// +// (BOOST_MSVC) != 0 && (BOOST_MSVC) <= 1200 +// +// When used for workarounds that apply to the latest known version +// and all earlier versions of a compiler, the following convention +// should be observed: +// +// #if BOOST_WORKAROUND(BOOST_MSVC, BOOST_TESTED_AT(1301)) +// +// The version number in this case corresponds to the last version in +// which the workaround was known to have been required. When +// BOOST_DETECT_OUTDATED_WORKAROUNDS is not the defined, the macro +// BOOST_TESTED_AT(x) expands to "!= 0", which effectively activates +// the workaround for any version of the compiler. When +// BOOST_DETECT_OUTDATED_WORKAROUNDS is defined, a compiler warning or +// error will be issued if the compiler version exceeds the argument +// to BOOST_TESTED_AT(). This can be used to locate workarounds which +// may be obsoleted by newer versions. + +# ifndef BOOST_STRICT_CONFIG + +# define BOOST_WORKAROUND(symbol, test) \ + ((symbol != 0) && (1 % (( (symbol test) ) + 1))) +// ^ ^ ^ ^ +// The extra level of parenthesis nesting above, along with the +// BOOST_OPEN_PAREN indirection below, is required to satisfy the +// broken preprocessor in MWCW 8.3 and earlier. +// +// The basic mechanism works as follows: +// (symbol test) + 1 => if (symbol test) then 2 else 1 +// 1 % ((symbol test) + 1) => if (symbol test) then 1 else 0 +// +// The complication with % is for cooperation with BOOST_TESTED_AT(). +// When "test" is BOOST_TESTED_AT(x) and +// BOOST_DETECT_OUTDATED_WORKAROUNDS is #defined, +// +// symbol test => if (symbol <= x) then 1 else -1 +// (symbol test) + 1 => if (symbol <= x) then 2 else 0 +// 1 % ((symbol test) + 1) => if (symbol <= x) then 1 else divide-by-zero +// + +# ifdef BOOST_DETECT_OUTDATED_WORKAROUNDS +# define BOOST_OPEN_PAREN ( +# define BOOST_TESTED_AT(value) > value) ?(-1): BOOST_OPEN_PAREN 1 +# else +# define BOOST_TESTED_AT(value) != ((value)-(value)) +# endif + +# else + +# define BOOST_WORKAROUND(symbol, test) 0 + +# endif + +#endif // WORKAROUND_DWA2002126_HPP diff --git a/src/boost/enable_shared_from_this.hpp b/src/boost/enable_shared_from_this.hpp new file mode 100644 index 000000000000..1d52e72c2077 --- /dev/null +++ b/src/boost/enable_shared_from_this.hpp @@ -0,0 +1,68 @@ +#ifndef BOOST_ENABLE_SHARED_FROM_THIS_HPP_INCLUDED +#define BOOST_ENABLE_SHARED_FROM_THIS_HPP_INCLUDED + +// +// enable_shared_from_this.hpp +// +// Copyright (c) 2002 Peter Dimov +// +// Permission to copy, use, modify, sell and distribute this software +// is granted provided this copyright notice appears in all copies. +// This software is provided "as is" without express or implied +// warranty, and with no claim as to its suitability for any purpose. +// +// http://www.boost.org/libs/smart_ptr/enable_shared_from_this.html +// + +#include <boost/weak_ptr.hpp> +#include <boost/shared_ptr.hpp> +#include <boost/assert.hpp> +//#include <boost/config.hpp> + +namespace boost +{ + +template<class T> class enable_shared_from_this +{ +protected: + + enable_shared_from_this() + { + } + + enable_shared_from_this(enable_shared_from_this const &) + { + } + + enable_shared_from_this & operator=(enable_shared_from_this const &) + { + return *this; + } + + ~enable_shared_from_this() + { + } + +public: + + shared_ptr<T> shared_from_this() + { + shared_ptr<T> p(_internal_weak_this); + BOOST_ASSERT(p.get() == this); + return p; + } + + shared_ptr<T const> shared_from_this() const + { + shared_ptr<T const> p(_internal_weak_this); + BOOST_ASSERT(p.get() == this); + return p; + } + + typedef T _internal_element_type; // for bcc 5.5.1 + weak_ptr<_internal_element_type> _internal_weak_this; +}; + +} // namespace boost + +#endif // #ifndef BOOST_ENABLE_SHARED_FROM_THIS_HPP_INCLUDED diff --git a/src/boost/format.hpp b/src/boost/format.hpp new file mode 100644 index 000000000000..f965f0f33e9a --- /dev/null +++ b/src/boost/format.hpp @@ -0,0 +1,64 @@ +// -*- C++ -*- +// Boost general library 'format' --------------------------- +// See http://www.boost.org for updates, documentation, and revision history. + +// (C) Samuel Krempp 2001 +// krempp@crans.ens-cachan.fr +// Permission to copy, use, modify, sell and +// distribute this software is granted provided this copyright notice appears +// in all copies. This software is provided "as is" without express or implied +// warranty, and with no claim as to its suitability for any purpose. + +// ideas taken from Rüdiger Loos's format class +// and Karl Nelson's ofstream + +// ---------------------------------------------------------------------------- +// format.hpp : primary header +// ---------------------------------------------------------------------------- + +#ifndef BOOST_FORMAT_HPP +#define BOOST_FORMAT_HPP + +#include <vector> +#include <string> +#include <sstream> +#include <cassert> + +#if HAVE_LOCALE +#include <locale> +#else +#define BOOST_NO_STD_LOCALE +#define BOOST_NO_LOCALE_ISIDIGIT +#include <cctype> +#endif + +#include <boost/format/macros_default.hpp> + + +// **** Forward declarations ---------------------------------- +#include <boost/format/format_fwd.hpp> // basic_format<Ch,Tr>, and other frontends +#include <boost/format/internals_fwd.hpp> // misc forward declarations for internal use + + +// **** Auxiliary structs (stream_format_state<Ch,Tr> , and format_item<Ch,Tr> ) +#include <boost/format/internals.hpp> + +// **** Format class interface -------------------------------- +#include <boost/format/format_class.hpp> + +// **** Exceptions ----------------------------------------------- +#include <boost/format/exceptions.hpp> + +// **** Implementation ------------------------------------------- +//#include <boost/format/format_implementation.hpp> // member functions + +#include <boost/format/group.hpp> // class for grouping arguments + +#include <boost/format/feed_args.hpp> // argument-feeding functions +//#include <boost/format/parsing.hpp> // format-string parsing (member-)functions + +// **** Implementation of the free functions ---------------------- +//#include <boost/format/free_funcs.hpp> + + +#endif // BOOST_FORMAT_HPP diff --git a/src/boost/format/exceptions.hpp b/src/boost/format/exceptions.hpp new file mode 100644 index 000000000000..79e452449ef8 --- /dev/null +++ b/src/boost/format/exceptions.hpp @@ -0,0 +1,96 @@ +// -*- C++ -*- +// Boost general library 'format' --------------------------- +// See http://www.boost.org for updates, documentation, and revision history. + +// (C) Samuel Krempp 2001 +// krempp@crans.ens-cachan.fr +// Permission to copy, use, modify, sell and +// distribute this software is granted provided this copyright notice appears +// in all copies. This software is provided "as is" without express or implied +// warranty, and with no claim as to its suitability for any purpose. + +// ideas taken from Rüdiger Loos's format class +// and Karl Nelson's ofstream (also took its parsing code as basis for printf parsing) + +// ------------------------------------------------------------------------------ +// exceptions.hpp +// ------------------------------------------------------------------------------ + + +#ifndef BOOST_FORMAT_EXCEPTIONS_HPP +#define BOOST_FORMAT_EXCEPTIONS_HPP + + +#include <stdexcept> + + +namespace boost { + +namespace io { + +// **** exceptions ----------------------------------------------- + +class format_error : public std::exception +{ +public: + format_error() {} + virtual const char *what() const throw() + { + return "boost::format_error: " + "format generic failure"; + } +}; + +class bad_format_string : public format_error +{ +public: + bad_format_string() {} + virtual const char *what() const throw() + { + return "boost::bad_format_string: " + "format-string is ill-formed"; + } +}; + +class too_few_args : public format_error +{ +public: + too_few_args() {} + virtual const char *what() const throw() + { + return "boost::too_few_args: " + "format-string refered to more arguments than were passed"; + } +}; + +class too_many_args : public format_error +{ +public: + too_many_args() {} + virtual const char *what() const throw() + { + return "boost::too_many_args: " + "format-string refered to less arguments than were passed"; + } +}; + + +class out_of_range : public format_error +{ +public: + out_of_range() {} + virtual const char *what() const throw() + { + return "boost::out_of_range: " + "tried to refer to an argument (or item) number which is out of range, " + "according to the format string."; + } +}; + + +} // namespace io + +} // namespace boost + + +#endif // BOOST_FORMAT_EXCEPTIONS_HPP diff --git a/src/boost/format/feed_args.hpp b/src/boost/format/feed_args.hpp new file mode 100644 index 000000000000..3d0b47b4a12e --- /dev/null +++ b/src/boost/format/feed_args.hpp @@ -0,0 +1,247 @@ +// -*- C++ -*- +// Boost general library 'format' --------------------------- +// See http://www.boost.org for updates, documentation, and revision history. + +// (C) Samuel Krempp 2001 +// krempp@crans.ens-cachan.fr +// Permission to copy, use, modify, sell and +// distribute this software is granted provided this copyright notice appears +// in all copies. This software is provided "as is" without express or implied +// warranty, and with no claim as to its suitability for any purpose. + +// ideas taken from Rüdiger Loos's format class +// and Karl Nelson's ofstream + +// ---------------------------------------------------------------------------- +// feed_args.hpp : functions for processing each argument +// (feed, feed_manip, and distribute) +// ---------------------------------------------------------------------------- + + +#ifndef BOOST_FORMAT_FEED_ARGS_HPP +#define BOOST_FORMAT_FEED_ARGS_HPP + +#include "boost/format/format_class.hpp" +#include "boost/format/group.hpp" + +#include "boost/throw_exception.hpp" + +namespace boost { +namespace io { +namespace detail { +namespace { + + inline + void empty_buf(BOOST_IO_STD ostringstream & os) { + static const std::string emptyStr; + os.str(emptyStr); + } + + void do_pad( std::string & s, + std::streamsize w, + const char c, + std::ios::fmtflags f, + bool center) + // applies centered / left / right padding to the string s. + // Effects : string s is padded. + { + std::streamsize n=w-s.size(); + if(n<=0) { + return; + } + if(center) + { + s.reserve(w); // allocate once for the 2 inserts + const std::streamsize n1 = n /2, n0 = n - n1; + s.insert(s.begin(), n0, c); + s.append(n1, c); + } + else + { + if(f & std::ios::left) { + s.append(n, c); + } + else { + s.insert(s.begin(), n, c); + } + } + } // -do_pad(..) + + + template<class T> inline + void put_head(BOOST_IO_STD ostream& , const T& ) { + } + + template<class T> inline + void put_head( BOOST_IO_STD ostream& os, const group1<T>& x ) { + os << group_head(x.a1_); // send the first N-1 items, not the last + } + + template<class T> inline + void put_last( BOOST_IO_STD ostream& os, const T& x ) { + os << x ; + } + + template<class T> inline + void put_last( BOOST_IO_STD ostream& os, const group1<T>& x ) { + os << group_last(x.a1_); // this selects the last element + } + +#ifndef BOOST_NO_OVERLOAD_FOR_NON_CONST + template<class T> inline + void put_head( BOOST_IO_STD ostream& , T& ) { + } + + template<class T> inline + void put_last( BOOST_IO_STD ostream& os, T& x ) { + os << x ; + } +#endif + + + + +template<class T> +void put( T x, + const format_item& specs, + std::string & res, + BOOST_IO_STD ostringstream& oss_ ) +{ + // does the actual conversion of x, with given params, into a string + // using the *supplied* strinstream. (the stream state is important) + + typedef std::string string_t; + typedef format_item format_item_t; + + stream_format_state prev_state(oss_); + + specs.state_.apply_on(oss_); + + // in case x is a group, apply the manip part of it, + // in order to find width + put_head( oss_, x ); + empty_buf( oss_); + + const std::streamsize w=oss_.width(); + const std::ios::fmtflags fl=oss_.flags(); + const bool internal = (fl & std::ios::internal) != 0; + const bool two_stepped_padding = internal + && ! ( specs.pad_scheme_ & format_item_t::spacepad ) + && specs.truncate_ < 0 ; + + + if(! two_stepped_padding) + { + if(w>0) // handle simple padding via do_pad, not natively in stream + oss_.width(0); + put_last( oss_, x); + res = oss_.str(); + + if (specs.truncate_ >= 0) + res.erase(specs.truncate_); + + // complex pads : + if(specs.pad_scheme_ & format_item_t::spacepad) + { + if( res.size()==0 || ( res[0]!='+' && res[0]!='-' )) + { + res.insert(res.begin(), 1, ' '); // insert 1 space at pos 0 + } + } + if(w > 0) // need do_pad + { + do_pad(res,w,oss_.fill(), fl, (specs.pad_scheme_ & format_item_t::centered) !=0 ); + } + } + else // 2-stepped padding + { + put_last( oss_, x); // oss_.width() may result in padding. + res = oss_.str(); + + if (specs.truncate_ >= 0) + res.erase(specs.truncate_); + + if( res.size() - w > 0) + { // length w exceeded + // either it was multi-output with first output padding up all width.. + // either it was one big arg and we are fine. + empty_buf( oss_); + oss_.width(0); + put_last(oss_, x ); + string_t tmp = oss_.str(); // minimal-length output + std::streamsize d; + if( (d=w - tmp.size()) <=0 ) + { + // minimal length is already >= w, so no padding (cool!) + res.swap(tmp); + } + else + { // hum.. we need to pad (it was necessarily multi-output) + typedef typename string_t::size_type size_type; + size_type i = 0; + while( i<tmp.size() && tmp[i] == res[i] ) // find where we should pad. + ++i; + tmp.insert(i, static_cast<size_type>( d ), oss_.fill()); + res.swap( tmp ); + } + } + else + { // okay, only one thing was printed and padded, so res is fine. + } + } + + prev_state.apply_on(oss_); + empty_buf( oss_); + oss_.clear(); +} // end- put(..) + + +} // local namespace + + + + + +template<class T> +void distribute(basic_format& self, T x) + // call put(x, ..) on every occurence of the current argument : +{ + if(self.cur_arg_ >= self.num_args_) + { + if( self.exceptions() & too_many_args_bit ) + boost::throw_exception(too_many_args()); // too many variables have been supplied ! + else return; + } + for(unsigned long i=0; i < self.items_.size(); ++i) + { + if(self.items_[i].argN_ == self.cur_arg_) + { + put<T> (x, self.items_[i], self.items_[i].res_, self.oss_ ); + } + } +} + +template<class T> +basic_format& feed(basic_format& self, T x) +{ + if(self.dumped_) self.clear(); + distribute<T> (self, x); + ++self.cur_arg_; + if(self.bound_.size() != 0) + { + while( self.cur_arg_ < self.num_args_ && self.bound_[self.cur_arg_] ) + ++self.cur_arg_; + } + + // this arg is finished, reset the stream's format state + self.state0_.apply_on(self.oss_); + return self; +} + + +} // namespace detail +} // namespace io +} // namespace boost + + +#endif // BOOST_FORMAT_FEED_ARGS_HPP diff --git a/src/boost/format/format_class.hpp b/src/boost/format/format_class.hpp new file mode 100644 index 000000000000..6875623acb47 --- /dev/null +++ b/src/boost/format/format_class.hpp @@ -0,0 +1,135 @@ +// -*- C++ -*- +// Boost general library 'format' --------------------------- +// See http://www.boost.org for updates, documentation, and revision history. + +// (C) Samuel Krempp 2001 +// krempp@crans.ens-cachan.fr +// Permission to copy, use, modify, sell and +// distribute this software is granted provided this copyright notice appears +// in all copies. This software is provided "as is" without express or implied +// warranty, and with no claim as to its suitability for any purpose. + +// ideas taken from Rüdiger Loos's format class +// and Karl Nelson's ofstream (also took its parsing code as basis for printf parsing) + +// ------------------------------------------------------------------------------ +// format_class.hpp : class interface +// ------------------------------------------------------------------------------ + + +#ifndef BOOST_FORMAT_CLASS_HPP +#define BOOST_FORMAT_CLASS_HPP + +#include <vector> +#include <string> + +#include <boost/format/format_fwd.hpp> +#include <boost/format/internals_fwd.hpp> + +#include <boost/format/internals.hpp> + +namespace boost { + +class basic_format +{ +public: + typedef std::string string_t; + typedef BOOST_IO_STD ostringstream internal_stream_t; +private: + typedef BOOST_IO_STD ostream stream_t; + typedef io::detail::stream_format_state stream_format_state; + typedef io::detail::format_item format_item_t; + +public: + basic_format(const char* str); + basic_format(const string_t& s); +#ifndef BOOST_NO_STD_LOCALE + basic_format(const char* str, const std::locale & loc); + basic_format(const string_t& s, const std::locale & loc); +#endif // no locale + basic_format(const basic_format& x); + basic_format& operator= (const basic_format& x); + + basic_format& clear(); // empty the string buffers (except bound arguments, see clear_binds() ) + + // pass arguments through those operators : + template<class T> basic_format& operator%(const T& x) + { + return io::detail::feed<const T&>(*this,x); + } + +#ifndef BOOST_NO_OVERLOAD_FOR_NON_CONST + template<class T> basic_format& operator%(T& x) + { + return io::detail::feed<T&>(*this,x); + } +#endif + + + // system for binding arguments : + template<class T> + basic_format& bind_arg(int argN, const T& val) + { + return io::detail::bind_arg_body(*this, argN, val); + } + basic_format& clear_bind(int argN); + basic_format& clear_binds(); + + // modify the params of a directive, by applying a manipulator : + template<class T> + basic_format& modify_item(int itemN, const T& manipulator) + { + return io::detail::modify_item_body(*this, itemN, manipulator) ; + } + + // Choosing which errors will throw exceptions : + unsigned char exceptions() const; + unsigned char exceptions(unsigned char newexcept); + + // final output + string_t str() const; + friend BOOST_IO_STD ostream& + operator<< ( BOOST_IO_STD ostream& , const basic_format& ); + + + template<class T> friend basic_format& + io::detail::feed(basic_format&, T); + + template<class T> friend + void io::detail::distribute(basic_format&, T); + + template<class T> friend + basic_format& io::detail::modify_item_body(basic_format&, int, const T&); + + template<class T> friend + basic_format& io::detail::bind_arg_body(basic_format&, int, const T&); + +// make the members private only if the friend templates are supported +private: + + // flag bits, used for style_ + enum style_values { ordered = 1, // set only if all directives are positional directives + special_needs = 4 }; + + // parse the format string : + void parse(const string_t&); + + int style_; // style of format-string : positional or not, etc + int cur_arg_; // keep track of wich argument will come + int num_args_; // number of expected arguments + mutable bool dumped_; // true only after call to str() or << + std::vector<format_item_t> items_; // vector of directives (aka items) + string_t prefix_; // piece of string to insert before first item + + std::vector<bool> bound_; // stores which arguments were bound + // size = num_args OR zero + internal_stream_t oss_; // the internal stream. + stream_format_state state0_; // reference state for oss_ + unsigned char exceptions_; +}; // class basic_format + + +} // namespace boost + + +#endif // BOOST_FORMAT_CLASS_HPP diff --git a/src/boost/format/format_fwd.hpp b/src/boost/format/format_fwd.hpp new file mode 100644 index 000000000000..97c55f6684c3 --- /dev/null +++ b/src/boost/format/format_fwd.hpp @@ -0,0 +1,49 @@ +// -*- C++ -*- +// Boost general library 'format' --------------------------- +// See http://www.boost.org for updates, documentation, and revision history. + +// (C) Samuel Krempp 2001 +// krempp@crans.ens-cachan.fr +// Permission to copy, use, modify, sell and +// distribute this software is granted provided this copyright notice appears +// in all copies. This software is provided "as is" without express or implied +// warranty, and with no claim as to its suitability for any purpose. + +// ideas taken from Rüdiger Loos's format class +// and Karl Nelson's ofstream (also took its parsing code as basis for printf parsing) + +// ------------------------------------------------------------------------------ +// format_fwd.hpp : forward declarations, for primary header format.hpp +// ------------------------------------------------------------------------------ + +#ifndef BOOST_FORMAT_FWD_HPP +#define BOOST_FORMAT_FWD_HPP + +#include <string> +#include <iosfwd> + +namespace boost { + +class basic_format; + +typedef basic_format format; + +namespace io { +enum format_error_bits { bad_format_string_bit = 1, + too_few_args_bit = 2, too_many_args_bit = 4, + out_of_range_bit = 8, + all_error_bits = 255, no_error_bits=0 }; + +// Convertion: format to string +std::string str(const basic_format& ) ; + +} // namespace io + + +BOOST_IO_STD ostream& +operator<<( BOOST_IO_STD ostream&, const basic_format&); + + +} // namespace boost + +#endif // BOOST_FORMAT_FWD_HPP diff --git a/src/boost/format/format_implementation.cc b/src/boost/format/format_implementation.cc new file mode 100644 index 000000000000..aa191afe1132 --- /dev/null +++ b/src/boost/format/format_implementation.cc @@ -0,0 +1,256 @@ +// -*- C++ -*- +// Boost general library format --------------------------- +// See http://www.boost.org for updates, documentation, and revision history. + +// (C) Samuel Krempp 2001 +// krempp@crans.ens-cachan.fr +// Permission to copy, use, modify, sell and +// distribute this software is granted provided this copyright notice appears +// in all copies. This software is provided "as is" without express or implied +// warranty, and with no claim as to its suitability for any purpose. + +// ideas taken from Rüdiger Loos's format class +// and Karl Nelson's ofstream + +// ---------------------------------------------------------------------------- +// format_implementation.hpp Implementation of the basic_format class +// ---------------------------------------------------------------------------- + + +#ifndef BOOST_FORMAT_IMPLEMENTATION_HPP +#define BOOST_FORMAT_IMPLEMENTATION_HPP + +#include <boost/throw_exception.hpp> +#include <boost/assert.hpp> +#include <boost/format.hpp> + +namespace boost { + +// -------- format:: ------------------------------------------- +basic_format::basic_format(const char* str) + : style_(0), cur_arg_(0), num_args_(0), dumped_(false), + items_(), oss_(), exceptions_(io::all_error_bits) +{ + state0_.set_by_stream(oss_); + string_t emptyStr; + if( !str) str = emptyStr.c_str(); + parse( str ); +} + +#ifndef BOOST_NO_STD_LOCALE +basic_format::basic_format(const char* str, const std::locale & loc) + : style_(0), cur_arg_(0), num_args_(0), dumped_(false), + items_(), oss_(), exceptions_(io::all_error_bits) +{ + oss_.imbue( loc ); + state0_.set_by_stream(oss_); + string_t emptyStr; + if( !str) str = emptyStr.c_str(); + parse( str ); +} + +basic_format::basic_format(const string_t& s, const std::locale & loc) + : style_(0), cur_arg_(0), num_args_(0), dumped_(false), + items_(), oss_(), exceptions_(io::all_error_bits) +{ + oss_.imbue( loc ); + state0_.set_by_stream(oss_); + parse(s); +} +#endif //BOOST_NO_STD_LOCALE + +basic_format::basic_format(const string_t& s) + : style_(0), cur_arg_(0), num_args_(0), dumped_(false), + items_(), oss_(), exceptions_(io::all_error_bits) +{ + state0_.set_by_stream(oss_); + parse(s); +} + +basic_format:: basic_format(const basic_format& x) + : style_(x.style_), cur_arg_(x.cur_arg_), num_args_(x.num_args_), dumped_(false), + items_(x.items_), prefix_(x.prefix_), bound_(x.bound_), + oss_(), // <- we obviously can't copy x.oss_ + state0_(x.state0_), exceptions_(x.exceptions_) +{ + state0_.apply_on(oss_); +} + +basic_format& basic_format::operator= (const basic_format& x) +{ + if(this == &x) + return *this; + state0_ = x.state0_; + state0_.apply_on(oss_); + + // plus all the other (trivial) assignments : + exceptions_ = x.exceptions_; + items_ = x.items_; + prefix_ = x.prefix_; + bound_=x.bound_; + style_=x.style_; + cur_arg_=x.cur_arg_; + num_args_=x.num_args_; + dumped_=x.dumped_; + return *this; +} + + +unsigned char basic_format::exceptions() const +{ + return exceptions_; +} + +unsigned char basic_format::exceptions(unsigned char newexcept) +{ + unsigned char swp = exceptions_; + exceptions_ = newexcept; + return swp; +} + + +basic_format& basic_format ::clear() + // empty the string buffers (except bound arguments, see clear_binds() ) + // and make the format object ready for formatting a new set of arguments +{ + BOOST_ASSERT( bound_.size()==0 || num_args_ == static_cast<int>(bound_.size()) ); + + for(unsigned long i=0; i<items_.size(); ++i){ + items_[i].state_ = items_[i].ref_state_; + // clear converted strings only if the corresponding argument is not bound : + if( bound_.size()==0 || !bound_[ items_[i].argN_ ] ) items_[i].res_.resize(0); + } + cur_arg_=0; dumped_=false; + // maybe first arg is bound: + if(bound_.size() != 0) + { + while(cur_arg_ < num_args_ && bound_[cur_arg_] ) ++cur_arg_; + } + return *this; +} + +basic_format& basic_format ::clear_binds() + // cancel all bindings, and clear() +{ + bound_.resize(0); + clear(); + return *this; +} + +basic_format& basic_format::clear_bind(int argN) + // cancel the binding of ONE argument, and clear() +{ + if(argN<1 || argN > num_args_ || bound_.size()==0 || !bound_[argN-1] ) + { + if( exceptions() & io::out_of_range_bit ) + boost::throw_exception(io::out_of_range()); // arg not in range. + else return *this; + } + bound_[argN-1]=false; + clear(); + return *this; +} + + + +std::string basic_format::str() const +{ + dumped_=true; + if(items_.size()==0) + return prefix_; + if( cur_arg_ < num_args_) + if( exceptions() & io::too_few_args_bit ) + boost::throw_exception(io::too_few_args()); // not enough variables have been supplied ! + + unsigned long sz = prefix_.size(); + unsigned long i; + for(i=0; i < items_.size(); ++i) + sz += items_[i].res_.size() + items_[i].appendix_.size(); + string_t res; + res.reserve(sz); + + res += prefix_; + for(i=0; i < items_.size(); ++i) + { + const format_item_t& item = items_[i]; + res += item.res_; + if( item.argN_ == format_item_t::argN_tabulation) + { + BOOST_ASSERT( item.pad_scheme_ & format_item_t::tabulation); + std::streamsize n = item.state_.width_ - res.size(); + if( n > 0 ) + res.append( n, item.state_.fill_ ); + } + res += item.appendix_; + } + return res; +} + +namespace io { +namespace detail { + +template<class T> +basic_format& bind_arg_body( basic_format& self, + int argN, + const T& val) + // bind one argument to a fixed value + // this is persistent over clear() calls, thus also over str() and << +{ + if(self.dumped_) self.clear(); // needed, because we will modify cur_arg_.. + if(argN<1 || argN > self.num_args_) + { + if( self.exceptions() & io::out_of_range_bit ) + boost::throw_exception(io::out_of_range()); // arg not in range. + else return self; + } + if(self.bound_.size()==0) + self.bound_.assign(self.num_args_,false); + else + BOOST_ASSERT( self.num_args_ == static_cast<signed int>(self.bound_.size()) ); + int o_cur_arg = self.cur_arg_; + self.cur_arg_ = argN-1; // arrays begin at 0 + + self.bound_[self.cur_arg_]=false; // if already set, we unset and re-sets.. + self.operator%(val); // put val at the right place, because cur_arg is set + + + // Now re-position cur_arg before leaving : + self.cur_arg_ = o_cur_arg; + self.bound_[argN-1]=true; + if(self.cur_arg_ == argN-1 ) + // hum, now this arg is bound, so move to next free arg + { + while(self.cur_arg_ < self.num_args_ && self.bound_[self.cur_arg_]) ++self.cur_arg_; + } + // In any case, we either have all args, or are on a non-binded arg : + BOOST_ASSERT( self.cur_arg_ >= self.num_args_ || ! self.bound_[self.cur_arg_]); + return self; +} + +template<class T> +basic_format& modify_item_body( basic_format& self, + int itemN, + const T& manipulator) + // applies a manipulator to the format_item describing a given directive. + // this is a permanent change, clear or clear_binds won't cancel that. +{ + if(itemN<1 || itemN >= static_cast<signed int>(self.items_.size() )) + { + if( self.exceptions() & io::out_of_range_bit ) + boost::throw_exception(io::out_of_range()); // item not in range. + else return self; + } + self.items_[itemN-1].ref_state_.apply_manip( manipulator ); + self.items_[itemN-1].state_ = self.items_[itemN-1].ref_state_; + return self; +} + +} // namespace detail + +} // namespace io + +} // namespace boost + + + +#endif // BOOST_FORMAT_IMPLEMENTATION_HPP diff --git a/src/boost/format/free_funcs.cc b/src/boost/format/free_funcs.cc new file mode 100644 index 000000000000..151db37a0ac9 --- /dev/null +++ b/src/boost/format/free_funcs.cc @@ -0,0 +1,71 @@ +// -*- C++ -*- +// Boost general library 'format' --------------------------- +// See http://www.boost.org for updates, documentation, and revision history. + +// (C) Samuel Krempp 2001 +// krempp@crans.ens-cachan.fr +// Permission to copy, use, modify, sell and +// distribute this software is granted provided this copyright notice appears +// in all copies. This software is provided "as is" without express or implied +// warranty, and with no claim as to its suitability for any purpose. + +// ideas taken from Rüdiger Loos's format class +// and Karl Nelson's ofstream (also took its parsing code as basis for printf parsing) + +// ------------------------------------------------------------------------------ +// free_funcs.hpp : implementation of the free functions declared in namespace format +// ------------------------------------------------------------------------------ + +#ifndef BOOST_FORMAT_FUNCS_HPP +#define BOOST_FORMAT_FUNCS_HPP + +#include "boost/format.hpp" +#include "boost/throw_exception.hpp" + +namespace boost { + +namespace io { + inline + std::string str(const basic_format& f) + // adds up all pieces of strings and converted items, and return the formatted string + { + return f.str(); + } +} // - namespace io + +BOOST_IO_STD ostream& +operator<<( BOOST_IO_STD ostream& os, + const boost::basic_format& f) + // effect: "return os << str(f);" but we can try to do it faster +{ + typedef boost::basic_format format_t; + if(f.items_.size()==0) + os << f.prefix_; + else { + if(f.cur_arg_ < f.num_args_) + if( f.exceptions() & io::too_few_args_bit ) + boost::throw_exception(io::too_few_args()); // not enough variables have been supplied ! + if(f.style_ & format_t::special_needs) + os << f.str(); + else { + // else we dont have to count chars output, so we dump directly to os : + os << f.prefix_; + for(unsigned long i=0; i<f.items_.size(); ++i) + { + const format_t::format_item_t& item = f.items_[i]; + os << item.res_; + os << item.appendix_; + + } + } + } + f.dumped_=true; + return os; +} + + + +} // namespace boost + + +#endif // BOOST_FORMAT_FUNCS_HPP diff --git a/src/boost/format/group.hpp b/src/boost/format/group.hpp new file mode 100644 index 000000000000..ac63f3f0bab0 --- /dev/null +++ b/src/boost/format/group.hpp @@ -0,0 +1,680 @@ + +// -*- C++ -*- +// Boost general library 'format' --------------------------- +// See http://www.boost.org for updates, documentation, and revision history. + +// (C) Samuel Krempp 2001 +// krempp@crans.ens-cachan.fr +// Permission to copy, use, modify, sell and +// distribute this software is granted provided this copyright notice appears +// in all copies. This software is provided "as is" without express or implied +// warranty, and with no claim as to its suitability for any purpose. + +// ideas taken from Rüdiger Loos's format class +// and Karl Nelson's ofstream + +// ---------------------------------------------------------------------------- + +// group.hpp : encapsulates a group of manipulators along with an argument +// +// group_head : cut the last element of a group out. +// (is overloaded below on each type of group) + +// group_last : returns the last element of a group +// (is overloaded below on each type of group) + +// ---------------------------------------------------------------------------- + + +#ifndef BOOST_FORMAT_GROUP_HPP +#define BOOST_FORMAT_GROUP_HPP + + +namespace boost { +namespace io { + + +namespace detail { + + +// empty group, but useful even though. +struct group0 +{ + group0() {} +}; + +template <class Ch, class Tr> +inline +BOOST_IO_STD ostream& +operator << ( BOOST_IO_STD ostream& os, + const group0& ) +{ + return os; +} + +template <class T1> +struct group1 +{ + T1 a1_; + group1(T1 a1) + : a1_(a1) + {} +}; + +template <class Ch, class Tr, class T1> +inline +BOOST_IO_STD ostream& +operator << (BOOST_IO_STD ostream& os, + const group1<T1>& x) +{ + os << x.a1_; + return os; +} + + + + +template <class T1,class T2> +struct group2 +{ + T1 a1_; + T2 a2_; + group2(T1 a1,T2 a2) + : a1_(a1),a2_(a2) + {} +}; + +template <class Ch, class Tr, class T1,class T2> +inline +BOOST_IO_STD ostream& +operator << (BOOST_IO_STD ostream& os, + const group2<T1,T2>& x) +{ + os << x.a1_<< x.a2_; + return os; +} + +template <class T1,class T2,class T3> +struct group3 +{ + T1 a1_; + T2 a2_; + T3 a3_; + group3(T1 a1,T2 a2,T3 a3) + : a1_(a1),a2_(a2),a3_(a3) + {} +}; + +template <class Ch, class Tr, class T1,class T2,class T3> +inline +BOOST_IO_STD ostream& +operator << (BOOST_IO_STD ostream& os, + const group3<T1,T2,T3>& x) +{ + os << x.a1_<< x.a2_<< x.a3_; + return os; +} + +template <class T1,class T2,class T3,class T4> +struct group4 +{ + T1 a1_; + T2 a2_; + T3 a3_; + T4 a4_; + group4(T1 a1,T2 a2,T3 a3,T4 a4) + : a1_(a1),a2_(a2),a3_(a3),a4_(a4) + {} +}; + +template <class Ch, class Tr, class T1,class T2,class T3,class T4> +inline +BOOST_IO_STD ostream& +operator << (BOOST_IO_STD ostream& os, + const group4<T1,T2,T3,T4>& x) +{ + os << x.a1_<< x.a2_<< x.a3_<< x.a4_; + return os; +} + +template <class T1,class T2,class T3,class T4,class T5> +struct group5 +{ + T1 a1_; + T2 a2_; + T3 a3_; + T4 a4_; + T5 a5_; + group5(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5) + : a1_(a1),a2_(a2),a3_(a3),a4_(a4),a5_(a5) + {} +}; + +template <class Ch, class Tr, class T1,class T2,class T3,class T4,class T5> +inline +BOOST_IO_STD ostream& +operator << (BOOST_IO_STD ostream& os, + const group5<T1,T2,T3,T4,T5>& x) +{ + os << x.a1_<< x.a2_<< x.a3_<< x.a4_<< x.a5_; + return os; +} + +template <class T1,class T2,class T3,class T4,class T5,class T6> +struct group6 +{ + T1 a1_; + T2 a2_; + T3 a3_; + T4 a4_; + T5 a5_; + T6 a6_; + group6(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6) + : a1_(a1),a2_(a2),a3_(a3),a4_(a4),a5_(a5),a6_(a6) + {} +}; + +template <class Ch, class Tr, class T1,class T2,class T3,class T4,class T5,class T6> +inline +BOOST_IO_STD ostream& +operator << (BOOST_IO_STD ostream& os, + const group6<T1,T2,T3,T4,T5,T6>& x) +{ + os << x.a1_<< x.a2_<< x.a3_<< x.a4_<< x.a5_<< x.a6_; + return os; +} + +template <class T1,class T2,class T3,class T4,class T5,class T6,class T7> +struct group7 +{ + T1 a1_; + T2 a2_; + T3 a3_; + T4 a4_; + T5 a5_; + T6 a6_; + T7 a7_; + group7(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7) + : a1_(a1),a2_(a2),a3_(a3),a4_(a4),a5_(a5),a6_(a6),a7_(a7) + {} +}; + +template <class Ch, class Tr, class T1,class T2,class T3,class T4,class T5,class T6,class T7> +inline +BOOST_IO_STD ostream& +operator << (BOOST_IO_STD ostream& os, + const group7<T1,T2,T3,T4,T5,T6,T7>& x) +{ + os << x.a1_<< x.a2_<< x.a3_<< x.a4_<< x.a5_<< x.a6_<< x.a7_; + return os; +} + +template <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8> +struct group8 +{ + T1 a1_; + T2 a2_; + T3 a3_; + T4 a4_; + T5 a5_; + T6 a6_; + T7 a7_; + T8 a8_; + group8(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7,T8 a8) + : a1_(a1),a2_(a2),a3_(a3),a4_(a4),a5_(a5),a6_(a6),a7_(a7),a8_(a8) + {} +}; + +template <class Ch, class Tr, class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8> +inline +BOOST_IO_STD ostream& +operator << (BOOST_IO_STD ostream& os, + const group8<T1,T2,T3,T4,T5,T6,T7,T8>& x) +{ + os << x.a1_<< x.a2_<< x.a3_<< x.a4_<< x.a5_<< x.a6_<< x.a7_<< x.a8_; + return os; +} + +template <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8,class T9> +struct group9 +{ + T1 a1_; + T2 a2_; + T3 a3_; + T4 a4_; + T5 a5_; + T6 a6_; + T7 a7_; + T8 a8_; + T9 a9_; + group9(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7,T8 a8,T9 a9) + : a1_(a1),a2_(a2),a3_(a3),a4_(a4),a5_(a5),a6_(a6),a7_(a7),a8_(a8),a9_(a9) + {} +}; + +template <class Ch, class Tr, class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8,class T9> +inline +BOOST_IO_STD ostream& +operator << (BOOST_IO_STD ostream& os, + const group9<T1,T2,T3,T4,T5,T6,T7,T8,T9>& x) +{ + os << x.a1_<< x.a2_<< x.a3_<< x.a4_<< x.a5_<< x.a6_<< x.a7_<< x.a8_<< x.a9_; + return os; +} + +template <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8,class T9,class T10> +struct group10 +{ + T1 a1_; + T2 a2_; + T3 a3_; + T4 a4_; + T5 a5_; + T6 a6_; + T7 a7_; + T8 a8_; + T9 a9_; + T10 a10_; + group10(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7,T8 a8,T9 a9,T10 a10) + : a1_(a1),a2_(a2),a3_(a3),a4_(a4),a5_(a5),a6_(a6),a7_(a7),a8_(a8),a9_(a9),a10_(a10) + {} +}; + +template <class Ch, class Tr, class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8,class T9,class T10> +inline +BOOST_IO_STD ostream& +operator << (BOOST_IO_STD ostream& os, + const group10<T1,T2,T3,T4,T5,T6,T7,T8,T9,T10>& x) +{ + os << x.a1_<< x.a2_<< x.a3_<< x.a4_<< x.a5_<< x.a6_<< x.a7_<< x.a8_<< x.a9_<< x.a10_; + return os; +} + + + + +template <class T1,class T2> +inline +group1<T1> +group_head( group2<T1,T2> const& x) +{ + return group1<T1> (x.a1_); +} + +template <class T1,class T2> +inline +group1<T2> +group_last( group2<T1,T2> const& x) +{ + return group1<T2> (x.a2_); +} + + + +template <class T1,class T2,class T3> +inline +group2<T1,T2> +group_head( group3<T1,T2,T3> const& x) +{ + return group2<T1,T2> (x.a1_,x.a2_); +} + +template <class T1,class T2,class T3> +inline +group1<T3> +group_last( group3<T1,T2,T3> const& x) +{ + return group1<T3> (x.a3_); +} + + + +template <class T1,class T2,class T3,class T4> +inline +group3<T1,T2,T3> +group_head( group4<T1,T2,T3,T4> const& x) +{ + return group3<T1,T2,T3> (x.a1_,x.a2_,x.a3_); +} + +template <class T1,class T2,class T3,class T4> +inline +group1<T4> +group_last( group4<T1,T2,T3,T4> const& x) +{ + return group1<T4> (x.a4_); +} + + + +template <class T1,class T2,class T3,class T4,class T5> +inline +group4<T1,T2,T3,T4> +group_head( group5<T1,T2,T3,T4,T5> const& x) +{ + return group4<T1,T2,T3,T4> (x.a1_,x.a2_,x.a3_,x.a4_); +} + +template <class T1,class T2,class T3,class T4,class T5> +inline +group1<T5> +group_last( group5<T1,T2,T3,T4,T5> const& x) +{ + return group1<T5> (x.a5_); +} + + + +template <class T1,class T2,class T3,class T4,class T5,class T6> +inline +group5<T1,T2,T3,T4,T5> +group_head( group6<T1,T2,T3,T4,T5,T6> const& x) +{ + return group5<T1,T2,T3,T4,T5> (x.a1_,x.a2_,x.a3_,x.a4_,x.a5_); +} + +template <class T1,class T2,class T3,class T4,class T5,class T6> +inline +group1<T6> +group_last( group6<T1,T2,T3,T4,T5,T6> const& x) +{ + return group1<T6> (x.a6_); +} + + + +template <class T1,class T2,class T3,class T4,class T5,class T6,class T7> +inline +group6<T1,T2,T3,T4,T5,T6> +group_head( group7<T1,T2,T3,T4,T5,T6,T7> const& x) +{ + return group6<T1,T2,T3,T4,T5,T6> (x.a1_,x.a2_,x.a3_,x.a4_,x.a5_,x.a6_); +} + +template <class T1,class T2,class T3,class T4,class T5,class T6,class T7> +inline +group1<T7> +group_last( group7<T1,T2,T3,T4,T5,T6,T7> const& x) +{ + return group1<T7> (x.a7_); +} + + + +template <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8> +inline +group7<T1,T2,T3,T4,T5,T6,T7> +group_head( group8<T1,T2,T3,T4,T5,T6,T7,T8> const& x) +{ + return group7<T1,T2,T3,T4,T5,T6,T7> (x.a1_,x.a2_,x.a3_,x.a4_,x.a5_,x.a6_,x.a7_); +} + +template <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8> +inline +group1<T8> +group_last( group8<T1,T2,T3,T4,T5,T6,T7,T8> const& x) +{ + return group1<T8> (x.a8_); +} + + + +template <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8,class T9> +inline +group8<T1,T2,T3,T4,T5,T6,T7,T8> +group_head( group9<T1,T2,T3,T4,T5,T6,T7,T8,T9> const& x) +{ + return group8<T1,T2,T3,T4,T5,T6,T7,T8> (x.a1_,x.a2_,x.a3_,x.a4_,x.a5_,x.a6_,x.a7_,x.a8_); +} + +template <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8,class T9> +inline +group1<T9> +group_last( group9<T1,T2,T3,T4,T5,T6,T7,T8,T9> const& x) +{ + return group1<T9> (x.a9_); +} + + + +template <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8,class T9,class T10> +inline +group9<T1,T2,T3,T4,T5,T6,T7,T8,T9> +group_head( group10<T1,T2,T3,T4,T5,T6,T7,T8,T9,T10> const& x) +{ + return group9<T1,T2,T3,T4,T5,T6,T7,T8,T9> (x.a1_,x.a2_,x.a3_,x.a4_,x.a5_,x.a6_,x.a7_,x.a8_,x.a9_); +} + +template <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8,class T9,class T10> +inline +group1<T10> +group_last( group10<T1,T2,T3,T4,T5,T6,T7,T8,T9,T10> const& x) +{ + return group1<T10> (x.a10_); +} + + + + + +} // namespace detail + + + +// helper functions + + +inline detail::group1< detail::group0 > +group() { return detail::group1< detail::group0 > ( detail::group0() ); } + +template <class T1, class Var> +inline +detail::group1< detail::group2<T1, Var const&> > + group(T1 a1, Var const& var) +{ + return detail::group1< detail::group2<T1, Var const&> > + ( detail::group2<T1, Var const&> + (a1, var) + ); +} + +template <class T1,class T2, class Var> +inline +detail::group1< detail::group3<T1,T2, Var const&> > + group(T1 a1,T2 a2, Var const& var) +{ + return detail::group1< detail::group3<T1,T2, Var const&> > + ( detail::group3<T1,T2, Var const&> + (a1,a2, var) + ); +} + +template <class T1,class T2,class T3, class Var> +inline +detail::group1< detail::group4<T1,T2,T3, Var const&> > + group(T1 a1,T2 a2,T3 a3, Var const& var) +{ + return detail::group1< detail::group4<T1,T2,T3, Var const&> > + ( detail::group4<T1,T2,T3, Var const&> + (a1,a2,a3, var) + ); +} + +template <class T1,class T2,class T3,class T4, class Var> +inline +detail::group1< detail::group5<T1,T2,T3,T4, Var const&> > + group(T1 a1,T2 a2,T3 a3,T4 a4, Var const& var) +{ + return detail::group1< detail::group5<T1,T2,T3,T4, Var const&> > + ( detail::group5<T1,T2,T3,T4, Var const&> + (a1,a2,a3,a4, var) + ); +} + +template <class T1,class T2,class T3,class T4,class T5, class Var> +inline +detail::group1< detail::group6<T1,T2,T3,T4,T5, Var const&> > + group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5, Var const& var) +{ + return detail::group1< detail::group6<T1,T2,T3,T4,T5, Var const&> > + ( detail::group6<T1,T2,T3,T4,T5, Var const&> + (a1,a2,a3,a4,a5, var) + ); +} + +template <class T1,class T2,class T3,class T4,class T5,class T6, class Var> +inline +detail::group1< detail::group7<T1,T2,T3,T4,T5,T6, Var const&> > + group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6, Var const& var) +{ + return detail::group1< detail::group7<T1,T2,T3,T4,T5,T6, Var const&> > + ( detail::group7<T1,T2,T3,T4,T5,T6, Var const&> + (a1,a2,a3,a4,a5,a6, var) + ); +} + +template <class T1,class T2,class T3,class T4,class T5,class T6,class T7, class Var> +inline +detail::group1< detail::group8<T1,T2,T3,T4,T5,T6,T7, Var const&> > + group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7, Var const& var) +{ + return detail::group1< detail::group8<T1,T2,T3,T4,T5,T6,T7, Var const&> > + ( detail::group8<T1,T2,T3,T4,T5,T6,T7, Var const&> + (a1,a2,a3,a4,a5,a6,a7, var) + ); +} + +template <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8, class Var> +inline +detail::group1< detail::group9<T1,T2,T3,T4,T5,T6,T7,T8, Var const&> > + group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7,T8 a8, Var const& var) +{ + return detail::group1< detail::group9<T1,T2,T3,T4,T5,T6,T7,T8, Var const&> > + ( detail::group9<T1,T2,T3,T4,T5,T6,T7,T8, Var const&> + (a1,a2,a3,a4,a5,a6,a7,a8, var) + ); +} + +template <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8,class T9, class Var> +inline +detail::group1< detail::group10<T1,T2,T3,T4,T5,T6,T7,T8,T9, Var const&> > + group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7,T8 a8,T9 a9, Var const& var) +{ + return detail::group1< detail::group10<T1,T2,T3,T4,T5,T6,T7,T8,T9, Var const&> > + ( detail::group10<T1,T2,T3,T4,T5,T6,T7,T8,T9, Var const&> + (a1,a2,a3,a4,a5,a6,a7,a8,a9, var) + ); +} + + +#ifndef BOOST_NO_OVERLOAD_FOR_NON_CONST + +template <class T1, class Var> +inline +detail::group1< detail::group2<T1, Var&> > + group(T1 a1, Var& var) +{ + return detail::group1< detail::group2<T1, Var&> > + ( detail::group2<T1, Var&> + (a1, var) + ); +} + +template <class T1,class T2, class Var> +inline +detail::group1< detail::group3<T1,T2, Var&> > + group(T1 a1,T2 a2, Var& var) +{ + return detail::group1< detail::group3<T1,T2, Var&> > + ( detail::group3<T1,T2, Var&> + (a1,a2, var) + ); +} + +template <class T1,class T2,class T3, class Var> +inline +detail::group1< detail::group4<T1,T2,T3, Var&> > + group(T1 a1,T2 a2,T3 a3, Var& var) +{ + return detail::group1< detail::group4<T1,T2,T3, Var&> > + ( detail::group4<T1,T2,T3, Var&> + (a1,a2,a3, var) + ); +} + +template <class T1,class T2,class T3,class T4, class Var> +inline +detail::group1< detail::group5<T1,T2,T3,T4, Var&> > + group(T1 a1,T2 a2,T3 a3,T4 a4, Var& var) +{ + return detail::group1< detail::group5<T1,T2,T3,T4, Var&> > + ( detail::group5<T1,T2,T3,T4, Var&> + (a1,a2,a3,a4, var) + ); +} + +template <class T1,class T2,class T3,class T4,class T5, class Var> +inline +detail::group1< detail::group6<T1,T2,T3,T4,T5, Var&> > + group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5, Var& var) +{ + return detail::group1< detail::group6<T1,T2,T3,T4,T5, Var&> > + ( detail::group6<T1,T2,T3,T4,T5, Var&> + (a1,a2,a3,a4,a5, var) + ); +} + +template <class T1,class T2,class T3,class T4,class T5,class T6, class Var> +inline +detail::group1< detail::group7<T1,T2,T3,T4,T5,T6, Var&> > + group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6, Var& var) +{ + return detail::group1< detail::group7<T1,T2,T3,T4,T5,T6, Var&> > + ( detail::group7<T1,T2,T3,T4,T5,T6, Var&> + (a1,a2,a3,a4,a5,a6, var) + ); +} + +template <class T1,class T2,class T3,class T4,class T5,class T6,class T7, class Var> +inline +detail::group1< detail::group8<T1,T2,T3,T4,T5,T6,T7, Var&> > + group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7, Var& var) +{ + return detail::group1< detail::group8<T1,T2,T3,T4,T5,T6,T7, Var&> > + ( detail::group8<T1,T2,T3,T4,T5,T6,T7, Var&> + (a1,a2,a3,a4,a5,a6,a7, var) + ); +} + +template <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8, class Var> +inline +detail::group1< detail::group9<T1,T2,T3,T4,T5,T6,T7,T8, Var&> > + group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7,T8 a8, Var& var) +{ + return detail::group1< detail::group9<T1,T2,T3,T4,T5,T6,T7,T8, Var&> > + ( detail::group9<T1,T2,T3,T4,T5,T6,T7,T8, Var&> + (a1,a2,a3,a4,a5,a6,a7,a8, var) + ); +} + +template <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8,class T9, class Var> +inline +detail::group1< detail::group10<T1,T2,T3,T4,T5,T6,T7,T8,T9, Var&> > + group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7,T8 a8,T9 a9, Var& var) +{ + return detail::group1< detail::group10<T1,T2,T3,T4,T5,T6,T7,T8,T9, Var&> > + ( detail::group10<T1,T2,T3,T4,T5,T6,T7,T8,T9, Var&> + (a1,a2,a3,a4,a5,a6,a7,a8,a9, var) + ); +} + + +#endif //end- #ifndef BOOST_NO_OVERLOAD_FOR_NON_CONST + + +} // namespace io + +} // namespace boost + + +#endif // BOOST_FORMAT_GROUP_HPP diff --git a/src/boost/format/internals.hpp b/src/boost/format/internals.hpp new file mode 100644 index 000000000000..d25eb4c864c4 --- /dev/null +++ b/src/boost/format/internals.hpp @@ -0,0 +1,167 @@ +// -*- C++ -*- +// Boost general library 'format' --------------------------- +// See http://www.boost.org for updates, documentation, and revision history. + +// (C) Samuel Krempp 2001 +// krempp@crans.ens-cachan.fr +// Permission to copy, use, modify, sell and +// distribute this software is granted provided this copyright notice appears +// in all copies. This software is provided "as is" without express or implied +// warranty, and with no claim as to its suitability for any purpose. + +// ideas taken from Rüdiger Loos's format class +// and Karl Nelson's ofstream + +// ---------------------------------------------------------------------------- +// internals.hpp : internal structs. included by format.hpp +// stream_format_state, and format_item +// ---------------------------------------------------------------------------- + + +#ifndef BOOST_FORMAT_INTERNALS_HPP +#define BOOST_FORMAT_INTERNALS_HPP + + +#include <string> +#include <sstream> + +namespace boost { +namespace io { +namespace detail { + + +// -------------- +// set of params that define the format state of a stream + +struct stream_format_state +{ + typedef std::ios basic_ios; + + std::streamsize width_; + std::streamsize precision_; + char fill_; + std::ios::fmtflags flags_; + + stream_format_state() : width_(-1), precision_(-1), fill_(0), flags_(std::ios::dec) {} + stream_format_state(basic_ios& os) {set_by_stream(os); } + + void apply_on(basic_ios & os) const; //- applies format_state to the stream + template<class T> void apply_manip(T manipulator) //- modifies state by applying manipulator. + { apply_manip_body<T>( *this, manipulator) ; } + void reset(); //- sets to default state. + void set_by_stream(const basic_ios& os); //- sets to os's state. +}; + + + +// -------------- +// format_item : stores all parameters that can be defined by directives in the format-string + +struct format_item +{ + enum pad_values { zeropad = 1, spacepad =2, centered=4, tabulation = 8 }; + + enum arg_values { argN_no_posit = -1, // non-positional directive. argN will be set later. + argN_tabulation = -2, // tabulation directive. (no argument read) + argN_ignored = -3 // ignored directive. (no argument read) + }; + typedef BOOST_IO_STD ios basic_ios; + typedef detail::stream_format_state stream_format_state; + typedef std::string string_t; + typedef BOOST_IO_STD ostringstream internal_stream_t; + + + int argN_; //- argument number (starts at 0, eg : %1 => argN=0) + // negative values are used for items that don't process + // an argument + string_t res_; //- result of the formatting of this item + string_t appendix_; //- piece of string between this item and the next + + stream_format_state ref_state_;// set by parsing the format_string, is only affected by modify_item + stream_format_state state_; // always same as ref_state, _unless_ modified by manipulators 'group(..)' + + // non-stream format-state parameters + signed int truncate_; //- is >=0 for directives like %.5s (take 5 chars from the string) + unsigned int pad_scheme_; //- several possible padding schemes can mix. see pad_values + + format_item() : argN_(argN_no_posit), truncate_(-1), pad_scheme_(0) {} + + void compute_states(); // sets states according to truncate and pad_scheme. +}; + + + +// ----------------------------------------------------------- +// Definitions +// ----------------------------------------------------------- + +// --- stream_format_state:: ------------------------------------------- +inline +void stream_format_state::apply_on(basic_ios & os) const + // set the state of this stream according to our params +{ + if(width_ != -1) + os.width(width_); + if(precision_ != -1) + os.precision(precision_); + if(fill_ != 0) + os.fill(fill_); + os.flags(flags_); +} + +inline +void stream_format_state::set_by_stream(const basic_ios& os) + // set our params according to the state of this stream +{ + flags_ = os.flags(); + width_ = os.width(); + precision_ = os.precision(); + fill_ = os.fill(); +} + +template<class T> inline +void apply_manip_body( stream_format_state& self, + T manipulator) + // modify our params according to the manipulator +{ + BOOST_IO_STD stringstream ss; + self.apply_on( ss ); + ss << manipulator; + self.set_by_stream( ss ); +} + +inline +void stream_format_state::reset() + // set our params to standard's default state +{ + width_=-1; precision_=-1; fill_=0; + flags_ = std::ios::dec; +} + + +// --- format_items:: ------------------------------------------- +inline +void format_item::compute_states() + // reflect pad_scheme_ on state_ and ref_state_ + // because some pad_schemes has complex consequences on several state params. +{ + if(pad_scheme_ & zeropad) + { + if(ref_state_.flags_ & std::ios::left) + { + pad_scheme_ = pad_scheme_ & (~zeropad); // ignore zeropad in left alignment + } + else + { + ref_state_.fill_='0'; + ref_state_.flags_ |= std::ios::internal; + } + } + state_ = ref_state_; +} + + +} } } // namespaces boost :: io :: detail + + +#endif // BOOST_FORMAT_INTERNALS_HPP diff --git a/src/boost/format/internals_fwd.hpp b/src/boost/format/internals_fwd.hpp new file mode 100644 index 000000000000..a8ebf7c3abc1 --- /dev/null +++ b/src/boost/format/internals_fwd.hpp @@ -0,0 +1,65 @@ +// -*- C++ -*- +// Boost general library 'format' --------------------------- +// See http://www.boost.org for updates, documentation, and revision history. + +// (C) Samuel Krempp 2001 +// krempp@crans.ens-cachan.fr +// Permission to copy, use, modify, sell and +// distribute this software is granted provided this copyright notice appears +// in all copies. This software is provided "as is" without express or implied +// warranty, and with no claim as to its suitability for any purpose. + +// ideas taken from Rüdiger Loos's format class +// and Karl Nelson's ofstream (also took its parsing code as basis for printf parsing) + +// ------------------------------------------------------------------------------ +// internals_fwd.hpp : forward declarations, for internal headers +// ------------------------------------------------------------------------------ + +#ifndef BOOST_FORMAT_INTERNAL_FWD_HPP +#define BOOST_FORMAT_INTERNAL_FWD_HPP + +#include "boost/format/format_fwd.hpp" + + +namespace boost { +namespace io { + +namespace detail { + struct stream_format_state; + struct format_item; +} + + +namespace detail { + + // these functions were intended as methods, + // but MSVC have problems with template member functions : + + // defined in format_implementation.hpp : + template<class T> + basic_format& modify_item_body( basic_format& self, + int itemN, const T& manipulator); + + template<class T> + basic_format& bind_arg_body( basic_format& self, + int argN, const T& val); + + template<class T> + void apply_manip_body( stream_format_state& self, + T manipulator); + + // argument feeding (defined in feed_args.hpp ) : + template<class T> + void distribute(basic_format& self, T x); + + template<class T> + basic_format& feed(basic_format& self, T x); + +} // namespace detail + +} // namespace io +} // namespace boost + + +#endif // BOOST_FORMAT_INTERNAL_FWD_HPP diff --git a/src/boost/format/local.mk b/src/boost/format/local.mk new file mode 100644 index 000000000000..3776eff382fe --- /dev/null +++ b/src/boost/format/local.mk @@ -0,0 +1,7 @@ +libraries += libformat + +libformat_NAME = libnixformat + +libformat_DIR := $(d) + +libformat_SOURCES := $(wildcard $(d)/*.cc) diff --git a/src/boost/format/macros_default.hpp b/src/boost/format/macros_default.hpp new file mode 100644 index 000000000000..4fd84a163fb3 --- /dev/null +++ b/src/boost/format/macros_default.hpp @@ -0,0 +1,48 @@ +// -*- C++ -*- +// Boost general library 'format' --------------------------- +// See http://www.boost.org for updates, documentation, and revision history. + +// (C) Samuel Krempp 2001 +// krempp@crans.ens-cachan.fr +// Permission to copy, use, modify, sell and +// distribute this software is granted provided this copyright notice appears +// in all copies. This software is provided "as is" without express or implied +// warranty, and with no claim as to its suitability for any purpose. + +// ideas taken from Rüdiger Loos's format class +// and Karl Nelson's ofstream (also took its parsing code as basis for printf parsing) + +// ------------------------------------------------------------------------------ +// macros_default.hpp : configuration for the format library +// provides default values for the stl workaround macros +// ------------------------------------------------------------------------------ + +#ifndef BOOST_FORMAT_MACROS_DEFAULT_HPP +#define BOOST_FORMAT_MACROS_DEFAULT_HPP + +// *** This should go to "boost/config/suffix.hpp". + +#ifndef BOOST_IO_STD +# define BOOST_IO_STD std:: +#endif + +// **** Workaround for io streams, stlport and msvc. +#ifdef BOOST_IO_NEEDS_USING_DECLARATION +namespace boost { + using std::char_traits; + using std::basic_ostream; + using std::basic_ostringstream; + namespace io { + using std::basic_ostream; + namespace detail { + using std::basic_ios; + using std::basic_ostream; + using std::basic_ostringstream; + } + } +} +#endif + +// ------------------------------------------------------------------------------ + +#endif // BOOST_FORMAT_MACROS_DEFAULT_HPP diff --git a/src/boost/format/parsing.cc b/src/boost/format/parsing.cc new file mode 100644 index 000000000000..34c36adeb734 --- /dev/null +++ b/src/boost/format/parsing.cc @@ -0,0 +1,454 @@ +// -*- C++ -*- +// Boost general library 'format' --------------------------- +// See http://www.boost.org for updates, documentation, and revision history. + +// (C) Samuel Krempp 2001 +// krempp@crans.ens-cachan.fr +// Permission to copy, use, modify, sell and +// distribute this software is granted provided this copyright notice appears +// in all copies. This software is provided "as is" without express or implied +// warranty, and with no claim as to its suitability for any purpose. + +// ideas taken from Rudiger Loos's format class +// and Karl Nelson's ofstream (also took its parsing code as basis for printf parsing) + +// ------------------------------------------------------------------------------ +// parsing.hpp : implementation of the parsing member functions +// ( parse, parse_printf_directive) +// ------------------------------------------------------------------------------ + + +#ifndef BOOST_FORMAT_PARSING_HPP +#define BOOST_FORMAT_PARSING_HPP + + +#include <boost/format.hpp> +#include <boost/throw_exception.hpp> +#include <boost/assert.hpp> + + +namespace boost { +namespace io { +namespace detail { + + template<class Stream> inline + bool wrap_isdigit(char c, Stream &os) + { +#ifndef BOOST_NO_LOCALE_ISIDIGIT + return std::isdigit(c, os.rdbuf()->getloc() ); +# else + using namespace std; + return isdigit(c); +#endif + } //end- wrap_isdigit(..) + + template<class Res> inline + Res str2int(const std::string& s, + std::string::size_type start, + BOOST_IO_STD ios &os, + const Res = Res(0) ) + // Input : char string, with starting index + // a basic_ios& merely to call its widen/narrow member function in the desired locale. + // Effects : reads s[start:] and converts digits into an integral n, of type Res + // Returns : n + { + Res n = 0; + while(start<s.size() && wrap_isdigit(s[start], os) ) { + char cur_ch = s[start]; + BOOST_ASSERT(cur_ch != 0 ); // since we called isdigit, this should not happen. + n *= 10; + n += cur_ch - '0'; // 22.2.1.1.2 of the C++ standard + ++start; + } + return n; + } + + void skip_asterisk(const std::string & buf, + std::string::size_type * pos_p, + BOOST_IO_STD ios &os) + // skip printf's "asterisk-fields" directives in the format-string buf + // Input : char string, with starting index *pos_p + // a basic_ios& merely to call its widen/narrow member function in the desired locale. + // Effects : advance *pos_p by skipping printf's asterisk fields. + // Returns : nothing + { + using namespace std; + BOOST_ASSERT( pos_p != 0); + if(*pos_p >= buf.size() ) return; + if(buf[ *pos_p]=='*') { + ++ (*pos_p); + while (*pos_p < buf.size() && wrap_isdigit(buf[*pos_p],os)) ++(*pos_p); + if(buf[*pos_p]=='$') ++(*pos_p); + } + } + + + inline void maybe_throw_exception( unsigned char exceptions) + // auxiliary func called by parse_printf_directive + // for centralising error handling + // it either throws if user sets the corresponding flag, or does nothing. + { + if(exceptions & io::bad_format_string_bit) + boost::throw_exception(io::bad_format_string()); + } + + + + bool parse_printf_directive(const std::string & buf, + std::string::size_type * pos_p, + detail::format_item * fpar, + BOOST_IO_STD ios &os, + unsigned char exceptions) + // Input : a 'printf-directive' in the format-string, starting at buf[ *pos_p ] + // a basic_ios& merely to call its widen/narrow member function in the desired locale. + // a bitset'excpetions' telling whether to throw exceptions on errors. + // Returns : true if parse somehow succeeded (possibly ignoring errors if exceptions disabled) + // false if it failed so bad that the directive should be printed verbatim + // Effects : - *pos_p is incremented so that buf[*pos_p] is the first char after the directive + // - *fpar is set with the parameters read in the directive + { + typedef format_item format_item_t; + BOOST_ASSERT( pos_p != 0); + std::string::size_type &i1 = *pos_p, + i0; + fpar->argN_ = format_item_t::argN_no_posit; // if no positional-directive + + bool in_brackets=false; + if(buf[i1]=='|') + { + in_brackets=true; + if( ++i1 >= buf.size() ) { + maybe_throw_exception(exceptions); + return false; + } + } + + // the flag '0' would be picked as a digit for argument order, but here it's a flag : + if(buf[i1]=='0') + goto parse_flags; + + // handle argument order (%2$d) or possibly width specification: %2d + i0 = i1; // save position before digits + while (i1 < buf.size() && wrap_isdigit(buf[i1], os)) + ++i1; + if (i1!=i0) + { + if( i1 >= buf.size() ) { + maybe_throw_exception(exceptions); + return false; + } + int n=str2int(buf,i0, os, int(0) ); + + // %N% case : this is already the end of the directive + if( buf[i1] == '%' ) + { + fpar->argN_ = n-1; + ++i1; + if( in_brackets) + maybe_throw_exception(exceptions); + // but don't return. maybe "%" was used in lieu of '$', so we go on. + else return true; + } + + if ( buf[i1]=='$' ) + { + fpar->argN_ = n-1; + ++i1; + } + else + { + // non-positionnal directive + fpar->ref_state_.width_ = n; + fpar->argN_ = format_item_t::argN_no_posit; + goto parse_precision; + } + } + + parse_flags: + // handle flags + while ( i1 <buf.size()) // as long as char is one of + - = # 0 l h or ' ' + { + // misc switches + switch (buf[i1]) + { + case '\'' : break; // no effect yet. (painful to implement) + case 'l': + case 'h': // short/long modifier : for printf-comaptibility (no action needed) + break; + case '-': + fpar->ref_state_.flags_ |= std::ios::left; + break; + case '=': + fpar->pad_scheme_ |= format_item_t::centered; + break; + case ' ': + fpar->pad_scheme_ |= format_item_t::spacepad; + break; + case '+': + fpar->ref_state_.flags_ |= std::ios::showpos; + break; + case '0': + fpar->pad_scheme_ |= format_item_t::zeropad; + // need to know alignment before really setting flags, + // so just add 'zeropad' flag for now, it will be processed later. + break; + case '#': + fpar->ref_state_.flags_ |= std::ios::showpoint | std::ios::showbase; + break; + default: + goto parse_width; + } + ++i1; + } // loop on flag. + if( i1>=buf.size()) { + maybe_throw_exception(exceptions); + return true; + } + + parse_width: + // handle width spec + skip_asterisk(buf, &i1, os); // skips 'asterisk fields' : *, or *N$ + i0 = i1; // save position before digits + while (i1<buf.size() && wrap_isdigit(buf[i1], os)) + i1++; + + if (i1!=i0) + { fpar->ref_state_.width_ = str2int( buf,i0, os, std::streamsize(0) ); } + + parse_precision: + if( i1>=buf.size()) { + maybe_throw_exception(exceptions); + return true; + } + // handle precision spec + if (buf[i1]=='.') + { + ++i1; + skip_asterisk(buf, &i1, os); + i0 = i1; // save position before digits + while (i1<buf.size() && wrap_isdigit(buf[i1], os)) + ++i1; + + if(i1==i0) + fpar->ref_state_.precision_ = 0; + else + fpar->ref_state_.precision_ = str2int(buf,i0, os, std::streamsize(0) ); + } + + // handle formatting-type flags : + while( i1<buf.size() && + ( buf[i1]=='l' || buf[i1]=='L' || buf[i1]=='h') ) + ++i1; + if( i1>=buf.size()) { + maybe_throw_exception(exceptions); + return true; + } + + if( in_brackets && buf[i1]=='|' ) + { + ++i1; + return true; + } + switch (buf[i1]) + { + case 'X': + fpar->ref_state_.flags_ |= std::ios::uppercase; + case 'p': // pointer => set hex. + case 'x': + fpar->ref_state_.flags_ &= ~std::ios::basefield; + fpar->ref_state_.flags_ |= std::ios::hex; + break; + + case 'o': + fpar->ref_state_.flags_ &= ~std::ios::basefield; + fpar->ref_state_.flags_ |= std::ios::oct; + break; + + case 'E': + fpar->ref_state_.flags_ |= std::ios::uppercase; + case 'e': + fpar->ref_state_.flags_ &= ~std::ios::floatfield; + fpar->ref_state_.flags_ |= std::ios::scientific; + + fpar->ref_state_.flags_ &= ~std::ios::basefield; + fpar->ref_state_.flags_ |= std::ios::dec; + break; + + case 'f': + fpar->ref_state_.flags_ &= ~std::ios::floatfield; + fpar->ref_state_.flags_ |= std::ios::fixed; + case 'u': + case 'd': + case 'i': + fpar->ref_state_.flags_ &= ~std::ios::basefield; + fpar->ref_state_.flags_ |= std::ios::dec; + break; + + case 'T': + ++i1; + if( i1 >= buf.size()) + maybe_throw_exception(exceptions); + else + fpar->ref_state_.fill_ = buf[i1]; + fpar->pad_scheme_ |= format_item_t::tabulation; + fpar->argN_ = format_item_t::argN_tabulation; + break; + case 't': + fpar->ref_state_.fill_ = ' '; + fpar->pad_scheme_ |= format_item_t::tabulation; + fpar->argN_ = format_item_t::argN_tabulation; + break; + + case 'G': + fpar->ref_state_.flags_ |= std::ios::uppercase; + break; + case 'g': // 'g' conversion is default for floats. + fpar->ref_state_.flags_ &= ~std::ios::basefield; + fpar->ref_state_.flags_ |= std::ios::dec; + + // CLEAR all floatield flags, so stream will CHOOSE + fpar->ref_state_.flags_ &= ~std::ios::floatfield; + break; + + case 'C': + case 'c': + fpar->truncate_ = 1; + break; + case 'S': + case 's': + fpar->truncate_ = fpar->ref_state_.precision_; + fpar->ref_state_.precision_ = -1; + break; + case 'n' : + fpar->argN_ = format_item_t::argN_ignored; + break; + default: + maybe_throw_exception(exceptions); + } + ++i1; + + if( in_brackets ) + { + if( i1<buf.size() && buf[i1]=='|' ) + { + ++i1; + return true; + } + else maybe_throw_exception(exceptions); + } + return true; + } + +} // detail namespace +} // io namespace + + +// ----------------------------------------------- +// format :: parse(..) + +void basic_format::parse(const string_t & buf) + // parse the format-string +{ + using namespace std; + const char arg_mark = '%'; + bool ordered_args=true; + int max_argN=-1; + string_t::size_type i1=0; + int num_items=0; + + // A: find upper_bound on num_items and allocates arrays + i1=0; + while( (i1=buf.find(arg_mark,i1)) != string::npos ) + { + if( i1+1 >= buf.size() ) { + if(exceptions() & io::bad_format_string_bit) + boost::throw_exception(io::bad_format_string()); // must not end in "bla bla %" + else break; // stop there, ignore last '%' + } + if(buf[i1+1] == buf[i1] ) { i1+=2; continue; } // escaped "%%" / "##" + ++i1; + + // in case of %N% directives, dont count it double (wastes allocations..) : + while(i1 < buf.size() && io::detail::wrap_isdigit(buf[i1],oss_)) ++i1; + if( i1 < buf.size() && buf[i1] == arg_mark ) ++ i1; + + ++num_items; + } + items_.assign( num_items, format_item_t() ); + + // B: Now the real parsing of the format string : + num_items=0; + i1 = 0; + string_t::size_type i0 = i1; + bool special_things=false; + int cur_it=0; + while( (i1=buf.find(arg_mark,i1)) != string::npos ) + { + string_t & piece = (cur_it==0) ? prefix_ : items_[cur_it-1].appendix_; + + if( buf[i1+1] == buf[i1] ) // escaped mark, '%%' + { + piece += buf.substr(i0, i1-i0) + buf[i1]; + i1+=2; i0=i1; + continue; + } + BOOST_ASSERT( static_cast<unsigned int>(cur_it) < items_.size() || cur_it==0); + + if(i1!=i0) piece += buf.substr(i0, i1-i0); + ++i1; + + bool parse_ok; + parse_ok = io::detail::parse_printf_directive(buf, &i1, &items_[cur_it], oss_, exceptions()); + if( ! parse_ok ) continue; // the directive will be printed verbatim + + i0=i1; + items_[cur_it].compute_states(); // process complex options, like zeropad, into stream params. + + int argN=items_[cur_it].argN_; + if(argN == format_item_t::argN_ignored) + continue; + if(argN ==format_item_t::argN_no_posit) + ordered_args=false; + else if(argN == format_item_t::argN_tabulation) special_things=true; + else if(argN > max_argN) max_argN = argN; + ++num_items; + ++cur_it; + } // loop on %'s + BOOST_ASSERT(cur_it == num_items); + + // store the final piece of string + string_t & piece = (cur_it==0) ? prefix_ : items_[cur_it-1].appendix_; + piece += buf.substr(i0); + + if( !ordered_args) + { + if(max_argN >= 0 ) // dont mix positional with non-positionnal directives + { + if(exceptions() & io::bad_format_string_bit) + boost::throw_exception(io::bad_format_string()); + // else do nothing. => positionnal arguments are processed as non-positionnal + } + // set things like it would have been with positional directives : + int non_ordered_items = 0; + for(int i=0; i< num_items; ++i) + if(items_[i].argN_ == format_item_t::argN_no_posit) + { + items_[i].argN_ = non_ordered_items; + ++non_ordered_items; + } + max_argN = non_ordered_items-1; + } + + // C: set some member data : + items_.resize(num_items); + + if(special_things) style_ |= special_needs; + num_args_ = max_argN + 1; + if(ordered_args) style_ |= ordered; + else style_ &= ~ordered; +} + +} // namespace boost + + +#endif // BOOST_FORMAT_PARSING_HPP diff --git a/src/boost/shared_ptr.hpp b/src/boost/shared_ptr.hpp new file mode 100644 index 000000000000..1702ad268811 --- /dev/null +++ b/src/boost/shared_ptr.hpp @@ -0,0 +1,474 @@ +#ifndef BOOST_SHARED_PTR_HPP_INCLUDED +#define BOOST_SHARED_PTR_HPP_INCLUDED + +// +// shared_ptr.hpp +// +// (C) Copyright Greg Colvin and Beman Dawes 1998, 1999. +// Copyright (c) 2001, 2002, 2003 Peter Dimov +// +// Permission to copy, use, modify, sell and distribute this software +// is granted provided this copyright notice appears in all copies. +// This software is provided "as is" without express or implied +// warranty, and with no claim as to its suitability for any purpose. +// +// See http://www.boost.org/libs/smart_ptr/shared_ptr.htm for documentation. +// + +//#include <boost/config.hpp> // for broken compiler workarounds + +#if defined(BOOST_NO_MEMBER_TEMPLATES) && !defined(BOOST_MSVC6_MEMBER_TEMPLATES) +#include <boost/detail/shared_ptr_nmt.hpp> +#else + +#include <boost/assert.hpp> +#include <boost/checked_delete.hpp> +#include <boost/throw_exception.hpp> +#include <boost/detail/shared_count.hpp> +#include <boost/detail/workaround.hpp> + +#include <memory> // for std::auto_ptr +#include <algorithm> // for std::swap +#include <functional> // for std::less +#include <typeinfo> // for std::bad_cast +#include <iosfwd> // for std::basic_ostream + +#ifdef BOOST_MSVC // moved here to work around VC++ compiler crash +# pragma warning(push) +# pragma warning(disable:4284) // odd return type for operator-> +#endif + +namespace boost +{ + +template<class T> class weak_ptr; +template<class T> class enable_shared_from_this; + +namespace detail +{ + +struct static_cast_tag {}; +struct const_cast_tag {}; +struct dynamic_cast_tag {}; +struct polymorphic_cast_tag {}; + +template<class T> struct shared_ptr_traits +{ + typedef T & reference; +}; + +template<> struct shared_ptr_traits<void> +{ + typedef void reference; +}; + +#if !defined(BOOST_NO_CV_VOID_SPECIALIZATIONS) + +template<> struct shared_ptr_traits<void const> +{ + typedef void reference; +}; + +template<> struct shared_ptr_traits<void volatile> +{ + typedef void reference; +}; + +template<> struct shared_ptr_traits<void const volatile> +{ + typedef void reference; +}; + +#endif + +// enable_shared_from_this support + +template<class T, class Y> void sp_enable_shared_from_this(boost::enable_shared_from_this<T> * pe, Y * px, shared_count const & pn) +{ + if(pe != 0) pe->_internal_weak_this._internal_assign(px, pn); +} + +inline void sp_enable_shared_from_this(void const volatile *, void const volatile *, shared_count const &) +{ +} + +} // namespace detail + + +// +// shared_ptr +// +// An enhanced relative of scoped_ptr with reference counted copy semantics. +// The object pointed to is deleted when the last shared_ptr pointing to it +// is destroyed or reset. +// + +template<class T> class shared_ptr +{ +private: + + // Borland 5.5.1 specific workaround + typedef shared_ptr<T> this_type; + +public: + + typedef T element_type; + typedef T value_type; + typedef T * pointer; + typedef typename detail::shared_ptr_traits<T>::reference reference; + + shared_ptr(): px(0), pn() // never throws in 1.30+ + { + } + + template<class Y> + explicit shared_ptr(Y * p): px(p), pn(p, checked_deleter<Y>()) // Y must be complete + { + detail::sp_enable_shared_from_this(p, p, pn); + } + + // + // Requirements: D's copy constructor must not throw + // + // shared_ptr will release p by calling d(p) + // + + template<class Y, class D> shared_ptr(Y * p, D d): px(p), pn(p, d) + { + detail::sp_enable_shared_from_this(p, p, pn); + } + +// generated copy constructor, assignment, destructor are fine... + +// except that Borland C++ has a bug, and g++ with -Wsynth warns +#if defined(__BORLANDC__) || defined(__GNUC__) + + shared_ptr & operator=(shared_ptr const & r) // never throws + { + px = r.px; + pn = r.pn; // shared_count::op= doesn't throw + return *this; + } + +#endif + + template<class Y> + explicit shared_ptr(weak_ptr<Y> const & r): pn(r.pn) // may throw + { + // it is now safe to copy r.px, as pn(r.pn) did not throw + px = r.px; + } + + template<class Y> + shared_ptr(shared_ptr<Y> const & r): px(r.px), pn(r.pn) // never throws + { + } + + template<class Y> + shared_ptr(shared_ptr<Y> const & r, detail::static_cast_tag): px(static_cast<element_type *>(r.px)), pn(r.pn) + { + } + + template<class Y> + shared_ptr(shared_ptr<Y> const & r, detail::const_cast_tag): px(const_cast<element_type *>(r.px)), pn(r.pn) + { + } + + template<class Y> + shared_ptr(shared_ptr<Y> const & r, detail::dynamic_cast_tag): px(dynamic_cast<element_type *>(r.px)), pn(r.pn) + { + if(px == 0) // need to allocate new counter -- the cast failed + { + pn = detail::shared_count(); + } + } + + template<class Y> + shared_ptr(shared_ptr<Y> const & r, detail::polymorphic_cast_tag): px(dynamic_cast<element_type *>(r.px)), pn(r.pn) + { + if(px == 0) + { + boost::throw_exception(std::bad_cast()); + } + } + +#ifndef BOOST_NO_AUTO_PTR + + template<class Y> + explicit shared_ptr(std::auto_ptr<Y> & r): px(r.get()), pn() + { + Y * tmp = r.get(); + pn = detail::shared_count(r); + detail::sp_enable_shared_from_this(tmp, tmp, pn); + } + +#endif + +#if !defined(BOOST_MSVC) || (BOOST_MSVC > 1200) + + template<class Y> + shared_ptr & operator=(shared_ptr<Y> const & r) // never throws + { + px = r.px; + pn = r.pn; // shared_count::op= doesn't throw + return *this; + } + +#endif + +#ifndef BOOST_NO_AUTO_PTR + + template<class Y> + shared_ptr & operator=(std::auto_ptr<Y> & r) + { + this_type(r).swap(*this); + return *this; + } + +#endif + + void reset() // never throws in 1.30+ + { + this_type().swap(*this); + } + + template<class Y> void reset(Y * p) // Y must be complete + { + BOOST_ASSERT(p == 0 || p != px); // catch self-reset errors + this_type(p).swap(*this); + } + + template<class Y, class D> void reset(Y * p, D d) + { + this_type(p, d).swap(*this); + } + + reference operator* () const // never throws + { + BOOST_ASSERT(px != 0); + return *px; + } + + T * operator-> () const // never throws + { + BOOST_ASSERT(px != 0); + return px; + } + + T * get() const // never throws + { + return px; + } + + // implicit conversion to "bool" + +#if defined(__SUNPRO_CC) && BOOST_WORKAROUND(__SUNPRO_CC, <= 0x530) + + operator bool () const + { + return px != 0; + } + +#elif defined(__MWERKS__) && BOOST_WORKAROUND(__MWERKS__, BOOST_TESTED_AT(0x3003)) + typedef T * (this_type::*unspecified_bool_type)() const; + + operator unspecified_bool_type() const // never throws + { + return px == 0? 0: &this_type::get; + } + +#else + + typedef T * this_type::*unspecified_bool_type; + + operator unspecified_bool_type() const // never throws + { + return px == 0? 0: &this_type::px; + } + +#endif + + // operator! is redundant, but some compilers need it + + bool operator! () const // never throws + { + return px == 0; + } + + bool unique() const // never throws + { + return pn.unique(); + } + + long use_count() const // never throws + { + return pn.use_count(); + } + + void swap(shared_ptr<T> & other) // never throws + { + std::swap(px, other.px); + pn.swap(other.pn); + } + + template<class Y> bool _internal_less(shared_ptr<Y> const & rhs) const + { + return pn < rhs.pn; + } + + void * _internal_get_deleter(std::type_info const & ti) const + { + return pn.get_deleter(ti); + } + +// Tasteless as this may seem, making all members public allows member templates +// to work in the absence of member template friends. (Matthew Langston) + +#if 0 + +private: + + template<class Y> friend class shared_ptr; + template<class Y> friend class weak_ptr; + + +#endif + + T * px; // contained pointer + detail::shared_count pn; // reference counter + +}; // shared_ptr + +template<class T, class U> inline bool operator==(shared_ptr<T> const & a, shared_ptr<U> const & b) +{ + return a.get() == b.get(); +} + +template<class T, class U> inline bool operator!=(shared_ptr<T> const & a, shared_ptr<U> const & b) +{ + return a.get() != b.get(); +} + +#if __GNUC__ == 2 && __GNUC_MINOR__ <= 96 + +// Resolve the ambiguity between our op!= and the one in rel_ops + +template<class T> inline bool operator!=(shared_ptr<T> const & a, shared_ptr<T> const & b) +{ + return a.get() != b.get(); +} + +#endif + +template<class T, class U> inline bool operator<(shared_ptr<T> const & a, shared_ptr<U> const & b) +{ + return a._internal_less(b); +} + +template<class T> inline void swap(shared_ptr<T> & a, shared_ptr<T> & b) +{ + a.swap(b); +} + +template<class T, class U> shared_ptr<T> static_pointer_cast(shared_ptr<U> const & r) +{ + return shared_ptr<T>(r, detail::static_cast_tag()); +} + +template<class T, class U> shared_ptr<T> const_pointer_cast(shared_ptr<U> const & r) +{ + return shared_ptr<T>(r, detail::const_cast_tag()); +} + +template<class T, class U> shared_ptr<T> dynamic_pointer_cast(shared_ptr<U> const & r) +{ + return shared_ptr<T>(r, detail::dynamic_cast_tag()); +} + +// shared_*_cast names are deprecated. Use *_pointer_cast instead. + +template<class T, class U> shared_ptr<T> shared_static_cast(shared_ptr<U> const & r) +{ + return shared_ptr<T>(r, detail::static_cast_tag()); +} + +template<class T, class U> shared_ptr<T> shared_dynamic_cast(shared_ptr<U> const & r) +{ + return shared_ptr<T>(r, detail::dynamic_cast_tag()); +} + +template<class T, class U> shared_ptr<T> shared_polymorphic_cast(shared_ptr<U> const & r) +{ + return shared_ptr<T>(r, detail::polymorphic_cast_tag()); +} + +template<class T, class U> shared_ptr<T> shared_polymorphic_downcast(shared_ptr<U> const & r) +{ + BOOST_ASSERT(dynamic_cast<T *>(r.get()) == r.get()); + return shared_static_cast<T>(r); +} + +// get_pointer() enables boost::mem_fn to recognize shared_ptr + +template<class T> inline T * get_pointer(shared_ptr<T> const & p) +{ + return p.get(); +} + +// operator<< + +#if defined(__GNUC__) && (__GNUC__ < 3) + +template<class Y> std::ostream & operator<< (std::ostream & os, shared_ptr<Y> const & p) +{ + os << p.get(); + return os; +} + +#else + +# if defined(BOOST_MSVC) && BOOST_WORKAROUND(BOOST_MSVC, <= 1200 && __SGI_STL_PORT) +// MSVC6 has problems finding std::basic_ostream through the using declaration in namespace _STL +using std::basic_ostream; +template<class E, class T, class Y> basic_ostream<E, T> & operator<< (basic_ostream<E, T> & os, shared_ptr<Y> const & p) +# else +template<class E, class T, class Y> std::basic_ostream<E, T> & operator<< (std::basic_ostream<E, T> & os, shared_ptr<Y> const & p) +# endif +{ + os << p.get(); + return os; +} + +#endif + +// get_deleter (experimental) + +#if (defined(__GNUC__) && (__GNUC__ < 3)) || (defined(__EDG_VERSION__) && (__EDG_VERSION__ <= 238)) + +// g++ 2.9x doesn't allow static_cast<X const *>(void *) +// apparently EDG 2.38 also doesn't accept it + +template<class D, class T> D * get_deleter(shared_ptr<T> const & p) +{ + void const * q = p._internal_get_deleter(typeid(D)); + return const_cast<D *>(static_cast<D const *>(q)); +} + +#else + +template<class D, class T> D * get_deleter(shared_ptr<T> const & p) +{ + return static_cast<D *>(p._internal_get_deleter(typeid(D))); +} + +#endif + +} // namespace boost + +#ifdef BOOST_MSVC +# pragma warning(pop) +#endif + +#endif // #if defined(BOOST_NO_MEMBER_TEMPLATES) && !defined(BOOST_MSVC6_MEMBER_TEMPLATES) + +#endif // #ifndef BOOST_SHARED_PTR_HPP_INCLUDED diff --git a/src/boost/throw_exception.hpp b/src/boost/throw_exception.hpp new file mode 100644 index 000000000000..07b4ae5ceae7 --- /dev/null +++ b/src/boost/throw_exception.hpp @@ -0,0 +1,47 @@ +#ifndef BOOST_THROW_EXCEPTION_HPP_INCLUDED +#define BOOST_THROW_EXCEPTION_HPP_INCLUDED + +// MS compatible compilers support #pragma once + +#if defined(_MSC_VER) && (_MSC_VER >= 1020) +# pragma once +#endif + +// +// boost/throw_exception.hpp +// +// Copyright (c) 2002 Peter Dimov and Multi Media Ltd. +// +// Permission to copy, use, modify, sell and distribute this software +// is granted provided this copyright notice appears in all copies. +// This software is provided "as is" without express or implied +// warranty, and with no claim as to its suitability for any purpose. +// +// http://www.boost.org/libs/utility/throw_exception.html +// + +//#include <boost/config.hpp> + +#ifdef BOOST_NO_EXCEPTIONS +# include <exception> +#endif + +namespace boost +{ + +#ifdef BOOST_NO_EXCEPTIONS + +void throw_exception(std::exception const & e); // user defined + +#else + +template<class E> void throw_exception(E const & e) +{ + throw e; +} + +#endif + +} // namespace boost + +#endif // #ifndef BOOST_THROW_EXCEPTION_HPP_INCLUDED diff --git a/src/boost/weak_ptr.hpp b/src/boost/weak_ptr.hpp new file mode 100644 index 000000000000..3d3b3eb79df1 --- /dev/null +++ b/src/boost/weak_ptr.hpp @@ -0,0 +1,193 @@ +#ifndef BOOST_WEAK_PTR_HPP_INCLUDED +#define BOOST_WEAK_PTR_HPP_INCLUDED + +// +// weak_ptr.hpp +// +// Copyright (c) 2001, 2002, 2003 Peter Dimov +// +// Permission to copy, use, modify, sell and distribute this software +// is granted provided this copyright notice appears in all copies. +// This software is provided "as is" without express or implied +// warranty, and with no claim as to its suitability for any purpose. +// +// See http://www.boost.org/libs/smart_ptr/weak_ptr.htm for documentation. +// + +#include <boost/shared_ptr.hpp> + +#ifdef BOOST_MSVC // moved here to work around VC++ compiler crash +# pragma warning(push) +# pragma warning(disable:4284) // odd return type for operator-> +#endif + +namespace boost +{ + +template<class T> class weak_ptr +{ +private: + + // Borland 5.5.1 specific workarounds + typedef weak_ptr<T> this_type; + +public: + + typedef T element_type; + + weak_ptr(): px(0), pn() // never throws in 1.30+ + { + } + +// generated copy constructor, assignment, destructor are fine + + +// +// The "obvious" converting constructor implementation: +// +// template<class Y> +// weak_ptr(weak_ptr<Y> const & r): px(r.px), pn(r.pn) // never throws +// { +// } +// +// has a serious problem. +// +// r.px may already have been invalidated. The px(r.px) +// conversion may require access to *r.px (virtual inheritance). +// +// It is not possible to avoid spurious access violations since +// in multithreaded programs r.px may be invalidated at any point. +// + + template<class Y> + weak_ptr(weak_ptr<Y> const & r): pn(r.pn) // never throws + { + px = r.lock().get(); + } + + template<class Y> + weak_ptr(shared_ptr<Y> const & r): px(r.px), pn(r.pn) // never throws + { + } + +#if !defined(BOOST_MSVC) || (BOOST_MSVC > 1200) + + template<class Y> + weak_ptr & operator=(weak_ptr<Y> const & r) // never throws + { + px = r.lock().get(); + pn = r.pn; + return *this; + } + + template<class Y> + weak_ptr & operator=(shared_ptr<Y> const & r) // never throws + { + px = r.px; + pn = r.pn; + return *this; + } + +#endif + + shared_ptr<T> lock() const // never throws + { +#if defined(BOOST_HAS_THREADS) + + // optimization: avoid throw overhead + if(expired()) + { + return shared_ptr<element_type>(); + } + + try + { + return shared_ptr<element_type>(*this); + } + catch(bad_weak_ptr const &) + { + // Q: how can we get here? + // A: another thread may have invalidated r after the use_count test above. + return shared_ptr<element_type>(); + } + +#else + + // optimization: avoid try/catch overhead when single threaded + return expired()? shared_ptr<element_type>(): shared_ptr<element_type>(*this); + +#endif + } + + long use_count() const // never throws + { + return pn.use_count(); + } + + bool expired() const // never throws + { + return pn.use_count() == 0; + } + + void reset() // never throws in 1.30+ + { + this_type().swap(*this); + } + + void swap(this_type & other) // never throws + { + std::swap(px, other.px); + pn.swap(other.pn); + } + + void _internal_assign(T * px2, detail::shared_count const & pn2) + { + px = px2; + pn = pn2; + } + + template<class Y> bool _internal_less(weak_ptr<Y> const & rhs) const + { + return pn < rhs.pn; + } + +// Tasteless as this may seem, making all members public allows member templates +// to work in the absence of member template friends. (Matthew Langston) + +#if 0 + +private: + + template<class Y> friend class weak_ptr; + template<class Y> friend class shared_ptr; + +#endif + + T * px; // contained pointer + detail::weak_count pn; // reference counter + +}; // weak_ptr + +template<class T, class U> inline bool operator<(weak_ptr<T> const & a, weak_ptr<U> const & b) +{ + return a._internal_less(b); +} + +template<class T> void swap(weak_ptr<T> & a, weak_ptr<T> & b) +{ + a.swap(b); +} + +// deprecated, provided for backward compatibility +template<class T> shared_ptr<T> make_shared(weak_ptr<T> const & r) +{ + return r.lock(); +} + +} // namespace boost + +#ifdef BOOST_MSVC +# pragma warning(pop) +#endif + +#endif // #ifndef BOOST_WEAK_PTR_HPP_INCLUDED diff --git a/src/bsdiff-4.3/bsdiff.1 b/src/bsdiff-4.3/bsdiff.1 new file mode 100644 index 000000000000..ead6c4deb57f --- /dev/null +++ b/src/bsdiff-4.3/bsdiff.1 @@ -0,0 +1,63 @@ +.\"- +.\" Copyright 2003-2005 Colin Percival +.\" All rights reserved +.\" +.\" Redistribution and use in source and binary forms, with or without +.\" modification, are permitted providing that the following conditions +.\" are met: +.\" 1. Redistributions of source code must retain the above copyright +.\" notice, this list of conditions and the following disclaimer. +.\" 2. Redistributions in binary form must reproduce the above copyright +.\" notice, this list of conditions and the following disclaimer in the +.\" documentation and/or other materials provided with the distribution. +.\" +.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +.\" IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +.\" WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY +.\" DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +.\" STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING +.\" IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +.\" POSSIBILITY OF SUCH DAMAGE. +.\" +.\" $FreeBSD: src/usr.bin/bsdiff/bsdiff/bsdiff.1,v 1.1 2005/08/06 01:59:05 cperciva Exp $ +.\" +.Dd May 18, 2003 +.Dt BSDIFF 1 +.Os FreeBSD +.Sh NAME +.Nm bsdiff +.Nd generate a patch between two binary files +.Sh SYNOPSIS +.Nm +.Ao Ar oldfile Ac Ao Ar newfile Ac Ao Ar patchfile Ac +.Sh DESCRIPTION +.Nm +compares +.Ao Ar oldfile Ac +to +.Ao Ar newfile Ac +and writes to +.Ao Ar patchfile Ac +a binary patch suitable for use by bspatch(1). +When +.Ao Ar oldfile Ac +and +.Ao Ar newfile Ac +are two versions of an executable program, the +patches produced are on average a factor of five smaller +than those produced by any other binary patch tool known +to the author. +.Pp +.Nm +uses memory equal to 17 times the size of +.Ao Ar oldfile Ac , +and requires +an absolute minimum working set size of 8 times the size of oldfile. +.Sh SEE ALSO +.Xr bspatch 1 +.Sh AUTHORS +.An Colin Percival Aq cperciva@freebsd.org diff --git a/src/bsdiff-4.3/bsdiff.c b/src/bsdiff-4.3/bsdiff.c new file mode 100644 index 000000000000..374ed038fa1f --- /dev/null +++ b/src/bsdiff-4.3/bsdiff.c @@ -0,0 +1,405 @@ +/*- + * Copyright 2003-2005 Colin Percival + * All rights reserved + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted providing that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#if 0 +__FBSDID("$FreeBSD: src/usr.bin/bsdiff/bsdiff/bsdiff.c,v 1.1 2005/08/06 01:59:05 cperciva Exp $"); +#endif + +#include <sys/types.h> + +#include <bzlib.h> +#include <err.h> +#include <fcntl.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> + +#define MIN(x,y) (((x)<(y)) ? (x) : (y)) + +static void split(off_t *I,off_t *V,off_t start,off_t len,off_t h) +{ + off_t i,j,k,x,tmp,jj,kk; + + if(len<16) { + for(k=start;k<start+len;k+=j) { + j=1;x=V[I[k]+h]; + for(i=1;k+i<start+len;i++) { + if(V[I[k+i]+h]<x) { + x=V[I[k+i]+h]; + j=0; + }; + if(V[I[k+i]+h]==x) { + tmp=I[k+j];I[k+j]=I[k+i];I[k+i]=tmp; + j++; + }; + }; + for(i=0;i<j;i++) V[I[k+i]]=k+j-1; + if(j==1) I[k]=-1; + }; + return; + }; + + x=V[I[start+len/2]+h]; + jj=0;kk=0; + for(i=start;i<start+len;i++) { + if(V[I[i]+h]<x) jj++; + if(V[I[i]+h]==x) kk++; + }; + jj+=start;kk+=jj; + + i=start;j=0;k=0; + while(i<jj) { + if(V[I[i]+h]<x) { + i++; + } else if(V[I[i]+h]==x) { + tmp=I[i];I[i]=I[jj+j];I[jj+j]=tmp; + j++; + } else { + tmp=I[i];I[i]=I[kk+k];I[kk+k]=tmp; + k++; + }; + }; + + while(jj+j<kk) { + if(V[I[jj+j]+h]==x) { + j++; + } else { + tmp=I[jj+j];I[jj+j]=I[kk+k];I[kk+k]=tmp; + k++; + }; + }; + + if(jj>start) split(I,V,start,jj-start,h); + + for(i=0;i<kk-jj;i++) V[I[jj+i]]=kk-1; + if(jj==kk-1) I[jj]=-1; + + if(start+len>kk) split(I,V,kk,start+len-kk,h); +} + +static void qsufsort(off_t *I,off_t *V,u_char *old,off_t oldsize) +{ + off_t buckets[256]; + off_t i,h,len; + + for(i=0;i<256;i++) buckets[i]=0; + for(i=0;i<oldsize;i++) buckets[old[i]]++; + for(i=1;i<256;i++) buckets[i]+=buckets[i-1]; + for(i=255;i>0;i--) buckets[i]=buckets[i-1]; + buckets[0]=0; + + for(i=0;i<oldsize;i++) I[++buckets[old[i]]]=i; + I[0]=oldsize; + for(i=0;i<oldsize;i++) V[i]=buckets[old[i]]; + V[oldsize]=0; + for(i=1;i<256;i++) if(buckets[i]==buckets[i-1]+1) I[buckets[i]]=-1; + I[0]=-1; + + for(h=1;I[0]!=-(oldsize+1);h+=h) { + len=0; + for(i=0;i<oldsize+1;) { + if(I[i]<0) { + len-=I[i]; + i-=I[i]; + } else { + if(len) I[i-len]=-len; + len=V[I[i]]+1-i; + split(I,V,i,len,h); + i+=len; + len=0; + }; + }; + if(len) I[i-len]=-len; + }; + + for(i=0;i<oldsize+1;i++) I[V[i]]=i; +} + +static off_t matchlen(u_char *old,off_t oldsize,u_char *new,off_t newsize) +{ + off_t i; + + for(i=0;(i<oldsize)&&(i<newsize);i++) + if(old[i]!=new[i]) break; + + return i; +} + +static off_t search(off_t *I,u_char *old,off_t oldsize, + u_char *new,off_t newsize,off_t st,off_t en,off_t *pos) +{ + off_t x,y; + + if(en-st<2) { + x=matchlen(old+I[st],oldsize-I[st],new,newsize); + y=matchlen(old+I[en],oldsize-I[en],new,newsize); + + if(x>y) { + *pos=I[st]; + return x; + } else { + *pos=I[en]; + return y; + } + }; + + x=st+(en-st)/2; + if(memcmp(old+I[x],new,MIN(oldsize-I[x],newsize))<0) { + return search(I,old,oldsize,new,newsize,x,en,pos); + } else { + return search(I,old,oldsize,new,newsize,st,x,pos); + }; +} + +static void offtout(off_t x,u_char *buf) +{ + off_t y; + + if(x<0) y=-x; else y=x; + + buf[0]=y%256;y-=buf[0]; + y=y/256;buf[1]=y%256;y-=buf[1]; + y=y/256;buf[2]=y%256;y-=buf[2]; + y=y/256;buf[3]=y%256;y-=buf[3]; + y=y/256;buf[4]=y%256;y-=buf[4]; + y=y/256;buf[5]=y%256;y-=buf[5]; + y=y/256;buf[6]=y%256;y-=buf[6]; + y=y/256;buf[7]=y%256; + + if(x<0) buf[7]|=0x80; +} + +int main(int argc,char *argv[]) +{ + int fd; + u_char *old,*new; + off_t oldsize,newsize; + off_t *I,*V; + off_t scan,pos,len; + off_t lastscan,lastpos,lastoffset; + off_t oldscore,scsc; + off_t s,Sf,lenf,Sb,lenb; + off_t overlap,Ss,lens; + off_t i; + off_t dblen,eblen; + u_char *db,*eb; + u_char buf[8]; + u_char header[32]; + FILE * pf; + BZFILE * pfbz2; + int bz2err; + + if(argc!=4) errx(1,"usage: %s oldfile newfile patchfile\n",argv[0]); + + /* Allocate oldsize+1 bytes instead of oldsize bytes to ensure + that we never try to malloc(0) and get a NULL pointer */ + if(((fd=open(argv[1],O_RDONLY,0))<0) || + ((oldsize=lseek(fd,0,SEEK_END))==-1) || + ((old=malloc(oldsize+1))==NULL) || + (lseek(fd,0,SEEK_SET)!=0) || + (read(fd,old,oldsize)!=oldsize) || + (close(fd)==-1)) err(1,"%s",argv[1]); + + if(((I=malloc((oldsize+1)*sizeof(off_t)))==NULL) || + ((V=malloc((oldsize+1)*sizeof(off_t)))==NULL)) err(1,NULL); + + qsufsort(I,V,old,oldsize); + + free(V); + + /* Allocate newsize+1 bytes instead of newsize bytes to ensure + that we never try to malloc(0) and get a NULL pointer */ + if(((fd=open(argv[2],O_RDONLY,0))<0) || + ((newsize=lseek(fd,0,SEEK_END))==-1) || + ((new=malloc(newsize+1))==NULL) || + (lseek(fd,0,SEEK_SET)!=0) || + (read(fd,new,newsize)!=newsize) || + (close(fd)==-1)) err(1,"%s",argv[2]); + + if(((db=malloc(newsize+1))==NULL) || + ((eb=malloc(newsize+1))==NULL)) err(1,NULL); + dblen=0; + eblen=0; + + /* Create the patch file */ + if ((pf = fopen(argv[3], "w")) == NULL) + err(1, "%s", argv[3]); + + /* Header is + 0 8 "BSDIFF40" + 8 8 length of bzip2ed ctrl block + 16 8 length of bzip2ed diff block + 24 8 length of new file */ + /* File is + 0 32 Header + 32 ?? Bzip2ed ctrl block + ?? ?? Bzip2ed diff block + ?? ?? Bzip2ed extra block */ + memcpy(header,"BSDIFF40",8); + offtout(0, header + 8); + offtout(0, header + 16); + offtout(newsize, header + 24); + if (fwrite(header, 32, 1, pf) != 1) + err(1, "fwrite(%s)", argv[3]); + + /* Compute the differences, writing ctrl as we go */ + if ((pfbz2 = BZ2_bzWriteOpen(&bz2err, pf, 9, 0, 0)) == NULL) + errx(1, "BZ2_bzWriteOpen, bz2err = %d", bz2err); + scan=0;len=0; + lastscan=0;lastpos=0;lastoffset=0; + while(scan<newsize) { + oldscore=0; + + for(scsc=scan+=len;scan<newsize;scan++) { + len=search(I,old,oldsize,new+scan,newsize-scan, + 0,oldsize,&pos); + if (len > 64 * 1024) break; + + for(;scsc<scan+len;scsc++) + if((scsc+lastoffset<oldsize) && + (old[scsc+lastoffset] == new[scsc])) + oldscore++; + + if(((len==oldscore) && (len!=0)) || + (len>oldscore+8)) break; + + if((scan+lastoffset<oldsize) && + (old[scan+lastoffset] == new[scan])) + oldscore--; + }; + + if((len!=oldscore) || (scan==newsize)) { + s=0;Sf=0;lenf=0; + for(i=0;(lastscan+i<scan)&&(lastpos+i<oldsize);) { + if(old[lastpos+i]==new[lastscan+i]) s++; + i++; + if(s*2-i>Sf*2-lenf) { Sf=s; lenf=i; }; + }; + + lenb=0; + if(scan<newsize) { + s=0;Sb=0; + for(i=1;(scan>=lastscan+i)&&(pos>=i);i++) { + if(old[pos-i]==new[scan-i]) s++; + if(s*2-i>Sb*2-lenb) { Sb=s; lenb=i; }; + }; + }; + + if(lastscan+lenf>scan-lenb) { + overlap=(lastscan+lenf)-(scan-lenb); + s=0;Ss=0;lens=0; + for(i=0;i<overlap;i++) { + if(new[lastscan+lenf-overlap+i]== + old[lastpos+lenf-overlap+i]) s++; + if(new[scan-lenb+i]== + old[pos-lenb+i]) s--; + if(s>Ss) { Ss=s; lens=i+1; }; + }; + + lenf+=lens-overlap; + lenb-=lens; + }; + + for(i=0;i<lenf;i++) + db[dblen+i]=new[lastscan+i]-old[lastpos+i]; + for(i=0;i<(scan-lenb)-(lastscan+lenf);i++) + eb[eblen+i]=new[lastscan+lenf+i]; + + dblen+=lenf; + eblen+=(scan-lenb)-(lastscan+lenf); + + offtout(lenf,buf); + BZ2_bzWrite(&bz2err, pfbz2, buf, 8); + if (bz2err != BZ_OK) + errx(1, "BZ2_bzWrite, bz2err = %d", bz2err); + + offtout((scan-lenb)-(lastscan+lenf),buf); + BZ2_bzWrite(&bz2err, pfbz2, buf, 8); + if (bz2err != BZ_OK) + errx(1, "BZ2_bzWrite, bz2err = %d", bz2err); + + offtout((pos-lenb)-(lastpos+lenf),buf); + BZ2_bzWrite(&bz2err, pfbz2, buf, 8); + if (bz2err != BZ_OK) + errx(1, "BZ2_bzWrite, bz2err = %d", bz2err); + + lastscan=scan-lenb; + lastpos=pos-lenb; + lastoffset=pos-scan; + }; + }; + BZ2_bzWriteClose(&bz2err, pfbz2, 0, NULL, NULL); + if (bz2err != BZ_OK) + errx(1, "BZ2_bzWriteClose, bz2err = %d", bz2err); + + /* Compute size of compressed ctrl data */ + if ((len = ftello(pf)) == -1) + err(1, "ftello"); + offtout(len-32, header + 8); + + /* Write compressed diff data */ + if ((pfbz2 = BZ2_bzWriteOpen(&bz2err, pf, 9, 0, 0)) == NULL) + errx(1, "BZ2_bzWriteOpen, bz2err = %d", bz2err); + BZ2_bzWrite(&bz2err, pfbz2, db, dblen); + if (bz2err != BZ_OK) + errx(1, "BZ2_bzWrite, bz2err = %d", bz2err); + BZ2_bzWriteClose(&bz2err, pfbz2, 0, NULL, NULL); + if (bz2err != BZ_OK) + errx(1, "BZ2_bzWriteClose, bz2err = %d", bz2err); + + /* Compute size of compressed diff data */ + if ((newsize = ftello(pf)) == -1) + err(1, "ftello"); + offtout(newsize - len, header + 16); + + /* Write compressed extra data */ + if ((pfbz2 = BZ2_bzWriteOpen(&bz2err, pf, 9, 0, 0)) == NULL) + errx(1, "BZ2_bzWriteOpen, bz2err = %d", bz2err); + BZ2_bzWrite(&bz2err, pfbz2, eb, eblen); + if (bz2err != BZ_OK) + errx(1, "BZ2_bzWrite, bz2err = %d", bz2err); + BZ2_bzWriteClose(&bz2err, pfbz2, 0, NULL, NULL); + if (bz2err != BZ_OK) + errx(1, "BZ2_bzWriteClose, bz2err = %d", bz2err); + + /* Seek to the beginning, write the header, and close the file */ + if (fseeko(pf, 0, SEEK_SET)) + err(1, "fseeko"); + if (fwrite(header, 32, 1, pf) != 1) + err(1, "fwrite(%s)", argv[3]); + if (fclose(pf)) + err(1, "fclose"); + + /* Free the memory we used */ + free(db); + free(eb); + free(I); + free(old); + free(new); + + return 0; +} diff --git a/src/bsdiff-4.3/bspatch.1 b/src/bsdiff-4.3/bspatch.1 new file mode 100644 index 000000000000..82a2781aa7dc --- /dev/null +++ b/src/bsdiff-4.3/bspatch.1 @@ -0,0 +1,59 @@ +.\"- +.\" Copyright 2003-2005 Colin Percival +.\" All rights reserved +.\" +.\" Redistribution and use in source and binary forms, with or without +.\" modification, are permitted providing that the following conditions +.\" are met: +.\" 1. Redistributions of source code must retain the above copyright +.\" notice, this list of conditions and the following disclaimer. +.\" 2. Redistributions in binary form must reproduce the above copyright +.\" notice, this list of conditions and the following disclaimer in the +.\" documentation and/or other materials provided with the distribution. +.\" +.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +.\" IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +.\" WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY +.\" DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +.\" STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING +.\" IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +.\" POSSIBILITY OF SUCH DAMAGE. +.\" +.\" $FreeBSD: src/usr.bin/bsdiff/bspatch/bspatch.1,v 1.1 2005/08/06 01:59:06 cperciva Exp $ +.\" +.Dd May 18, 2003 +.Dt BSPATCH 1 +.Os FreeBSD +.Sh NAME +.Nm bspatch +.Nd apply a patch built with bsdiff(1) +.Sh SYNOPSIS +.Nm +.Ao Ar oldfile Ac Ao Ar newfile Ac Ao Ar patchfile Ac +.Sh DESCRIPTION +.Nm +generates +.Ao Ar newfile Ac +from +.Ao Ar oldfile Ac +and +.Ao Ar patchfile Ac +where +.Ao Ar patchfile Ac +is a binary patch built by bsdiff(1). +.Pp +.Nm +uses memory equal to the size of +.Ao Ar oldfile Ac +plus the size of +.Ao Ar newfile Ac , +but can tolerate a very small working set without a dramatic loss +of performance. +.Sh SEE ALSO +.Xr bsdiff 1 +.Sh AUTHORS +.An Colin Percival Aq cperciva@freebsd.org diff --git a/src/bsdiff-4.3/bspatch.c b/src/bsdiff-4.3/bspatch.c new file mode 100644 index 000000000000..f9d33ddd64a2 --- /dev/null +++ b/src/bsdiff-4.3/bspatch.c @@ -0,0 +1,224 @@ +/*- + * Copyright 2003-2005 Colin Percival + * All rights reserved + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted providing that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#if 0 +__FBSDID("$FreeBSD: src/usr.bin/bsdiff/bspatch/bspatch.c,v 1.1 2005/08/06 01:59:06 cperciva Exp $"); +#endif + +#include <bzlib.h> +#include <stdlib.h> +#include <stdio.h> +#include <string.h> +#include <err.h> +#include <errno.h> +#include <unistd.h> +#include <fcntl.h> +#include <sys/types.h> + +static off_t offtin(u_char *buf) +{ + off_t y; + + y=buf[7]&0x7F; + y=y*256;y+=buf[6]; + y=y*256;y+=buf[5]; + y=y*256;y+=buf[4]; + y=y*256;y+=buf[3]; + y=y*256;y+=buf[2]; + y=y*256;y+=buf[1]; + y=y*256;y+=buf[0]; + + if(buf[7]&0x80) y=-y; + + return y; +} + + +void writeFull(const char * name, int fd, + const unsigned char * buf, size_t count) +{ + while (count) { + ssize_t res = write(fd, (char *) buf, count); + if (res == -1) { + if (errno == EINTR) continue; + err(1,"writing to %s",name); + } + count -= res; + buf += res; + } +} + + +int main(int argc,char * argv[]) +{ + FILE * f, * cpf, * dpf, * epf; + BZFILE * cpfbz2, * dpfbz2, * epfbz2; + int cbz2err, dbz2err, ebz2err; + int fd; + ssize_t oldsize,newsize; + ssize_t bzctrllen,bzdatalen; + u_char header[32],buf[8]; + u_char *old, *new; + off_t oldpos,newpos; + off_t ctrl[3]; + off_t lenread; + off_t i; + + if(argc!=4) errx(1,"usage: %s oldfile newfile patchfile\n",argv[0]); + + /* Open patch file */ + if ((f = fopen(argv[3], "r")) == NULL) + err(1, "fopen(%s)", argv[3]); + + /* + File format: + 0 8 "BSDIFF40" + 8 8 X + 16 8 Y + 24 8 sizeof(newfile) + 32 X bzip2(control block) + 32+X Y bzip2(diff block) + 32+X+Y ??? bzip2(extra block) + with control block a set of triples (x,y,z) meaning "add x bytes + from oldfile to x bytes from the diff block; copy y bytes from the + extra block; seek forwards in oldfile by z bytes". + */ + + /* Read header */ + if (fread(header, 1, 32, f) < 32) { + if (feof(f)) + errx(1, "Corrupt patch\n"); + err(1, "fread(%s)", argv[3]); + } + + /* Check for appropriate magic */ + if (memcmp(header, "BSDIFF40", 8) != 0) + errx(1, "Corrupt patch\n"); + + /* Read lengths from header */ + bzctrllen=offtin(header+8); + bzdatalen=offtin(header+16); + newsize=offtin(header+24); + if((bzctrllen<0) || (bzdatalen<0) || (newsize<0)) + errx(1,"Corrupt patch\n"); + + /* Close patch file and re-open it via libbzip2 at the right places */ + if (fclose(f)) + err(1, "fclose(%s)", argv[3]); + if ((cpf = fopen(argv[3], "r")) == NULL) + err(1, "fopen(%s)", argv[3]); + if (fseeko(cpf, 32, SEEK_SET)) + err(1, "fseeko(%s, %lld)", argv[3], + (long long)32); + if ((cpfbz2 = BZ2_bzReadOpen(&cbz2err, cpf, 0, 0, NULL, 0)) == NULL) + errx(1, "BZ2_bzReadOpen, bz2err = %d", cbz2err); + if ((dpf = fopen(argv[3], "r")) == NULL) + err(1, "fopen(%s)", argv[3]); + if (fseeko(dpf, 32 + bzctrllen, SEEK_SET)) + err(1, "fseeko(%s, %lld)", argv[3], + (long long)(32 + bzctrllen)); + if ((dpfbz2 = BZ2_bzReadOpen(&dbz2err, dpf, 0, 0, NULL, 0)) == NULL) + errx(1, "BZ2_bzReadOpen, bz2err = %d", dbz2err); + if ((epf = fopen(argv[3], "r")) == NULL) + err(1, "fopen(%s)", argv[3]); + if (fseeko(epf, 32 + bzctrllen + bzdatalen, SEEK_SET)) + err(1, "fseeko(%s, %lld)", argv[3], + (long long)(32 + bzctrllen + bzdatalen)); + if ((epfbz2 = BZ2_bzReadOpen(&ebz2err, epf, 0, 0, NULL, 0)) == NULL) + errx(1, "BZ2_bzReadOpen, bz2err = %d", ebz2err); + + if(((fd=open(argv[1],O_RDONLY,0))<0) || + ((oldsize=lseek(fd,0,SEEK_END))==-1) || + ((old=malloc(oldsize+1))==NULL) || + (lseek(fd,0,SEEK_SET)!=0) || + (read(fd,old,oldsize)!=oldsize) || + (close(fd)==-1)) err(1,"%s",argv[1]); + if((new=malloc(newsize+1))==NULL) err(1,NULL); + + oldpos=0;newpos=0; + while(newpos<newsize) { + /* Read control data */ + for(i=0;i<=2;i++) { + lenread = BZ2_bzRead(&cbz2err, cpfbz2, buf, 8); + if ((lenread < 8) || ((cbz2err != BZ_OK) && + (cbz2err != BZ_STREAM_END))) + errx(1, "Corrupt patch\n"); + ctrl[i]=offtin(buf); + }; + + /* Sanity-check */ + if(newpos+ctrl[0]>newsize) + errx(1,"Corrupt patch\n"); + + /* Read diff string */ + lenread = BZ2_bzRead(&dbz2err, dpfbz2, new + newpos, ctrl[0]); + if ((lenread < ctrl[0]) || + ((dbz2err != BZ_OK) && (dbz2err != BZ_STREAM_END))) + errx(1, "Corrupt patch\n"); + + /* Add old data to diff string */ + for(i=0;i<ctrl[0];i++) + if((oldpos+i>=0) && (oldpos+i<oldsize)) + new[newpos+i]+=old[oldpos+i]; + + /* Adjust pointers */ + newpos+=ctrl[0]; + oldpos+=ctrl[0]; + + /* Sanity-check */ + if(newpos+ctrl[1]>newsize) + errx(1,"Corrupt patch\n"); + + /* Read extra string */ + lenread = BZ2_bzRead(&ebz2err, epfbz2, new + newpos, ctrl[1]); + if ((lenread < ctrl[1]) || + ((ebz2err != BZ_OK) && (ebz2err != BZ_STREAM_END))) + errx(1, "Corrupt patch\n"); + + /* Adjust pointers */ + newpos+=ctrl[1]; + oldpos+=ctrl[2]; + }; + + /* Clean up the bzip2 reads */ + BZ2_bzReadClose(&cbz2err, cpfbz2); + BZ2_bzReadClose(&dbz2err, dpfbz2); + BZ2_bzReadClose(&ebz2err, epfbz2); + if (fclose(cpf) || fclose(dpf) || fclose(epf)) + err(1, "fclose(%s)", argv[3]); + + /* Write the new file */ + if((fd=open(argv[2],O_CREAT|O_TRUNC|O_WRONLY,0666))<0) + err(1,"%s",argv[2]); + writeFull(argv[2], fd, new, newsize); + if(close(fd)==-1) + err(1,"%s",argv[2]); + + free(new); + free(old); + + return 0; +} diff --git a/src/bsdiff-4.3/compat-include/err.h b/src/bsdiff-4.3/compat-include/err.h new file mode 100644 index 000000000000..a851ded6f907 --- /dev/null +++ b/src/bsdiff-4.3/compat-include/err.h @@ -0,0 +1,12 @@ +/* Simulate BSD's <err.h> functionality. */ + +#ifndef COMPAT_ERR_H_INCLUDED +#define COMPAT_ERR_H_INCLUDED 1 + +#include <stdio.h> +#include <stdlib.h> + +#define err(rc,...) do { fprintf(stderr,__VA_ARGS__); exit(rc); } while(0) +#define errx(rc,...) do { fprintf(stderr,__VA_ARGS__); exit(rc); } while(0) + +#endif diff --git a/src/bsdiff-4.3/local.mk b/src/bsdiff-4.3/local.mk new file mode 100644 index 000000000000..c957ceab0c0f --- /dev/null +++ b/src/bsdiff-4.3/local.mk @@ -0,0 +1,11 @@ +programs += bsdiff bspatch + +bsdiff_DIR := $(d) +bsdiff_SOURCES := $(d)/bsdiff.c +bsdiff_LDFLAGS = -lbz2 $(bsddiff_compat_include) +bsdiff_INSTALL_DIR = $(libexecdir)/nix + +bspatch_DIR := $(d) +bspatch_SOURCES := $(d)/bspatch.c +bspatch_LDFLAGS = -lbz2 $(bsddiff_compat_include) +bspatch_INSTALL_DIR = $(libexecdir)/nix diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc new file mode 100644 index 000000000000..4f28a549f035 --- /dev/null +++ b/src/libexpr/attr-path.cc @@ -0,0 +1,97 @@ +#include "attr-path.hh" +#include "eval-inline.hh" +#include "util.hh" + + +namespace nix { + + +static Strings parseAttrPath(const string & s) +{ + Strings res; + string cur; + string::const_iterator i = s.begin(); + while (i != s.end()) { + if (*i == '.') { + res.push_back(cur); + cur.clear(); + } else if (*i == '"') { + ++i; + while (1) { + if (i == s.end()) + throw Error(format("missing closing quote in selection path `%1%'") % s); + if (*i == '"') break; + cur.push_back(*i++); + } + } else + cur.push_back(*i); + ++i; + } + if (!cur.empty()) res.push_back(cur); + return res; +} + + +Value * findAlongAttrPath(EvalState & state, const string & attrPath, + Bindings & autoArgs, Value & vIn) +{ + Strings tokens = parseAttrPath(attrPath); + + Error attrError = + Error(format("attribute selection path `%1%' does not match expression") % attrPath); + + Value * v = &vIn; + + foreach (Strings::iterator, i, tokens) { + + /* Is *i an index (integer) or a normal attribute name? */ + enum { apAttr, apIndex } apType = apAttr; + string attr = *i; + unsigned int attrIndex; + if (string2Int(attr, attrIndex)) apType = apIndex; + + /* Evaluate the expression. */ + Value * vNew = state.allocValue(); + state.autoCallFunction(autoArgs, *v, *vNew); + v = vNew; + state.forceValue(*v); + + /* It should evaluate to either a set or an expression, + according to what is specified in the attrPath. */ + + if (apType == apAttr) { + + if (v->type != tAttrs) + throw TypeError( + format("the expression selected by the selection path `%1%' should be a set but is %2%") + % attrPath % showType(*v)); + + if (attr.empty()) + throw Error(format("empty attribute name in selection path `%1%'") % attrPath); + + Bindings::iterator a = v->attrs->find(state.symbols.create(attr)); + if (a == v->attrs->end()) + throw Error(format("attribute `%1%' in selection path `%2%' not found") % attr % attrPath); + v = &*a->value; + } + + else if (apType == apIndex) { + + if (v->type != tList) + throw TypeError( + format("the expression selected by the selection path `%1%' should be a list but is %2%") + % attrPath % showType(*v)); + + if (attrIndex >= v->list.length) + throw Error(format("list index %1% in selection path `%2%' is out of range") % attrIndex % attrPath); + + v = v->list.elems[attrIndex]; + } + + } + + return v; +} + + +} diff --git a/src/libexpr/attr-path.hh b/src/libexpr/attr-path.hh new file mode 100644 index 000000000000..46a341950939 --- /dev/null +++ b/src/libexpr/attr-path.hh @@ -0,0 +1,13 @@ +#pragma once + +#include "eval.hh" + +#include <string> +#include <map> + +namespace nix { + +Value * findAlongAttrPath(EvalState & state, const string & attrPath, + Bindings & autoArgs, Value & vIn); + +} diff --git a/src/libexpr/common-opts.cc b/src/libexpr/common-opts.cc new file mode 100644 index 000000000000..0ee889a995ef --- /dev/null +++ b/src/libexpr/common-opts.cc @@ -0,0 +1,59 @@ +#include "common-opts.hh" +#include "../libmain/shared.hh" +#include "util.hh" + + +namespace nix { + + +bool parseOptionArg(const string & arg, Strings::iterator & i, + const Strings::iterator & argsEnd, EvalState & state, + Bindings & autoArgs) +{ + if (arg != "--arg" && arg != "--argstr") return false; + + UsageError error(format("`%1%' requires two arguments") % arg); + + if (i == argsEnd) throw error; + string name = *i++; + if (i == argsEnd) throw error; + string value = *i++; + + /* !!! check for duplicates! */ + Value * v = state.allocValue(); + autoArgs.push_back(Attr(state.symbols.create(name), v)); + + if (arg == "--arg") + state.mkThunk_(*v, state.parseExprFromString(value, absPath("."))); + else + mkString(*v, value); + + autoArgs.sort(); // !!! inefficient + + return true; +} + + +bool parseSearchPathArg(const string & arg, Strings::iterator & i, + const Strings::iterator & argsEnd, EvalState & state) +{ + if (arg != "-I") return false; + if (i == argsEnd) throw UsageError(format("`%1%' requires an argument") % arg);; + state.addToSearchPath(*i++); + return true; +} + + +Path lookupFileArg(EvalState & state, string s) +{ + if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { + Path p = s.substr(1, s.size() - 2); + Path p2 = state.findFile(p); + if (p2 == "") throw Error(format("file `%1%' was not found in the Nix search path (add it using $NIX_PATH or -I)") % p); + return p2; + } else + return absPath(s); +} + + +} diff --git a/src/libexpr/common-opts.hh b/src/libexpr/common-opts.hh new file mode 100644 index 000000000000..e2e3fe77171c --- /dev/null +++ b/src/libexpr/common-opts.hh @@ -0,0 +1,17 @@ +#pragma once + +#include "eval.hh" + +namespace nix { + +/* Some common option parsing between nix-env and nix-instantiate. */ +bool parseOptionArg(const string & arg, Strings::iterator & i, + const Strings::iterator & argsEnd, EvalState & state, + Bindings & autoArgs); + +bool parseSearchPathArg(const string & arg, Strings::iterator & i, + const Strings::iterator & argsEnd, EvalState & state); + +Path lookupFileArg(EvalState & state, string s); + +} diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh new file mode 100644 index 000000000000..5801a20c3b10 --- /dev/null +++ b/src/libexpr/eval-inline.hh @@ -0,0 +1,59 @@ +#pragma once + +#include "eval.hh" + +#define LocalNoInline(f) static f __attribute__((noinline)); f +#define LocalNoInlineNoReturn(f) static f __attribute__((noinline, noreturn)); f + +namespace nix { + +LocalNoInlineNoReturn(void throwEvalError(const char * s)) +{ + throw EvalError(s); +} + +LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v)) +{ + throw TypeError(format(s) % showType(v)); +} + + +void EvalState::forceValue(Value & v) +{ + if (v.type == tThunk) { + Env * env = v.thunk.env; + Expr * expr = v.thunk.expr; + try { + v.type = tBlackhole; + //checkInterrupt(); + expr->eval(*this, *env, v); + } catch (Error & e) { + v.type = tThunk; + v.thunk.env = env; + v.thunk.expr = expr; + throw; + } + } + else if (v.type == tApp) + callFunction(*v.app.left, *v.app.right, v); + else if (v.type == tBlackhole) + throwEvalError("infinite recursion encountered"); +} + + +inline void EvalState::forceAttrs(Value & v) +{ + forceValue(v); + if (v.type != tAttrs) + throwTypeError("value is %1% while a set was expected", v); +} + + +inline void EvalState::forceList(Value & v) +{ + forceValue(v); + if (v.type != tList) + throwTypeError("value is %1% while a list was expected", v); +} + +} diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc new file mode 100644 index 000000000000..2087c7c43ffb --- /dev/null +++ b/src/libexpr/eval.cc @@ -0,0 +1,1435 @@ +#include "eval.hh" +#include "hash.hh" +#include "util.hh" +#include "store-api.hh" +#include "derivations.hh" +#include "globals.hh" +#include "eval-inline.hh" + +#include <cstring> +#include <unistd.h> +#include <sys/time.h> +#include <sys/resource.h> + +#if HAVE_BOEHMGC + +#include <gc/gc.h> +#include <gc/gc_cpp.h> + +#define NEW new (UseGC) + +#else + +#define GC_STRDUP strdup +#define GC_MALLOC malloc + +#define NEW new + +#endif + + +namespace nix { + + +Bindings::iterator Bindings::find(const Symbol & name) +{ + Attr key(name, 0); + iterator i = lower_bound(begin(), end(), key); + if (i != end() && i->name == name) return i; + return end(); +} + + +void Bindings::sort() +{ + std::sort(begin(), end()); +} + + +std::ostream & operator << (std::ostream & str, const Value & v) +{ + switch (v.type) { + case tInt: + str << v.integer; + break; + case tBool: + str << (v.boolean ? "true" : "false"); + break; + case tString: + str << "\""; + for (const char * i = v.string.s; *i; i++) + if (*i == '\"' || *i == '\\') str << "\\" << *i; + else if (*i == '\n') str << "\\n"; + else if (*i == '\r') str << "\\r"; + else if (*i == '\t') str << "\\t"; + else str << *i; + str << "\""; + break; + case tPath: + str << v.path; // !!! escaping? + break; + case tNull: + str << "null"; + break; + case tAttrs: { + str << "{ "; + typedef std::map<string, Value *> Sorted; + Sorted sorted; + foreach (Bindings::iterator, i, *v.attrs) + sorted[i->name] = i->value; + foreach (Sorted::iterator, i, sorted) + str << i->first << " = " << *i->second << "; "; + str << "}"; + break; + } + case tList: + str << "[ "; + for (unsigned int n = 0; n < v.list.length; ++n) + str << *v.list.elems[n] << " "; + str << "]"; + break; + case tThunk: + case tApp: + str << "<CODE>"; + break; + case tLambda: + str << "<LAMBDA>"; + break; + case tPrimOp: + str << "<PRIMOP>"; + break; + case tPrimOpApp: + str << "<PRIMOP-APP>"; + break; + default: + throw Error("invalid value"); + } + return str; +} + + +string showType(const Value & v) +{ + switch (v.type) { + case tInt: return "an integer"; + case tBool: return "a boolean"; + case tString: return "a string"; + case tPath: return "a path"; + case tNull: return "null"; + case tAttrs: return "a set"; + case tList: return "a list"; + case tThunk: return "a thunk"; + case tApp: return "a function application"; + case tLambda: return "a function"; + case tBlackhole: return "a black hole"; + case tPrimOp: return "a built-in function"; + case tPrimOpApp: return "a partially applied built-in function"; + } + abort(); +} + + +/* Called when the Boehm GC runs out of memory. */ +static void * oomHandler(size_t requested) +{ + /* Convert this to a proper C++ exception. */ + throw std::bad_alloc(); +} + + +static Symbol getName(const AttrName & name, EvalState & state, Env & env) { + if (name.symbol.set()) { + return name.symbol; + } else { + Value nameValue; + name.expr->eval(state, env, nameValue); + state.forceStringNoCtx(nameValue); + return state.symbols.create(nameValue.string.s); + } +} + + +EvalState::EvalState() + : sWith(symbols.create("<with>")) + , sOutPath(symbols.create("outPath")) + , sDrvPath(symbols.create("drvPath")) + , sType(symbols.create("type")) + , sMeta(symbols.create("meta")) + , sName(symbols.create("name")) + , sValue(symbols.create("value")) + , sSystem(symbols.create("system")) + , sOverrides(symbols.create("__overrides")) + , sOutputs(symbols.create("outputs")) + , sOutputName(symbols.create("outputName")) + , sIgnoreNulls(symbols.create("__ignoreNulls")) + , sFile(symbols.create("file")) + , sLine(symbols.create("line")) + , sColumn(symbols.create("column")) + , repair(false) + , baseEnv(allocEnv(128)) + , staticBaseEnv(false, 0) + , baseEnvDispl(0) +{ + nrEnvs = nrValuesInEnvs = nrValues = nrListElems = 0; + nrAttrsets = nrOpUpdates = nrOpUpdateValuesCopied = 0; + nrListConcats = nrPrimOpCalls = nrFunctionCalls = 0; + countCalls = getEnv("NIX_COUNT_CALLS", "0") != "0"; + +#if HAVE_BOEHMGC + static bool gcInitialised = false; + if (!gcInitialised) { + + /* Initialise the Boehm garbage collector. This isn't + necessary on most platforms, but for portability we do it + anyway. */ + GC_INIT(); + + GC_oom_fn = oomHandler; + + /* Set the initial heap size to something fairly big (25% of + physical RAM, up to a maximum of 384 MiB) so that in most + cases we don't need to garbage collect at all. (Collection + has a fairly significant overhead.) The heap size can be + overridden through libgc's GC_INITIAL_HEAP_SIZE environment + variable. We should probably also provide a nix.conf + setting for this. Note that GC_expand_hp() causes a lot of + virtual, but not physical (resident) memory to be + allocated. This might be a problem on systems that don't + overcommit. */ + if (!getenv("GC_INITIAL_HEAP_SIZE")) { + size_t maxSize = 384 * 1024 * 1024; + size_t size = 32 * 1024 * 1024; +#if HAVE_SYSCONF && defined(_SC_PAGESIZE) && defined(_SC_PHYS_PAGES) + long pageSize = sysconf(_SC_PAGESIZE); + long pages = sysconf(_SC_PHYS_PAGES); + if (pageSize != -1) + size = (pageSize * pages) / 4; // 25% of RAM + if (size > maxSize) size = maxSize; +#endif + debug(format("setting initial heap size to %1% bytes") % size); + GC_expand_hp(size); + } + + gcInitialised = true; + } +#endif + + /* Initialise the Nix expression search path. */ + searchPathInsertionPoint = searchPath.end(); + Strings paths = tokenizeString<Strings>(getEnv("NIX_PATH", ""), ":"); + foreach (Strings::iterator, i, paths) addToSearchPath(*i); + addToSearchPath("nix=" + settings.nixDataDir + "/nix/corepkgs"); + searchPathInsertionPoint = searchPath.begin(); + + createBaseEnv(); +} + + +EvalState::~EvalState() +{ +} + + +void EvalState::addConstant(const string & name, Value & v) +{ + Value * v2 = allocValue(); + *v2 = v; + staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl; + baseEnv.values[baseEnvDispl++] = v2; + string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name; + baseEnv.values[0]->attrs->push_back(Attr(symbols.create(name2), v2)); +} + + +void EvalState::addPrimOp(const string & name, + unsigned int arity, PrimOpFun primOp) +{ + Value * v = allocValue(); + string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name; + Symbol sym = symbols.create(name2); + v->type = tPrimOp; + v->primOp = NEW PrimOp(primOp, arity, sym); + staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl; + baseEnv.values[baseEnvDispl++] = v; + baseEnv.values[0]->attrs->push_back(Attr(sym, v)); +} + + +void EvalState::getBuiltin(const string & name, Value & v) +{ + v = *baseEnv.values[0]->attrs->find(symbols.create(name))->value; +} + + +/* Every "format" object (even temporary) takes up a few hundred bytes + of stack space, which is a real killer in the recursive + evaluator. So here are some helper functions for throwing + exceptions. */ + +LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2)) +{ + throw EvalError(format(s) % s2); +} + +LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, const string & s3)) +{ + throw EvalError(format(s) % s2 % s3); +} + +LocalNoInlineNoReturn(void throwEvalError(const char * s, const Symbol & sym, const Pos & p1, const Pos & p2)) +{ + throw EvalError(format(s) % sym % p1 % p2); +} + +LocalNoInlineNoReturn(void throwTypeError(const char * s)) +{ + throw TypeError(s); +} + +LocalNoInlineNoReturn(void throwTypeError(const char * s, const string & s1)) +{ + throw TypeError(format(s) % s1); +} + +LocalNoInlineNoReturn(void throwTypeError(const char * s, const string & s1, const string & s2)) +{ + throw TypeError(format(s) % s1 % s2); +} + +LocalNoInlineNoReturn(void throwTypeError(const char * s, const ExprLambda & fun, const Symbol & s2)) +{ + throw TypeError(format(s) % fun.showNamePos() % s2); +} + +LocalNoInlineNoReturn(void throwAssertionError(const char * s, const Pos & pos)) +{ + throw AssertionError(format(s) % pos); +} + +LocalNoInlineNoReturn(void throwUndefinedVarError(const char * s, const string & s1, const Pos & pos)) +{ + throw UndefinedVarError(format(s) % s1 % pos); +} + +LocalNoInline(void addErrorPrefix(Error & e, const char * s, const string & s2)) +{ + e.addPrefix(format(s) % s2); +} + +LocalNoInline(void addErrorPrefix(Error & e, const char * s, const ExprLambda & fun)) +{ + e.addPrefix(format(s) % fun.showNamePos()); +} + +LocalNoInline(void addErrorPrefix(Error & e, const char * s, const string & s2, const Pos & pos)) +{ + e.addPrefix(format(s) % s2 % pos); +} + + +void mkString(Value & v, const char * s) +{ + mkStringNoCopy(v, GC_STRDUP(s)); +} + + +void mkString(Value & v, const string & s, const PathSet & context) +{ + mkString(v, s.c_str()); + if (!context.empty()) { + unsigned int n = 0; + v.string.context = (const char * *) + GC_MALLOC((context.size() + 1) * sizeof(char *)); + foreach (PathSet::const_iterator, i, context) + v.string.context[n++] = GC_STRDUP(i->c_str()); + v.string.context[n] = 0; + } +} + + +void mkPath(Value & v, const char * s) +{ + mkPathNoCopy(v, GC_STRDUP(s)); +} + + +inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval) +{ + for (unsigned int l = var.level; l; --l, env = env->up) ; + + if (!var.fromWith) return env->values[var.displ]; + + while (1) { + if (!env->haveWithAttrs) { + if (noEval) return 0; + Value * v = allocValue(); + evalAttrs(*env->up, (Expr *) env->values[0], *v); + env->values[0] = v; + env->haveWithAttrs = true; + } + Bindings::iterator j = env->values[0]->attrs->find(var.name); + if (j != env->values[0]->attrs->end()) { + if (countCalls && j->pos) attrSelects[*j->pos]++; + return j->value; + } + if (!env->prevWith) + throwUndefinedVarError("undefined variable `%1%' at %2%", var.name, var.pos); + for (unsigned int l = env->prevWith; l; --l, env = env->up) ; + } +} + + +Value * EvalState::allocValue() +{ + nrValues++; + return (Value *) GC_MALLOC(sizeof(Value)); +} + + +Env & EvalState::allocEnv(unsigned int size) +{ + nrEnvs++; + nrValuesInEnvs += size; + Env * env = (Env *) GC_MALLOC(sizeof(Env) + size * sizeof(Value *)); + + /* Clear the values because maybeThunk() and lookupVar fromWith expects this. */ + for (unsigned i = 0; i < size; ++i) + env->values[i] = 0; + + return *env; +} + + +Value * EvalState::allocAttr(Value & vAttrs, const Symbol & name) +{ + Value * v = allocValue(); + vAttrs.attrs->push_back(Attr(name, v)); + return v; +} + + +void EvalState::mkList(Value & v, unsigned int length) +{ + v.type = tList; + v.list.length = length; + v.list.elems = length ? (Value * *) GC_MALLOC(length * sizeof(Value *)) : 0; + nrListElems += length; +} + + +void EvalState::mkAttrs(Value & v, unsigned int expected) +{ + clearValue(v); + v.type = tAttrs; + v.attrs = NEW Bindings; + v.attrs->reserve(expected); + nrAttrsets++; +} + + +unsigned long nrThunks = 0; + +static inline void mkThunk(Value & v, Env & env, Expr * expr) +{ + v.type = tThunk; + v.thunk.env = &env; + v.thunk.expr = expr; + nrThunks++; +} + + +void EvalState::mkThunk_(Value & v, Expr * expr) +{ + mkThunk(v, baseEnv, expr); +} + + +void EvalState::mkPos(Value & v, Pos * pos) +{ + if (pos) { + mkAttrs(v, 3); + mkString(*allocAttr(v, sFile), pos->file); + mkInt(*allocAttr(v, sLine), pos->line); + mkInt(*allocAttr(v, sColumn), pos->column); + v.attrs->sort(); + } else + mkNull(v); +} + + +/* Create a thunk for the delayed computation of the given expression + in the given environment. But if the expression is a variable, + then look it up right away. This significantly reduces the number + of thunks allocated. */ +Value * Expr::maybeThunk(EvalState & state, Env & env) +{ + Value * v = state.allocValue(); + mkThunk(*v, env, this); + return v; +} + + +unsigned long nrAvoided = 0; + +Value * ExprVar::maybeThunk(EvalState & state, Env & env) +{ + Value * v = state.lookupVar(&env, *this, true); + /* The value might not be initialised in the environment yet. + In that case, ignore it. */ + if (v) { nrAvoided++; return v; } + return Expr::maybeThunk(state, env); +} + + +Value * ExprString::maybeThunk(EvalState & state, Env & env) +{ + nrAvoided++; + return &v; +} + +Value * ExprInt::maybeThunk(EvalState & state, Env & env) +{ + nrAvoided++; + return &v; +} + +Value * ExprPath::maybeThunk(EvalState & state, Env & env) +{ + nrAvoided++; + return &v; +} + + +void EvalState::evalFile(const Path & path, Value & v) +{ + FileEvalCache::iterator i; + if ((i = fileEvalCache.find(path)) != fileEvalCache.end()) { + v = i->second; + return; + } + + Path path2 = resolveExprPath(path); + if ((i = fileEvalCache.find(path2)) != fileEvalCache.end()) { + v = i->second; + return; + } + + startNest(nest, lvlTalkative, format("evaluating file `%1%'") % path2); + Expr * e = parseExprFromFile(path2); + try { + eval(e, v); + } catch (Error & e) { + addErrorPrefix(e, "while evaluating the file `%1%':\n", path2); + throw; + } + + fileEvalCache[path2] = v; + if (path != path2) fileEvalCache[path] = v; +} + + +void EvalState::resetFileCache() +{ + fileEvalCache.clear(); +} + + +void EvalState::eval(Expr * e, Value & v) +{ + e->eval(*this, baseEnv, v); +} + + +inline bool EvalState::evalBool(Env & env, Expr * e, Value & v) +{ + e->eval(*this, env, v); + if (v.type != tBool) + throwTypeError("value is %1% while a Boolean was expected", v); + return v.boolean; +} + + +inline bool EvalState::evalBool(Env & env, Expr * e) +{ + Value v; + return evalBool(env, e, v); +} + + +inline void EvalState::evalAttrs(Env & env, Expr * e, Value & v) +{ + e->eval(*this, env, v); + if (v.type != tAttrs) + throwTypeError("value is %1% while a set was expected", v); +} + + +void Expr::eval(EvalState & state, Env & env, Value & v) +{ + abort(); +} + + +void ExprInt::eval(EvalState & state, Env & env, Value & v) +{ + v = this->v; +} + + +void ExprString::eval(EvalState & state, Env & env, Value & v) +{ + v = this->v; +} + + +void ExprPath::eval(EvalState & state, Env & env, Value & v) +{ + v = this->v; +} + + +void ExprAttrs::eval(EvalState & state, Env & env, Value & v) +{ + state.mkAttrs(v, attrs.size()); + Env *dynamicEnv = &env; + + if (recursive) { + /* Create a new environment that contains the attributes in + this `rec'. */ + Env & env2(state.allocEnv(attrs.size())); + env2.up = &env; + dynamicEnv = &env2; + + AttrDefs::iterator overrides = attrs.find(state.sOverrides); + bool hasOverrides = overrides != attrs.end(); + + /* The recursive attributes are evaluated in the new + environment, while the inherited attributes are evaluated + in the original environment. */ + unsigned int displ = 0; + foreach (AttrDefs::iterator, i, attrs) { + Value * vAttr; + if (hasOverrides && !i->second.inherited) { + vAttr = state.allocValue(); + mkThunk(*vAttr, env2, i->second.e); + } else + vAttr = i->second.e->maybeThunk(state, i->second.inherited ? env : env2); + env2.values[displ++] = vAttr; + v.attrs->push_back(Attr(i->first, vAttr, &i->second.pos)); + } + + /* If the rec contains an attribute called `__overrides', then + evaluate it, and add the attributes in that set to the rec. + This allows overriding of recursive attributes, which is + otherwise not possible. (You can use the // operator to + replace an attribute, but other attributes in the rec will + still reference the original value, because that value has + been substituted into the bodies of the other attributes. + Hence we need __overrides.) */ + if (hasOverrides) { + Value * vOverrides = (*v.attrs)[overrides->second.displ].value; + state.forceAttrs(*vOverrides); + foreach (Bindings::iterator, i, *vOverrides->attrs) { + AttrDefs::iterator j = attrs.find(i->name); + if (j != attrs.end()) { + (*v.attrs)[j->second.displ] = *i; + env2.values[j->second.displ] = i->value; + } else + v.attrs->push_back(*i); + } + v.attrs->sort(); + } + } + + else + foreach (AttrDefs::iterator, i, attrs) + v.attrs->push_back(Attr(i->first, i->second.e->maybeThunk(state, env), &i->second.pos)); + + /* dynamic attrs apply *after* rec and __overrides */ + foreach (DynamicAttrDefs::iterator, i, dynamicAttrs) { + Value nameVal; + i->nameExpr->eval(state, *dynamicEnv, nameVal); + state.forceStringNoCtx(nameVal); + Symbol nameSym = state.symbols.create(nameVal.string.s); + Bindings::iterator j = v.attrs->find(nameSym); + if (j != v.attrs->end()) + throwEvalError("dynamic attribute `%1%' at %2% already defined at %3%", nameSym, i->pos, *j->pos); + + i->valueExpr->setName(nameSym); + /* Keep sorted order so find can catch duplicates */ + v.attrs->insert(lower_bound(v.attrs->begin(), v.attrs->end(), Attr(nameSym, 0)), + Attr(nameSym, i->valueExpr->maybeThunk(state, *dynamicEnv), &i->pos)); + } +} + + +void ExprLet::eval(EvalState & state, Env & env, Value & v) +{ + /* Create a new environment that contains the attributes in this + `let'. */ + Env & env2(state.allocEnv(attrs->attrs.size())); + env2.up = &env; + + /* The recursive attributes are evaluated in the new environment, + while the inherited attributes are evaluated in the original + environment. */ + unsigned int displ = 0; + foreach (ExprAttrs::AttrDefs::iterator, i, attrs->attrs) + env2.values[displ++] = i->second.e->maybeThunk(state, i->second.inherited ? env : env2); + + body->eval(state, env2, v); +} + + +void ExprList::eval(EvalState & state, Env & env, Value & v) +{ + state.mkList(v, elems.size()); + for (unsigned int n = 0; n < v.list.length; ++n) + v.list.elems[n] = elems[n]->maybeThunk(state, env); +} + + +void ExprVar::eval(EvalState & state, Env & env, Value & v) +{ + Value * v2 = state.lookupVar(&env, *this, false); + state.forceValue(*v2); + v = *v2; +} + + +unsigned long nrLookups = 0; + +void ExprSelect::eval(EvalState & state, Env & env, Value & v) +{ + Value vTmp; + Pos * pos = 0; + Value * vAttrs = &vTmp; + + e->eval(state, env, vTmp); + + try { + + foreach (AttrPath::const_iterator, i, attrPath) { + nrLookups++; + Bindings::iterator j; + Symbol name = getName(*i, state, env); + if (def) { + state.forceValue(*vAttrs); + if (vAttrs->type != tAttrs || + (j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) + { + def->eval(state, env, v); + return; + } + } else { + state.forceAttrs(*vAttrs); + if ((j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) + throwEvalError("attribute `%1%' missing", showAttrPath(attrPath)); + } + vAttrs = j->value; + pos = j->pos; + if (state.countCalls && pos) state.attrSelects[*pos]++; + } + + state.forceValue(*vAttrs); + + } catch (Error & e) { + if (pos && pos->file != state.sDerivationNix) + addErrorPrefix(e, "while evaluating the attribute `%1%' at %2%:\n", + showAttrPath(attrPath), *pos); + throw; + } + + v = *vAttrs; +} + + +void ExprOpHasAttr::eval(EvalState & state, Env & env, Value & v) +{ + Value vTmp; + Value * vAttrs = &vTmp; + + e->eval(state, env, vTmp); + + foreach (AttrPath::const_iterator, i, attrPath) { + state.forceValue(*vAttrs); + Bindings::iterator j; + Symbol name = getName(*i, state, env); + if (vAttrs->type != tAttrs || + (j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) + { + mkBool(v, false); + return; + } else { + vAttrs = j->value; + } + } + + mkBool(v, true); +} + + +void ExprLambda::eval(EvalState & state, Env & env, Value & v) +{ + v.type = tLambda; + v.lambda.env = &env; + v.lambda.fun = this; +} + + +void ExprApp::eval(EvalState & state, Env & env, Value & v) +{ + e1->eval(state, env, v); + state.callFunction(v, *(e2->maybeThunk(state, env)), v); +} + + +void EvalState::callPrimOp(Value & fun, Value & arg, Value & v) +{ + /* Figure out the number of arguments still needed. */ + unsigned int argsDone = 0; + Value * primOp = &fun; + while (primOp->type == tPrimOpApp) { + argsDone++; + primOp = primOp->primOpApp.left; + } + assert(primOp->type == tPrimOp); + unsigned int arity = primOp->primOp->arity; + unsigned int argsLeft = arity - argsDone; + + if (argsLeft == 1) { + /* We have all the arguments, so call the primop. */ + + /* Put all the arguments in an array. */ + Value * vArgs[arity]; + unsigned int n = arity - 1; + vArgs[n--] = &arg; + for (Value * arg = &fun; arg->type == tPrimOpApp; arg = arg->primOpApp.left) + vArgs[n--] = arg->primOpApp.right; + + /* And call the primop. */ + nrPrimOpCalls++; + if (countCalls) primOpCalls[primOp->primOp->name]++; + primOp->primOp->fun(*this, vArgs, v); + } else { + Value * fun2 = allocValue(); + *fun2 = fun; + v.type = tPrimOpApp; + v.primOpApp.left = fun2; + v.primOpApp.right = &arg; + } +} + + +void EvalState::callFunction(Value & fun, Value & arg, Value & v) +{ + if (fun.type == tPrimOp || fun.type == tPrimOpApp) { + callPrimOp(fun, arg, v); + return; + } + + if (fun.type != tLambda) + throwTypeError("attempt to call something which is not a function but %1%", fun); + + ExprLambda & lambda(*fun.lambda.fun); + + unsigned int size = + (lambda.arg.empty() ? 0 : 1) + + (lambda.matchAttrs ? lambda.formals->formals.size() : 0); + Env & env2(allocEnv(size)); + env2.up = fun.lambda.env; + + unsigned int displ = 0; + + if (!lambda.matchAttrs) + env2.values[displ++] = &arg; + + else { + forceAttrs(arg); + + if (!lambda.arg.empty()) + env2.values[displ++] = &arg; + + /* For each formal argument, get the actual argument. If + there is no matching actual argument but the formal + argument has a default, use the default. */ + unsigned int attrsUsed = 0; + foreach (Formals::Formals_::iterator, i, lambda.formals->formals) { + Bindings::iterator j = arg.attrs->find(i->name); + if (j == arg.attrs->end()) { + if (!i->def) throwTypeError("%1% called without required argument `%2%'", + lambda, i->name); + env2.values[displ++] = i->def->maybeThunk(*this, env2); + } else { + attrsUsed++; + env2.values[displ++] = j->value; + } + } + + /* Check that each actual argument is listed as a formal + argument (unless the attribute match specifies a `...'). */ + if (!lambda.formals->ellipsis && attrsUsed != arg.attrs->size()) { + /* Nope, so show the first unexpected argument to the + user. */ + foreach (Bindings::iterator, i, *arg.attrs) + if (lambda.formals->argNames.find(i->name) == lambda.formals->argNames.end()) + throwTypeError("%1% called with unexpected argument `%2%'", lambda, i->name); + abort(); // can't happen + } + } + + nrFunctionCalls++; + if (countCalls) incrFunctionCall(&lambda); + + /* Evaluate the body. This is conditional on showTrace, because + catching exceptions makes this function not tail-recursive. */ + if (settings.showTrace) + try { + lambda.body->eval(*this, env2, v); + } catch (Error & e) { + addErrorPrefix(e, "while evaluating %1%:\n", lambda); + throw; + } + else + fun.lambda.fun->body->eval(*this, env2, v); +} + + +// Lifted out of callFunction() because it creates a temporary that +// prevents tail-call optimisation. +void EvalState::incrFunctionCall(ExprLambda * fun) +{ + functionCalls[fun]++; +} + + +void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) +{ + forceValue(fun); + + if (fun.type != tLambda || !fun.lambda.fun->matchAttrs) { + res = fun; + return; + } + + Value * actualArgs = allocValue(); + mkAttrs(*actualArgs, fun.lambda.fun->formals->formals.size()); + + foreach (Formals::Formals_::iterator, i, fun.lambda.fun->formals->formals) { + Bindings::iterator j = args.find(i->name); + if (j != args.end()) + actualArgs->attrs->push_back(*j); + else if (!i->def) + throwTypeError("cannot auto-call a function that has an argument without a default value (`%1%')", i->name); + } + + actualArgs->attrs->sort(); + + callFunction(fun, *actualArgs, res); +} + + +void ExprWith::eval(EvalState & state, Env & env, Value & v) +{ + Env & env2(state.allocEnv(1)); + env2.up = &env; + env2.prevWith = prevWith; + env2.haveWithAttrs = false; + env2.values[0] = (Value *) attrs; + + body->eval(state, env2, v); +} + + +void ExprIf::eval(EvalState & state, Env & env, Value & v) +{ + (state.evalBool(env, cond, v) ? then : else_)->eval(state, env, v); +} + + +void ExprAssert::eval(EvalState & state, Env & env, Value & v) +{ + if (!state.evalBool(env, cond, v)) + throwAssertionError("assertion failed at %1%", pos); + body->eval(state, env, v); +} + + +void ExprOpNot::eval(EvalState & state, Env & env, Value & v) +{ + mkBool(v, !state.evalBool(env, e)); +} + + +void ExprBuiltin::eval(EvalState & state, Env & env, Value & v) +{ + // Not a hot path at all, but would be nice to access state.baseEnv directly + Env *baseEnv = &env; + while (baseEnv->up) baseEnv = baseEnv->up; + Bindings::iterator binding = baseEnv->values[0]->attrs->find(name); + assert(binding != baseEnv->values[0]->attrs->end()); + v = *binding->value; +} + + +void ExprOpEq::eval(EvalState & state, Env & env, Value & v) +{ + Value v1; e1->eval(state, env, v1); + Value v2; e2->eval(state, env, v2); + mkBool(v, state.eqValues(v1, v2)); +} + + +void ExprOpNEq::eval(EvalState & state, Env & env, Value & v) +{ + Value v1; e1->eval(state, env, v1); + Value v2; e2->eval(state, env, v2); + mkBool(v, !state.eqValues(v1, v2)); +} + + +void ExprOpAnd::eval(EvalState & state, Env & env, Value & v) +{ + mkBool(v, state.evalBool(env, e1) && state.evalBool(env, e2)); +} + + +void ExprOpOr::eval(EvalState & state, Env & env, Value & v) +{ + mkBool(v, state.evalBool(env, e1) || state.evalBool(env, e2)); +} + + +void ExprOpImpl::eval(EvalState & state, Env & env, Value & v) +{ + mkBool(v, !state.evalBool(env, e1) || state.evalBool(env, e2)); +} + + +void ExprOpUpdate::eval(EvalState & state, Env & env, Value & v) +{ + Value v1, v2; + state.evalAttrs(env, e1, v1); + state.evalAttrs(env, e2, v2); + + state.nrOpUpdates++; + + if (v1.attrs->size() == 0) { v = v2; return; } + if (v2.attrs->size() == 0) { v = v1; return; } + + state.mkAttrs(v, v1.attrs->size() + v2.attrs->size()); + + /* Merge the sets, preferring values from the second set. Make + sure to keep the resulting vector in sorted order. */ + Bindings::iterator i = v1.attrs->begin(); + Bindings::iterator j = v2.attrs->begin(); + + while (i != v1.attrs->end() && j != v2.attrs->end()) { + if (i->name == j->name) { + v.attrs->push_back(*j); + ++i; ++j; + } + else if (i->name < j->name) + v.attrs->push_back(*i++); + else + v.attrs->push_back(*j++); + } + + while (i != v1.attrs->end()) v.attrs->push_back(*i++); + while (j != v2.attrs->end()) v.attrs->push_back(*j++); + + state.nrOpUpdateValuesCopied += v.attrs->size(); +} + + +void ExprOpConcatLists::eval(EvalState & state, Env & env, Value & v) +{ + Value v1; e1->eval(state, env, v1); + Value v2; e2->eval(state, env, v2); + Value * lists[2] = { &v1, &v2 }; + state.concatLists(v, 2, lists); +} + + +void EvalState::concatLists(Value & v, unsigned int nrLists, Value * * lists) +{ + nrListConcats++; + + Value * nonEmpty = 0; + unsigned int len = 0; + for (unsigned int n = 0; n < nrLists; ++n) { + forceList(*lists[n]); + unsigned int l = lists[n]->list.length; + len += l; + if (l) nonEmpty = lists[n]; + } + + if (nonEmpty && len == nonEmpty->list.length) { + v = *nonEmpty; + return; + } + + mkList(v, len); + for (unsigned int n = 0, pos = 0; n < nrLists; ++n) { + unsigned int l = lists[n]->list.length; + memcpy(v.list.elems + pos, lists[n]->list.elems, l * sizeof(Value *)); + pos += l; + } +} + + +void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) +{ + PathSet context; + std::ostringstream s; + NixInt n = 0; + + bool first = !forceString; + ValueType firstType = tString; + + foreach (vector<Expr *>::iterator, i, *es) { + Value vTmp; + (*i)->eval(state, env, vTmp); + + /* If the first element is a path, then the result will also + be a path, we don't copy anything (yet - that's done later, + since paths are copied when they are used in a derivation), + and none of the strings are allowed to have contexts. */ + if (first) { + firstType = vTmp.type; + first = false; + } + + if (firstType == tInt) { + if (vTmp.type != tInt) + throwEvalError("cannot add %1% to an integer", showType(vTmp)); + n += vTmp.integer; + } else + s << state.coerceToString(vTmp, context, false, firstType == tString); + } + + if (firstType == tInt) + mkInt(v, n); + else if (firstType == tPath) { + if (!context.empty()) + throwEvalError("a string that refers to a store path cannot be appended to a path, in `%1%'", s.str()); + mkPath(v, s.str().c_str()); + } else + mkString(v, s.str(), context); +} + + +void ExprPos::eval(EvalState & state, Env & env, Value & v) +{ + state.mkPos(v, &pos); +} + + +void EvalState::strictForceValue(Value & v) +{ + forceValue(v); + + if (v.type == tAttrs) { + foreach (Bindings::iterator, i, *v.attrs) + strictForceValue(*i->value); + } + + else if (v.type == tList) { + for (unsigned int n = 0; n < v.list.length; ++n) + strictForceValue(*v.list.elems[n]); + } +} + + +NixInt EvalState::forceInt(Value & v) +{ + forceValue(v); + if (v.type != tInt) + throwTypeError("value is %1% while an integer was expected", v); + return v.integer; +} + + +bool EvalState::forceBool(Value & v) +{ + forceValue(v); + if (v.type != tBool) + throwTypeError("value is %1% while a Boolean was expected", v); + return v.boolean; +} + + +void EvalState::forceFunction(Value & v) +{ + forceValue(v); + if (v.type != tLambda && v.type != tPrimOp && v.type != tPrimOpApp) + throwTypeError("value is %1% while a function was expected", v); +} + + +string EvalState::forceString(Value & v) +{ + forceValue(v); + if (v.type != tString) + throwTypeError("value is %1% while a string was expected", v); + return string(v.string.s); +} + + +void copyContext(const Value & v, PathSet & context) +{ + if (v.string.context) + for (const char * * p = v.string.context; *p; ++p) + context.insert(*p); +} + + +string EvalState::forceString(Value & v, PathSet & context) +{ + string s = forceString(v); + copyContext(v, context); + return s; +} + + +string EvalState::forceStringNoCtx(Value & v) +{ + string s = forceString(v); + if (v.string.context) + throwEvalError("the string `%1%' is not allowed to refer to a store path (such as `%2%')", + v.string.s, v.string.context[0]); + return s; +} + + +bool EvalState::isDerivation(Value & v) +{ + if (v.type != tAttrs) return false; + Bindings::iterator i = v.attrs->find(sType); + if (i == v.attrs->end()) return false; + forceValue(*i->value); + if (i->value->type != tString) return false; + return strcmp(i->value->string.s, "derivation") == 0; +} + + +string EvalState::coerceToString(Value & v, PathSet & context, + bool coerceMore, bool copyToStore) +{ + forceValue(v); + + string s; + + if (v.type == tString) { + copyContext(v, context); + return v.string.s; + } + + if (v.type == tPath) { + Path path(canonPath(v.path)); + return copyToStore ? copyPathToStore(context, path) : path; + } + + if (v.type == tAttrs) { + Bindings::iterator i = v.attrs->find(sOutPath); + if (i == v.attrs->end()) throwTypeError("cannot coerce a set to a string"); + return coerceToString(*i->value, context, coerceMore, copyToStore); + } + + if (coerceMore) { + + /* Note that `false' is represented as an empty string for + shell scripting convenience, just like `null'. */ + if (v.type == tBool && v.boolean) return "1"; + if (v.type == tBool && !v.boolean) return ""; + if (v.type == tInt) return int2String(v.integer); + if (v.type == tNull) return ""; + + if (v.type == tList) { + string result; + for (unsigned int n = 0; n < v.list.length; ++n) { + result += coerceToString(*v.list.elems[n], + context, coerceMore, copyToStore); + if (n < v.list.length - 1 + /* !!! not quite correct */ + && (v.list.elems[n]->type != tList || v.list.elems[n]->list.length != 0)) + result += " "; + } + return result; + } + } + + throwTypeError("cannot coerce %1% to a string", v); +} + + +string EvalState::copyPathToStore(PathSet & context, const Path & path) +{ + if (nix::isDerivation(path)) + throwEvalError("file names are not allowed to end in `%1%'", drvExtension); + + Path dstPath; + if (srcToStore[path] != "") + dstPath = srcToStore[path]; + else { + dstPath = settings.readOnlyMode + ? computeStorePathForPath(path).first + : store->addToStore(path, true, htSHA256, defaultPathFilter, repair); + srcToStore[path] = dstPath; + printMsg(lvlChatty, format("copied source `%1%' -> `%2%'") + % path % dstPath); + } + + context.insert(dstPath); + return dstPath; +} + + +Path EvalState::coerceToPath(Value & v, PathSet & context) +{ + string path = coerceToString(v, context, false, false); + if (path == "" || path[0] != '/') + throwEvalError("string `%1%' doesn't represent an absolute path", path); + return path; +} + + +bool EvalState::eqValues(Value & v1, Value & v2) +{ + forceValue(v1); + forceValue(v2); + + /* !!! Hack to support some old broken code that relies on pointer + equality tests between sets. (Specifically, builderDefs calls + uniqList on a list of sets.) Will remove this eventually. */ + if (&v1 == &v2) return true; + + if (v1.type != v2.type) return false; + + switch (v1.type) { + + case tInt: + return v1.integer == v2.integer; + + case tBool: + return v1.boolean == v2.boolean; + + case tString: { + /* Compare both the string and its context. */ + if (strcmp(v1.string.s, v2.string.s) != 0) return false; + const char * * p = v1.string.context, * * q = v2.string.context; + if (!p && !q) return true; + if (!p || !q) return false; + for ( ; *p && *q; ++p, ++q) + if (strcmp(*p, *q) != 0) return false; + if (*p || *q) return false; + return true; + } + + case tPath: + return strcmp(v1.path, v2.path) == 0; + + case tNull: + return true; + + case tList: + if (v1.list.length != v2.list.length) return false; + for (unsigned int n = 0; n < v1.list.length; ++n) + if (!eqValues(*v1.list.elems[n], *v2.list.elems[n])) return false; + return true; + + case tAttrs: { + /* If both sets denote a derivation (type = "derivation"), + then compare their outPaths. */ + if (isDerivation(v1) && isDerivation(v2)) { + Bindings::iterator i = v1.attrs->find(sOutPath); + Bindings::iterator j = v2.attrs->find(sOutPath); + if (i != v1.attrs->end() && j != v2.attrs->end()) + return eqValues(*i->value, *j->value); + } + + if (v1.attrs->size() != v2.attrs->size()) return false; + + /* Otherwise, compare the attributes one by one. */ + Bindings::iterator i, j; + for (i = v1.attrs->begin(), j = v2.attrs->begin(); i != v1.attrs->end(); ++i, ++j) + if (i->name != j->name || !eqValues(*i->value, *j->value)) + return false; + + return true; + } + + /* Functions are incomparable. */ + case tLambda: + case tPrimOp: + case tPrimOpApp: + return false; + + default: + throwEvalError("cannot compare %1% with %2%", showType(v1), showType(v2)); + } +} + + +void EvalState::printStats() +{ + bool showStats = getEnv("NIX_SHOW_STATS", "0") != "0"; + Verbosity v = showStats ? lvlInfo : lvlDebug; + printMsg(v, "evaluation statistics:"); + + struct rusage buf; + getrusage(RUSAGE_SELF, &buf); + float cpuTime = buf.ru_utime.tv_sec + ((float) buf.ru_utime.tv_usec / 1000000); + + printMsg(v, format(" time elapsed: %1%") % cpuTime); + printMsg(v, format(" size of a value: %1%") % sizeof(Value)); + printMsg(v, format(" environments allocated: %1% (%2% bytes)") + % nrEnvs % (nrEnvs * sizeof(Env) + nrValuesInEnvs * sizeof(Value *))); + printMsg(v, format(" list elements: %1% (%2% bytes)") + % nrListElems % (nrListElems * sizeof(Value *))); + printMsg(v, format(" list concatenations: %1%") % nrListConcats); + printMsg(v, format(" values allocated: %1% (%2% bytes)") + % nrValues % (nrValues * sizeof(Value))); + printMsg(v, format(" sets allocated: %1%") % nrAttrsets); + printMsg(v, format(" right-biased unions: %1%") % nrOpUpdates); + printMsg(v, format(" values copied in right-biased unions: %1%") % nrOpUpdateValuesCopied); + printMsg(v, format(" symbols in symbol table: %1%") % symbols.size()); + printMsg(v, format(" size of symbol table: %1%") % symbols.totalSize()); + printMsg(v, format(" number of thunks: %1%") % nrThunks); + printMsg(v, format(" number of thunks avoided: %1%") % nrAvoided); + printMsg(v, format(" number of attr lookups: %1%") % nrLookups); + printMsg(v, format(" number of primop calls: %1%") % nrPrimOpCalls); + printMsg(v, format(" number of function calls: %1%") % nrFunctionCalls); + + if (countCalls) { + v = lvlInfo; + + printMsg(v, format("calls to %1% primops:") % primOpCalls.size()); + typedef std::multimap<unsigned int, Symbol> PrimOpCalls_; + PrimOpCalls_ primOpCalls_; + foreach (PrimOpCalls::iterator, i, primOpCalls) + primOpCalls_.insert(std::pair<unsigned int, Symbol>(i->second, i->first)); + foreach_reverse (PrimOpCalls_::reverse_iterator, i, primOpCalls_) + printMsg(v, format("%1$10d %2%") % i->first % i->second); + + printMsg(v, format("calls to %1% functions:") % functionCalls.size()); + typedef std::multimap<unsigned int, ExprLambda *> FunctionCalls_; + FunctionCalls_ functionCalls_; + foreach (FunctionCalls::iterator, i, functionCalls) + functionCalls_.insert(std::pair<unsigned int, ExprLambda *>(i->second, i->first)); + foreach_reverse (FunctionCalls_::reverse_iterator, i, functionCalls_) + printMsg(v, format("%1$10d %2%") % i->first % i->second->showNamePos()); + + printMsg(v, format("evaluations of %1% attributes:") % attrSelects.size()); + typedef std::multimap<unsigned int, Pos> AttrSelects_; + AttrSelects_ attrSelects_; + foreach (AttrSelects::iterator, i, attrSelects) + attrSelects_.insert(std::pair<unsigned int, Pos>(i->second, i->first)); + foreach_reverse (AttrSelects_::reverse_iterator, i, attrSelects_) + printMsg(v, format("%1$10d %2%") % i->first % i->second); + + } +} + + +} diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh new file mode 100644 index 000000000000..5ae5a1e3cf24 --- /dev/null +++ b/src/libexpr/eval.hh @@ -0,0 +1,294 @@ +#pragma once + +#include "value.hh" +#include "nixexpr.hh" +#include "symbol-table.hh" +#include "hash.hh" + +#include <map> + +#if HAVE_BOEHMGC +#include <gc/gc_allocator.h> +#endif + + +namespace nix { + + +class EvalState; +struct Attr; + + +/* Sets are represented as a vector of attributes, sorted by symbol + (i.e. pointer to the attribute name in the symbol table). */ +#if HAVE_BOEHMGC +typedef std::vector<Attr, gc_allocator<Attr> > BindingsBase; +#else +typedef std::vector<Attr> BindingsBase; +#endif + + +class Bindings : public BindingsBase +{ +public: + iterator find(const Symbol & name); + void sort(); +}; + + +typedef void (* PrimOpFun) (EvalState & state, Value * * args, Value & v); + + +struct PrimOp +{ + PrimOpFun fun; + unsigned int arity; + Symbol name; + PrimOp(PrimOpFun fun, unsigned int arity, Symbol name) + : fun(fun), arity(arity), name(name) { } +}; + + +struct Env +{ + Env * up; + unsigned short prevWith; // nr of levels up to next `with' environment + bool haveWithAttrs; + Value * values[0]; +}; + + +struct Attr +{ + Symbol name; + Value * value; + Pos * pos; + Attr(Symbol name, Value * value, Pos * pos = &noPos) + : name(name), value(value), pos(pos) { }; + Attr() : pos(&noPos) { }; + bool operator < (const Attr & a) const + { + return name < a.name; + } +}; + + +void mkString(Value & v, const string & s, const PathSet & context = PathSet()); + +void copyContext(const Value & v, PathSet & context); + + +/* Cache for calls to addToStore(); maps source paths to the store + paths. */ +typedef std::map<Path, Path> SrcToStore; + + +std::ostream & operator << (std::ostream & str, const Value & v); + + +class EvalState +{ +public: + SymbolTable symbols; + + const Symbol sWith, sOutPath, sDrvPath, sType, sMeta, sName, sValue, + sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls, + sFile, sLine, sColumn; + Symbol sDerivationNix; + + /* If set, force copying files to the Nix store even if they + already exist there. */ + bool repair; + +private: + SrcToStore srcToStore; + + /* A cache from path names to values. */ +#if HAVE_BOEHMGC + typedef std::map<Path, Value, std::less<Path>, gc_allocator<std::pair<const Path, Value> > > FileEvalCache; +#else + typedef std::map<Path, Value> FileEvalCache; +#endif + FileEvalCache fileEvalCache; + + typedef list<std::pair<string, Path> > SearchPath; + SearchPath searchPath; + SearchPath::iterator searchPathInsertionPoint; + +public: + + EvalState(); + ~EvalState(); + + void addToSearchPath(const string & s); + + /* Parse a Nix expression from the specified file. */ + Expr * parseExprFromFile(const Path & path); + + /* Parse a Nix expression from the specified string. */ + Expr * parseExprFromString(const string & s, const Path & basePath, StaticEnv & staticEnv); + Expr * parseExprFromString(const string & s, const Path & basePath); + + /* Evaluate an expression read from the given file to normal + form. */ + void evalFile(const Path & path, Value & v); + + void resetFileCache(); + + /* Look up a file in the search path. */ + Path findFile(const string & path); + + /* Evaluate an expression to normal form, storing the result in + value `v'. */ + void eval(Expr * e, Value & v); + + /* Evaluation the expression, then verify that it has the expected + type. */ + inline bool evalBool(Env & env, Expr * e, Value & v); + inline bool evalBool(Env & env, Expr * e); + inline void evalAttrs(Env & env, Expr * e, Value & v); + + /* If `v' is a thunk, enter it and overwrite `v' with the result + of the evaluation of the thunk. If `v' is a delayed function + application, call the function and overwrite `v' with the + result. Otherwise, this is a no-op. */ + inline void forceValue(Value & v); + + /* Force a value, then recursively force list elements and + attributes. */ + void strictForceValue(Value & v); + + /* Force `v', and then verify that it has the expected type. */ + NixInt forceInt(Value & v); + bool forceBool(Value & v); + inline void forceAttrs(Value & v); + inline void forceList(Value & v); + void forceFunction(Value & v); // either lambda or primop + string forceString(Value & v); + string forceString(Value & v, PathSet & context); + string forceStringNoCtx(Value & v); + + /* Return true iff the value `v' denotes a derivation (i.e. a + set with attribute `type = "derivation"'). */ + bool isDerivation(Value & v); + + /* String coercion. Converts strings, paths and derivations to a + string. If `coerceMore' is set, also converts nulls, integers, + booleans and lists to a string. If `copyToStore' is set, + referenced paths are copied to the Nix store as a side effect. */ + string coerceToString(Value & v, PathSet & context, + bool coerceMore = false, bool copyToStore = true); + + string copyPathToStore(PathSet & context, const Path & path); + + /* Path coercion. Converts strings, paths and derivations to a + path. The result is guaranteed to be a canonicalised, absolute + path. Nothing is copied to the store. */ + Path coerceToPath(Value & v, PathSet & context); + +public: + + /* The base environment, containing the builtin functions and + values. */ + Env & baseEnv; + + /* The same, but used during parsing to resolve variables. */ + StaticEnv staticBaseEnv; // !!! should be private + +private: + + unsigned int baseEnvDispl; + + void createBaseEnv(); + + void addConstant(const string & name, Value & v); + + void addPrimOp(const string & name, + unsigned int arity, PrimOpFun primOp); + +public: + + void getBuiltin(const string & name, Value & v); + +private: + + inline Value * lookupVar(Env * env, const ExprVar & var, bool noEval); + + friend struct ExprVar; + friend struct ExprAttrs; + friend struct ExprLet; + + Expr * parse(const char * text, const Path & path, + const Path & basePath, StaticEnv & staticEnv); + +public: + + /* Do a deep equality test between two values. That is, list + elements and attributes are compared recursively. */ + bool eqValues(Value & v1, Value & v2); + + void callFunction(Value & fun, Value & arg, Value & v); + void callPrimOp(Value & fun, Value & arg, Value & v); + + /* Automatically call a function for which each argument has a + default value or has a binding in the `args' map. */ + void autoCallFunction(Bindings & args, Value & fun, Value & res); + + /* Allocation primitives. */ + Value * allocValue(); + Env & allocEnv(unsigned int size); + + Value * allocAttr(Value & vAttrs, const Symbol & name); + + void mkList(Value & v, unsigned int length); + void mkAttrs(Value & v, unsigned int expected); + void mkThunk_(Value & v, Expr * expr); + void mkPos(Value & v, Pos * pos); + + void concatLists(Value & v, unsigned int nrLists, Value * * lists); + + /* Print statistics. */ + void printStats(); + +private: + + unsigned long nrEnvs; + unsigned long nrValuesInEnvs; + unsigned long nrValues; + unsigned long nrListElems; + unsigned long nrAttrsets; + unsigned long nrOpUpdates; + unsigned long nrOpUpdateValuesCopied; + unsigned long nrListConcats; + unsigned long nrPrimOpCalls; + unsigned long nrFunctionCalls; + + bool countCalls; + + typedef std::map<Symbol, unsigned int> PrimOpCalls; + PrimOpCalls primOpCalls; + + typedef std::map<ExprLambda *, unsigned int> FunctionCalls; + FunctionCalls functionCalls; + + void incrFunctionCall(ExprLambda * fun); + + typedef std::map<Pos, unsigned int> AttrSelects; + AttrSelects attrSelects; + + friend struct ExprOpUpdate; + friend struct ExprOpConcatLists; + friend struct ExprSelect; + friend void prim_getAttr(EvalState & state, Value * * args, Value & v); +}; + + +/* Return a string representing the type of the value `v'. */ +string showType(const Value & v); + + +/* If `path' refers to a directory, then append "/default.nix". */ +Path resolveExprPath(Path path); + + +} diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc new file mode 100644 index 000000000000..0ed644e9bc5b --- /dev/null +++ b/src/libexpr/get-drvs.cc @@ -0,0 +1,305 @@ +#include "get-drvs.hh" +#include "util.hh" +#include "eval-inline.hh" + +#include <cstring> + + +namespace nix { + + +string DrvInfo::queryDrvPath() +{ + if (drvPath == "" && attrs) { + Bindings::iterator i = attrs->find(state->sDrvPath); + PathSet context; + drvPath = i != attrs->end() ? state->coerceToPath(*i->value, context) : ""; + } + return drvPath; +} + + +string DrvInfo::queryOutPath() +{ + if (outPath == "" && attrs) { + Bindings::iterator i = attrs->find(state->sOutPath); + PathSet context; + outPath = i != attrs->end() ? state->coerceToPath(*i->value, context) : ""; + } + return outPath; +} + + +DrvInfo::Outputs DrvInfo::queryOutputs() +{ + if (outputs.empty()) { + /* Get the ‘outputs’ list. */ + Bindings::iterator i; + if (attrs && (i = attrs->find(state->sOutputs)) != attrs->end()) { + state->forceList(*i->value); + + /* For each output... */ + for (unsigned int j = 0; j < i->value->list.length; ++j) { + /* Evaluate the corresponding set. */ + string name = state->forceStringNoCtx(*i->value->list.elems[j]); + Bindings::iterator out = attrs->find(state->symbols.create(name)); + if (out == attrs->end()) continue; // FIXME: throw error? + state->forceAttrs(*out->value); + + /* And evaluate its ‘outPath’ attribute. */ + Bindings::iterator outPath = out->value->attrs->find(state->sOutPath); + if (outPath == out->value->attrs->end()) continue; // FIXME: throw error? + PathSet context; + outputs[name] = state->coerceToPath(*outPath->value, context); + } + } else + outputs["out"] = queryOutPath(); + } + return outputs; +} + + +string DrvInfo::queryOutputName() +{ + if (outputName == "" && attrs) { + Bindings::iterator i = attrs->find(state->sOutputName); + outputName = i != attrs->end() ? state->forceStringNoCtx(*i->value) : ""; + } + return outputName; +} + + +Bindings * DrvInfo::getMeta() +{ + if (meta) return meta; + if (!attrs) return 0; + Bindings::iterator a = attrs->find(state->sMeta); + if (a == attrs->end()) return 0; + state->forceAttrs(*a->value); + meta = a->value->attrs; + return meta; +} + + +StringSet DrvInfo::queryMetaNames() +{ + StringSet res; + if (!getMeta()) return res; + foreach (Bindings::iterator, i, *meta) + res.insert(i->name); + return res; +} + + +bool DrvInfo::checkMeta(Value & v) +{ + state->forceValue(v); + if (v.type == tList) { + for (unsigned int n = 0; n < v.list.length; ++n) + if (!checkMeta(*v.list.elems[n])) return false; + return true; + } + else if (v.type == tAttrs) { + Bindings::iterator i = v.attrs->find(state->sOutPath); + if (i != v.attrs->end()) return false; + foreach (Bindings::iterator, i, *v.attrs) + if (!checkMeta(*i->value)) return false; + return true; + } + else return v.type == tInt || v.type == tBool || v.type == tString; +} + + +Value * DrvInfo::queryMeta(const string & name) +{ + if (!getMeta()) return 0; + Bindings::iterator a = meta->find(state->symbols.create(name)); + if (a == meta->end() || !checkMeta(*a->value)) return 0; + return a->value; +} + + +string DrvInfo::queryMetaString(const string & name) +{ + Value * v = queryMeta(name); + if (!v || v->type != tString) return ""; + return v->string.s; +} + + +int DrvInfo::queryMetaInt(const string & name, int def) +{ + Value * v = queryMeta(name); + if (!v) return def; + if (v->type == tInt) return v->integer; + if (v->type == tString) { + /* Backwards compatibility with before we had support for + integer meta fields. */ + int n; + if (string2Int(v->string.s, n)) return n; + } + return def; +} + + +bool DrvInfo::queryMetaBool(const string & name, bool def) +{ + Value * v = queryMeta(name); + if (!v) return def; + if (v->type == tBool) return v->boolean; + if (v->type == tString) { + /* Backwards compatibility with before we had support for + Boolean meta fields. */ + if (strcmp(v->string.s, "true") == 0) return true; + if (strcmp(v->string.s, "false") == 0) return false; + } + return def; +} + + +void DrvInfo::setMeta(const string & name, Value * v) +{ + getMeta(); + Bindings * old = meta; + meta = new Bindings(); + Symbol sym = state->symbols.create(name); + if (old) + foreach (Bindings::iterator, i, *old) + if (i->name != sym) + meta->push_back(*i); + if (v) meta->push_back(Attr(sym, v)); + meta->sort(); +} + + +/* Cache for already considered attrsets. */ +typedef set<Bindings *> Done; + + +/* Evaluate value `v'. If it evaluates to a set of type `derivation', + then put information about it in `drvs' (unless it's already in + `doneExprs'). The result boolean indicates whether it makes sense + for the caller to recursively search for derivations in `v'. */ +static bool getDerivation(EvalState & state, Value & v, + const string & attrPath, DrvInfos & drvs, Done & done, + bool ignoreAssertionFailures) +{ + try { + state.forceValue(v); + if (!state.isDerivation(v)) return true; + + /* Remove spurious duplicates (e.g., a set like `rec { x = + derivation {...}; y = x;}'. */ + if (done.find(v.attrs) != done.end()) return false; + done.insert(v.attrs); + + Bindings::iterator i = v.attrs->find(state.sName); + /* !!! We really would like to have a decent back trace here. */ + if (i == v.attrs->end()) throw TypeError("derivation name missing"); + + Bindings::iterator i2 = v.attrs->find(state.sSystem); + + DrvInfo drv( + state, + state.forceStringNoCtx(*i->value), + attrPath, + i2 == v.attrs->end() ? "unknown" : state.forceStringNoCtx(*i2->value), + v.attrs); + + drvs.push_back(drv); + return false; + + } catch (AssertionError & e) { + if (ignoreAssertionFailures) return false; + throw; + } +} + + +bool getDerivation(EvalState & state, Value & v, DrvInfo & drv, + bool ignoreAssertionFailures) +{ + Done done; + DrvInfos drvs; + getDerivation(state, v, "", drvs, done, ignoreAssertionFailures); + if (drvs.size() != 1) return false; + drv = drvs.front(); + return true; +} + + +static string addToPath(const string & s1, const string & s2) +{ + return s1.empty() ? s2 : s1 + "." + s2; +} + + +static void getDerivations(EvalState & state, Value & vIn, + const string & pathPrefix, Bindings & autoArgs, + DrvInfos & drvs, Done & done, + bool ignoreAssertionFailures) +{ + Value v; + state.autoCallFunction(autoArgs, vIn, v); + + /* Process the expression. */ + if (!getDerivation(state, v, pathPrefix, drvs, done, ignoreAssertionFailures)) ; + + else if (v.type == tAttrs) { + + /* !!! undocumented hackery to support combining channels in + nix-env.cc. */ + bool combineChannels = v.attrs->find(state.symbols.create("_combineChannels")) != v.attrs->end(); + + /* Consider the attributes in sorted order to get more + deterministic behaviour in nix-env operations (e.g. when + there are names clashes between derivations, the derivation + bound to the attribute with the "lower" name should take + precedence). */ + typedef std::map<string, Symbol> SortedSymbols; + SortedSymbols attrs; + foreach (Bindings::iterator, i, *v.attrs) + attrs.insert(std::pair<string, Symbol>(i->name, i->name)); + + foreach (SortedSymbols::iterator, i, attrs) { + startNest(nest, lvlDebug, format("evaluating attribute `%1%'") % i->first); + string pathPrefix2 = addToPath(pathPrefix, i->first); + Value & v2(*v.attrs->find(i->second)->value); + if (combineChannels) + getDerivations(state, v2, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); + else if (getDerivation(state, v2, pathPrefix2, drvs, done, ignoreAssertionFailures)) { + /* If the value of this attribute is itself a set, + should we recurse into it? => Only if it has a + `recurseForDerivations = true' attribute. */ + if (v2.type == tAttrs) { + Bindings::iterator j = v2.attrs->find(state.symbols.create("recurseForDerivations")); + if (j != v2.attrs->end() && state.forceBool(*j->value)) + getDerivations(state, v2, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); + } + } + } + } + + else if (v.type == tList) { + for (unsigned int n = 0; n < v.list.length; ++n) { + startNest(nest, lvlDebug, + format("evaluating list element")); + string pathPrefix2 = addToPath(pathPrefix, (format("%1%") % n).str()); + if (getDerivation(state, *v.list.elems[n], pathPrefix2, drvs, done, ignoreAssertionFailures)) + getDerivations(state, *v.list.elems[n], pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); + } + } + + else throw TypeError("expression does not evaluate to a derivation (or a set or list of those)"); +} + + +void getDerivations(EvalState & state, Value & v, const string & pathPrefix, + Bindings & autoArgs, DrvInfos & drvs, bool ignoreAssertionFailures) +{ + Done done; + getDerivations(state, v, pathPrefix, autoArgs, drvs, done, ignoreAssertionFailures); +} + + +} diff --git a/src/libexpr/get-drvs.hh b/src/libexpr/get-drvs.hh new file mode 100644 index 000000000000..aaafff1905e7 --- /dev/null +++ b/src/libexpr/get-drvs.hh @@ -0,0 +1,93 @@ +#pragma once + +#include "eval.hh" + +#include <string> +#include <map> + +#include <boost/shared_ptr.hpp> + + +namespace nix { + + +struct DrvInfo +{ +public: + typedef std::map<string, Path> Outputs; + +private: + EvalState * state; + + string drvPath; + string outPath; + string outputName; + Outputs outputs; + + bool failed; // set if we get an AssertionError + + Bindings * attrs, * meta; + + Bindings * getMeta(); + + bool checkMeta(Value & v); + +public: + string name; + string attrPath; /* path towards the derivation */ + string system; + + DrvInfo(EvalState & state) : state(&state), failed(false), attrs(0), meta(0) { }; + DrvInfo(EvalState & state, const string & name, const string & attrPath, const string & system, Bindings * attrs) + : state(&state), failed(false), attrs(attrs), meta(0), name(name), attrPath(attrPath), system(system) { }; + + string queryDrvPath(); + string queryOutPath(); + string queryOutputName(); + Outputs queryOutputs(); + + StringSet queryMetaNames(); + Value * queryMeta(const string & name); + string queryMetaString(const string & name); + int queryMetaInt(const string & name, int def); + bool queryMetaBool(const string & name, bool def); + void setMeta(const string & name, Value * v); + + /* + MetaInfo queryMetaInfo(EvalState & state) const; + MetaValue queryMetaInfo(EvalState & state, const string & name) const; + */ + + void setDrvPath(const string & s) + { + drvPath = s; + } + + void setOutPath(const string & s) + { + outPath = s; + } + + void setFailed() { failed = true; }; + bool hasFailed() { return failed; }; +}; + + +#if HAVE_BOEHMGC +typedef list<DrvInfo, traceable_allocator<DrvInfo> > DrvInfos; +#else +typedef list<DrvInfo> DrvInfos; +#endif + + +/* If value `v' denotes a derivation, store information about the + derivation in `drv' and return true. Otherwise, return false. */ +bool getDerivation(EvalState & state, Value & v, DrvInfo & drv, + bool ignoreAssertionFailures); + +void getDerivations(EvalState & state, Value & v, const string & pathPrefix, + Bindings & autoArgs, DrvInfos & drvs, + bool ignoreAssertionFailures); + + +} diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l new file mode 100644 index 000000000000..911850cc5ba5 --- /dev/null +++ b/src/libexpr/lexer.l @@ -0,0 +1,193 @@ +%option reentrant bison-bridge bison-locations +%option noyywrap +%option never-interactive + + +%x STRING +%x IND_STRING + + +%{ +#include "nixexpr.hh" +#include "parser-tab.hh" + +using namespace nix; + +namespace nix { + + +static void initLoc(YYLTYPE * loc) +{ + loc->first_line = loc->last_line = 1; + loc->first_column = loc->last_column = 1; +} + + +static void adjustLoc(YYLTYPE * loc, const char * s, size_t len) +{ + loc->first_line = loc->last_line; + loc->first_column = loc->last_column; + + while (len--) { + switch (*s++) { + case '\r': + if (*s == '\n') /* cr/lf */ + s++; + /* fall through */ + case '\n': + ++loc->last_line; + loc->last_column = 1; + break; + default: + ++loc->last_column; + } + } +} + + +static Expr * unescapeStr(SymbolTable & symbols, const char * s) +{ + string t; + char c; + while ((c = *s++)) { + if (c == '\\') { + assert(*s); + c = *s++; + if (c == 'n') t += '\n'; + else if (c == 'r') t += '\r'; + else if (c == 't') t += '\t'; + else t += c; + } + else if (c == '\r') { + /* Normalise CR and CR/LF into LF. */ + t += '\n'; + if (*s == '\n') s++; /* cr/lf */ + } + else t += c; + } + return new ExprString(symbols.create(t)); +} + + +} + +#define YY_USER_INIT initLoc(yylloc) +#define YY_USER_ACTION adjustLoc(yylloc, yytext, yyleng); + +%} + + +ID [a-zA-Z\_][a-zA-Z0-9\_\'\-]* +INT [0-9]+ +PATH [a-zA-Z0-9\.\_\-\+]*(\/[a-zA-Z0-9\.\_\-\+]+)+ +SPATH \<[a-zA-Z0-9\.\_\-\+]+(\/[a-zA-Z0-9\.\_\-\+]+)*\> +URI [a-zA-Z][a-zA-Z0-9\+\-\.]*\:[a-zA-Z0-9\%\/\?\:\@\&\=\+\$\,\-\_\.\!\~\*\']+ + + +%% + + +if { return IF; } +then { return THEN; } +else { return ELSE; } +assert { return ASSERT; } +with { return WITH; } +let { return LET; } +in { return IN; } +rec { return REC; } +inherit { return INHERIT; } +or { return OR_KW; } +\.\.\. { return ELLIPSIS; } + +\=\= { return EQ; } +\!\= { return NEQ; } +\<\= { return LEQ; } +\>\= { return GEQ; } +\&\& { return AND; } +\|\| { return OR; } +\-\> { return IMPL; } +\/\/ { return UPDATE; } +\+\+ { return CONCAT; } + +{ID} { yylval->id = strdup(yytext); return ID; } +{INT} { errno = 0; + yylval->n = strtol(yytext, 0, 10); + if (errno != 0) + throw ParseError(format("invalid integer `%1%'") % yytext); + return INT; + } + +\$\{ { return DOLLAR_CURLY; } + +\" { BEGIN(STRING); return '"'; } +<STRING>([^\$\"\\]|\$[^\{\"]|\\.)+ { + /* !!! Not quite right: we want a follow restriction on + "$", it shouldn't be followed by a "{". Right now + "$\"" will be consumed as part of a string, rather + than a "$" followed by the string terminator. + Disallow "$\"" for now. */ + yylval->e = unescapeStr(data->symbols, yytext); + return STR; + } +<STRING>\$\{ { BEGIN(INITIAL); return DOLLAR_CURLY; } +<STRING>\" { BEGIN(INITIAL); return '"'; } +<STRING>. return yytext[0]; /* just in case: shouldn't be reached */ + +\'\'(\ *\n)? { BEGIN(IND_STRING); return IND_STRING_OPEN; } +<IND_STRING>([^\$\']|\$[^\{\']|\'[^\'\$])+ { + yylval->e = new ExprIndStr(yytext); + return IND_STR; + } +<IND_STRING>\'\'\$ { + yylval->e = new ExprIndStr("$"); + return IND_STR; + } +<IND_STRING>\'\'\' { + yylval->e = new ExprIndStr("''"); + return IND_STR; + } +<IND_STRING>\'\'\\. { + yylval->e = unescapeStr(data->symbols, yytext + 2); + return IND_STR; + } +<IND_STRING>\$\{ { BEGIN(INITIAL); return DOLLAR_CURLY; } +<IND_STRING>\'\' { BEGIN(INITIAL); return IND_STRING_CLOSE; } +<IND_STRING>\' { + yylval->e = new ExprIndStr("'"); + return IND_STR; + } +<IND_STRING>. return yytext[0]; /* just in case: shouldn't be reached */ + +{PATH} { yylval->path = strdup(yytext); return PATH; } +{SPATH} { yylval->path = strdup(yytext); return SPATH; } +{URI} { yylval->uri = strdup(yytext); return URI; } + +[ \t\r\n]+ /* eat up whitespace */ +\#[^\r\n]* /* single-line comments */ +\/\*([^*]|\*[^\/])*\*\/ /* long comments */ + +. return yytext[0]; + + +%% + + +namespace nix { + +/* Horrible, disgusting hack: allow the parser to set the scanner + start condition back to STRING. Necessary in interpolations like + "foo${expr}bar"; after the close brace we have to go back to the + STRING state. */ +void backToString(yyscan_t scanner) +{ + struct yyguts_t * yyg = (struct yyguts_t *) scanner; + BEGIN(STRING); +} + +void backToIndString(yyscan_t scanner) +{ + struct yyguts_t * yyg = (struct yyguts_t *) scanner; + BEGIN(IND_STRING); +} + +} diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk new file mode 100644 index 000000000000..431f84b0999e --- /dev/null +++ b/src/libexpr/local.mk @@ -0,0 +1,24 @@ +libraries += libexpr + +libexpr_NAME = libnixexpr + +libexpr_DIR := $(d) + +libexpr_SOURCES := $(wildcard $(d)/*.cc) $(d)/lexer-tab.cc $(d)/parser-tab.cc + +libexpr_LIBS = libutil libstore libformat + +# The dependency on libgc must be propagated (i.e. meaning that +# programs/libraries that use libexpr must explicitly pass -lgc), +# because inline functions in libexpr's header files call libgc. +libexpr_LDFLAGS_PROPAGATED = $(BDW_GC_LIBS) + +$(d)/parser-tab.cc $(d)/parser-tab.hh: $(d)/parser.y + $(trace-gen) bison -v -o $(libexpr_DIR)/parser-tab.cc $< -d + +$(d)/lexer-tab.cc $(d)/lexer-tab.hh: $(d)/lexer.l + $(trace-gen) flex --outfile $(libexpr_DIR)/lexer-tab.cc --header-file=$(libexpr_DIR)/lexer-tab.hh $< + +clean-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer-tab.hh + +dist-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer-tab.hh diff --git a/src/libexpr/names.cc b/src/libexpr/names.cc new file mode 100644 index 000000000000..781c2b6468f9 --- /dev/null +++ b/src/libexpr/names.cc @@ -0,0 +1,104 @@ +#include "names.hh" +#include "util.hh" + + +namespace nix { + + +DrvName::DrvName() +{ + name = ""; +} + + +/* Parse a derivation name. The `name' part of a derivation name is + everything up to but not including the first dash *not* followed by + a letter. The `version' part is the rest (excluding the separating + dash). E.g., `apache-httpd-2.0.48' is parsed to (`apache-httpd', + '2.0.48'). */ +DrvName::DrvName(const string & s) : hits(0) +{ + name = fullName = s; + for (unsigned int i = 0; i < s.size(); ++i) { + /* !!! isalpha/isdigit are affected by the locale. */ + if (s[i] == '-' && i + 1 < s.size() && !isalpha(s[i + 1])) { + name = string(s, 0, i); + version = string(s, i + 1); + break; + } + } +} + + +bool DrvName::matches(DrvName & n) +{ + if (name != "*" && name != n.name) return false; + if (version != "" && version != n.version) return false; + return true; +} + + +static string nextComponent(string::const_iterator & p, + const string::const_iterator end) +{ + /* Skip any dots and dashes (component separators). */ + while (p != end && (*p == '.' || *p == '-')) ++p; + + if (p == end) return ""; + + /* If the first character is a digit, consume the longest sequence + of digits. Otherwise, consume the longest sequence of + non-digit, non-separator characters. */ + string s; + if (isdigit(*p)) + while (p != end && isdigit(*p)) s += *p++; + else + while (p != end && (!isdigit(*p) && *p != '.' && *p != '-')) + s += *p++; + + return s; +} + + +static bool componentsLT(const string & c1, const string & c2) +{ + int n1, n2; + bool c1Num = string2Int(c1, n1), c2Num = string2Int(c2, n2); + + if (c1Num && c2Num) return n1 < n2; + else if (c1 == "" && c2Num) return true; + else if (c1 == "pre" && c2 != "pre") return true; + else if (c2 == "pre") return false; + /* Assume that `2.3a' < `2.3.1'. */ + else if (c2Num) return true; + else if (c1Num) return false; + else return c1 < c2; +} + + +int compareVersions(const string & v1, const string & v2) +{ + string::const_iterator p1 = v1.begin(); + string::const_iterator p2 = v2.begin(); + + while (p1 != v1.end() || p2 != v2.end()) { + string c1 = nextComponent(p1, v1.end()); + string c2 = nextComponent(p2, v2.end()); + if (componentsLT(c1, c2)) return -1; + else if (componentsLT(c2, c1)) return 1; + } + + return 0; +} + + +DrvNames drvNamesFromArgs(const Strings & opArgs) +{ + DrvNames result; + foreach (Strings::const_iterator, i, opArgs) + result.push_back(DrvName(*i)); + return result; +} + + +} diff --git a/src/libexpr/names.hh b/src/libexpr/names.hh new file mode 100644 index 000000000000..ebe113e82ac1 --- /dev/null +++ b/src/libexpr/names.hh @@ -0,0 +1,24 @@ +#pragma once + +#include "types.hh" + +namespace nix { + +struct DrvName +{ + string fullName; + string name; + string version; + unsigned int hits; + + DrvName(); + DrvName(const string & s); + bool matches(DrvName & n); +}; + +typedef list<DrvName> DrvNames; + +int compareVersions(const string & v1, const string & v2); +DrvNames drvNamesFromArgs(const Strings & opArgs); + +} diff --git a/src/libexpr/nix.sdf b/src/libexpr/nix.sdf new file mode 100644 index 000000000000..42fb21c3baad --- /dev/null +++ b/src/libexpr/nix.sdf @@ -0,0 +1,141 @@ +%% Note: this SDF grammar is no longer used in the Nix expression +%% parser and may not be up to date. + +definition + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% Top level syntax. + +module Main +imports Nix-Exprs Nix-Layout + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% Expressions. + +module Nix-Exprs +imports Nix-Lexicals +exports + sorts Expr Formal Bind ExprList + context-free start-symbols Expr + context-free syntax + + Id -> Expr {cons("Var")} + Int -> Expr {cons("Int")} + Str -> Expr {cons("Str")} + Uri -> Expr {cons("Uri")} + Path -> Expr {cons("Path")} + + "(" Expr ")" -> Expr {bracket} + + Expr Expr -> Expr {cons("Call"), left} + + Id ":" Expr -> Expr {cons("Function1")} + "{" {Formal ","}* "}" ":" Expr -> Expr {cons("Function")} + Id -> Formal {cons("NoDefFormal")} + Id "?" Expr -> Formal {cons("DefFormal")} + + "assert" Expr ";" Expr -> Expr {cons("Assert")} + + "with" Expr ";" Expr -> Expr {cons("With")} + + "rec" "{" Bind* "}" -> Expr {cons("Rec")} + "let" Bind* "in" Expr -> Expr {cons("Let")} + "let" "{" Bind* "}" -> Expr {cons("LetRec")} + "{" Bind* "}" -> Expr {cons("Attrs")} + + Id "=" Expr ";" -> Bind {cons("Bind")} + "inherit" ("(" Expr ")")? Id* ";" -> Bind {cons("Inherit")} + + "[" ExprList "]" -> Expr {cons("List")} + -> ExprList {cons("ExprNil")} + Expr ExprList -> ExprList {cons("ExprCons")} + + Expr "." Id -> Expr {cons("Select")} + + "if" Expr "then" Expr "else" Expr -> Expr {cons("If")} + + Expr "==" Expr -> Expr {cons("OpEq"), non-assoc} + Expr "!=" Expr -> Expr {cons("OpNEq"), non-assoc} + + "!" Expr -> Expr {cons("OpNot")} + Expr "&&" Expr -> Expr {cons("OpAnd"), right} + Expr "||" Expr -> Expr {cons("OpOr"), right} + Expr "->" Expr -> Expr {cons("OpImpl"), right} + + Expr "//" Expr -> Expr {cons("OpUpdate"), right} + Expr "~" Expr -> Expr {cons("SubPath"), non-assoc} + Expr "?" Id -> Expr {cons("OpHasAttr")} + Expr "+" Expr -> Expr {cons("OpPlus"), left} + Expr "++" Expr -> Expr {cons("OpConcat"), right} + + context-free priorities + + Expr "." Id -> Expr + > Expr ExprList -> ExprList + > Expr Expr -> Expr + > Expr "~" Expr -> Expr + > Expr "?" Id -> Expr + > Expr "++" Expr -> Expr + > Expr "+" Expr -> Expr + > "!" Expr -> Expr + > Expr "//" Expr -> Expr + > { Expr "==" Expr -> Expr + Expr "!=" Expr -> Expr + } + > Expr "&&" Expr -> Expr + > Expr "||" Expr -> Expr + > Expr "->" Expr -> Expr + > "if" Expr "then" Expr "else" Expr -> Expr + > "assert" Expr ";" Expr -> Expr + > "with" Expr ";" Expr -> Expr + > Id ":" Expr -> Expr + > "{" {Formal ","}* "}" ":" Expr -> Expr + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% Lexical syntax. + +module Nix-Lexicals +exports + sorts Id Int Str Path Uri + lexical syntax + [a-zA-Z\_][a-zA-Z0-9\_\']* -> Id + "rec" | "let" | "if" | "then" | "else" | "assert" | "with" | "inherit" -> Id {reject} + + [0-9]+ -> Int + + "\"" (~[\"\\] | ("\\" ~[]) )* "\"" -> Str + "''" (~[\"\\] | ("\\" ~[]) )* "''" -> Str + + [a-zA-Z0-9\.\_\-\+]* ("/"[a-zA-Z0-9\.\_\-\+]+)+ -> Path + + [a-zA-Z] [a-zA-Z0-9\+\-\.]* ":" [a-zA-Z0-9\%\/\?\:\@\&\=\+\$\,\-\_\.\!\~\*\']+ -> Uri + + lexical restrictions + Id -/- [a-zA-Z0-9\_\'] + Int -/- [0-9] + Path -/- [a-zA-Z0-9\.\_\-\+\/] + Uri -/- [a-zA-Z0-9\%\/\?\:\@\&\=\+\$\,\-\_\.\!\~\*\'] + "rec" "let" "if" "then" "else" "assert" "with" "inherit" -/- [A-Za-z0-9\_\'] + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% Layout. + +module Nix-Layout +exports + sorts HashComment Asterisk Comment + lexical syntax + [\ \t\n] -> LAYOUT + HashComment -> LAYOUT + Comment -> LAYOUT + "#" ~[\n]* -> HashComment + "/*" ( ~[\*] | Asterisk )* "*/" -> Comment + [\*] ~[\/] -> Asterisk + lexical restrictions + HashComment -/- ~[\n] + context-free restrictions + LAYOUT? -/- [\ \t\n\#] + LAYOUT? -/- [\/].[\*] diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc new file mode 100644 index 000000000000..9f0bc2630ddc --- /dev/null +++ b/src/libexpr/nixexpr.cc @@ -0,0 +1,391 @@ +#include "nixexpr.hh" +#include "derivations.hh" +#include "util.hh" + +#include <cstdlib> + + +namespace nix { + + +/* Displaying abstract syntax trees. */ + +std::ostream & operator << (std::ostream & str, Expr & e) +{ + e.show(str); + return str; +} + +void Expr::show(std::ostream & str) +{ + abort(); +} + +void ExprInt::show(std::ostream & str) +{ + str << n; +} + +void ExprString::show(std::ostream & str) +{ + str << "\"" << s << "\""; // !!! escaping +} + +void ExprPath::show(std::ostream & str) +{ + str << s; +} + +void ExprVar::show(std::ostream & str) +{ + str << name; +} + +void ExprSelect::show(std::ostream & str) +{ + str << "(" << *e << ")." << showAttrPath(attrPath); + if (def) str << " or " << *def; +} + +void ExprOpHasAttr::show(std::ostream & str) +{ + str << "(" << *e << ") ? " << showAttrPath(attrPath); +} + +void ExprAttrs::show(std::ostream & str) +{ + if (recursive) str << "rec "; + str << "{ "; + foreach (AttrDefs::iterator, i, attrs) + if (i->second.inherited) + str << "inherit " << i->first << " " << "; "; + else + str << i->first << " = " << *i->second.e << "; "; + foreach (DynamicAttrDefs::iterator, i, dynamicAttrs) + str << "\"${" << *i->nameExpr << "}\" = " << *i->valueExpr << "; "; + str << "}"; +} + +void ExprList::show(std::ostream & str) +{ + str << "[ "; + foreach (vector<Expr *>::iterator, i, elems) + str << "(" << **i << ") "; + str << "]"; +} + +void ExprLambda::show(std::ostream & str) +{ + str << "("; + if (matchAttrs) { + str << "{ "; + bool first = true; + foreach (Formals::Formals_::iterator, i, formals->formals) { + if (first) first = false; else str << ", "; + str << i->name; + if (i->def) str << " ? " << *i->def; + } + str << " }"; + if (!arg.empty()) str << " @ "; + } + if (!arg.empty()) str << arg; + str << ": " << *body << ")"; +} + +void ExprLet::show(std::ostream & str) +{ + str << "let "; + foreach (ExprAttrs::AttrDefs::iterator, i, attrs->attrs) + if (i->second.inherited) + str << "inherit " << i->first << "; "; + else + str << i->first << " = " << *i->second.e << "; "; + str << "in " << *body; +} + +void ExprWith::show(std::ostream & str) +{ + str << "with " << *attrs << "; " << *body; +} + +void ExprIf::show(std::ostream & str) +{ + str << "if " << *cond << " then " << *then << " else " << *else_; +} + +void ExprAssert::show(std::ostream & str) +{ + str << "assert " << *cond << "; " << *body; +} + +void ExprOpNot::show(std::ostream & str) +{ + str << "! " << *e; +} + +void ExprBuiltin::show(std::ostream & str) +{ + str << "builtins." << name; +} + +void ExprConcatStrings::show(std::ostream & str) +{ + bool first = true; + foreach (vector<Expr *>::iterator, i, *es) { + if (first) first = false; else str << " + "; + str << **i; + } +} + +void ExprPos::show(std::ostream & str) +{ + str << "__curPos"; +} + + +std::ostream & operator << (std::ostream & str, const Pos & pos) +{ + if (!pos.line) + str << "undefined position"; + else + str << (format("`%1%:%2%:%3%'") % pos.file % pos.line % pos.column).str(); + return str; +} + + +string showAttrPath(const AttrPath & attrPath) +{ + std::ostringstream out; + bool first = true; + foreach (AttrPath::const_iterator, i, attrPath) { + if (!first) + out << '.'; + else + first = false; + if (i->symbol.set()) + out << i->symbol; + else + out << "\"${" << *i->expr << "}\""; + } + return out.str(); +} + + +Pos noPos; + + +/* Computing levels/displacements for variables. */ + +void Expr::bindVars(const StaticEnv & env) +{ + abort(); +} + +void ExprInt::bindVars(const StaticEnv & env) +{ +} + +void ExprString::bindVars(const StaticEnv & env) +{ +} + +void ExprPath::bindVars(const StaticEnv & env) +{ +} + +void ExprVar::bindVars(const StaticEnv & env) +{ + /* Check whether the variable appears in the environment. If so, + set its level and displacement. */ + const StaticEnv * curEnv; + unsigned int level; + int withLevel = -1; + for (curEnv = &env, level = 0; curEnv; curEnv = curEnv->up, level++) { + if (curEnv->isWith) { + if (withLevel == -1) withLevel = level; + } else { + StaticEnv::Vars::const_iterator i = curEnv->vars.find(name); + if (i != curEnv->vars.end()) { + fromWith = false; + this->level = level; + displ = i->second; + return; + } + } + } + + /* Otherwise, the variable must be obtained from the nearest + enclosing `with'. If there is no `with', then we can issue an + "undefined variable" error now. */ + if (withLevel == -1) throw UndefinedVarError(format("undefined variable `%1%' at %2%") % name % pos); + + fromWith = true; + this->level = withLevel; +} + +void ExprSelect::bindVars(const StaticEnv & env) +{ + e->bindVars(env); + if (def) def->bindVars(env); + foreach (AttrPath::iterator, i, attrPath) + if (!i->symbol.set()) + i->expr->bindVars(env); +} + +void ExprOpHasAttr::bindVars(const StaticEnv & env) +{ + e->bindVars(env); + foreach (AttrPath::iterator, i, attrPath) + if (!i->symbol.set()) + i->expr->bindVars(env); +} + +void ExprAttrs::bindVars(const StaticEnv & env) +{ + const StaticEnv *dynamicEnv = &env; + if (recursive) { + StaticEnv newEnv(false, &env); + dynamicEnv = &newEnv; + + unsigned int displ = 0; + foreach (AttrDefs::iterator, i, attrs) + newEnv.vars[i->first] = i->second.displ = displ++; + + foreach (AttrDefs::iterator, i, attrs) + i->second.e->bindVars(i->second.inherited ? env : newEnv); + } + + else + foreach (AttrDefs::iterator, i, attrs) + i->second.e->bindVars(env); + + foreach (DynamicAttrDefs::iterator, i, dynamicAttrs) { + i->nameExpr->bindVars(*dynamicEnv); + i->valueExpr->bindVars(*dynamicEnv); + } +} + +void ExprList::bindVars(const StaticEnv & env) +{ + foreach (vector<Expr *>::iterator, i, elems) + (*i)->bindVars(env); +} + +void ExprLambda::bindVars(const StaticEnv & env) +{ + StaticEnv newEnv(false, &env); + + unsigned int displ = 0; + + if (!arg.empty()) newEnv.vars[arg] = displ++; + + if (matchAttrs) { + foreach (Formals::Formals_::iterator, i, formals->formals) + newEnv.vars[i->name] = displ++; + + foreach (Formals::Formals_::iterator, i, formals->formals) + if (i->def) i->def->bindVars(newEnv); + } + + body->bindVars(newEnv); +} + +void ExprLet::bindVars(const StaticEnv & env) +{ + StaticEnv newEnv(false, &env); + + unsigned int displ = 0; + foreach (ExprAttrs::AttrDefs::iterator, i, attrs->attrs) + newEnv.vars[i->first] = i->second.displ = displ++; + + foreach (ExprAttrs::AttrDefs::iterator, i, attrs->attrs) + i->second.e->bindVars(i->second.inherited ? env : newEnv); + + body->bindVars(newEnv); +} + +void ExprWith::bindVars(const StaticEnv & env) +{ + /* Does this `with' have an enclosing `with'? If so, record its + level so that `lookupVar' can look up variables in the previous + `with' if this one doesn't contain the desired attribute. */ + const StaticEnv * curEnv; + unsigned int level; + prevWith = 0; + for (curEnv = &env, level = 1; curEnv; curEnv = curEnv->up, level++) + if (curEnv->isWith) { + prevWith = level; + break; + } + + attrs->bindVars(env); + StaticEnv newEnv(true, &env); + body->bindVars(newEnv); +} + +void ExprIf::bindVars(const StaticEnv & env) +{ + cond->bindVars(env); + then->bindVars(env); + else_->bindVars(env); +} + +void ExprAssert::bindVars(const StaticEnv & env) +{ + cond->bindVars(env); + body->bindVars(env); +} + +void ExprOpNot::bindVars(const StaticEnv & env) +{ + e->bindVars(env); +} + +void ExprBuiltin::bindVars(const StaticEnv & env) +{ +} + +void ExprConcatStrings::bindVars(const StaticEnv & env) +{ + foreach (vector<Expr *>::iterator, i, *es) + (*i)->bindVars(env); +} + +void ExprPos::bindVars(const StaticEnv & env) +{ +} + + +/* Storing function names. */ + +void Expr::setName(Symbol & name) +{ +} + + +void ExprLambda::setName(Symbol & name) +{ + this->name = name; + body->setName(name); +} + + +string ExprLambda::showNamePos() const +{ + return (format("%1% at %2%") % (name.set() ? "`" + (string) name + "'" : "an anonymous function") % pos).str(); +} + + + +/* Symbol table. */ + +size_t SymbolTable::totalSize() const +{ + size_t n = 0; + foreach (Symbols::const_iterator, i, symbols) + n += i->size(); + return n; +} + + +} diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh new file mode 100644 index 000000000000..7586fffb83d4 --- /dev/null +++ b/src/libexpr/nixexpr.hh @@ -0,0 +1,332 @@ +#pragma once + +#include "value.hh" +#include "symbol-table.hh" + +#include <map> + + +namespace nix { + + +MakeError(EvalError, Error) +MakeError(ParseError, Error) +MakeError(AssertionError, EvalError) +MakeError(ThrownError, AssertionError) +MakeError(Abort, EvalError) +MakeError(TypeError, EvalError) +MakeError(ImportError, EvalError) // error building an imported derivation +MakeError(UndefinedVarError, Error) + + +/* Position objects. */ + +struct Pos +{ + Symbol file; + unsigned int line, column; + Pos() : line(0), column(0) { }; + Pos(const Symbol & file, unsigned int line, unsigned int column) + : file(file), line(line), column(column) { }; + bool operator < (const Pos & p2) const + { + if (!line) return p2.line; + if (!p2.line) return false; + int d = ((string) file).compare((string) p2.file); + if (d < 0) return true; + if (d > 0) return false; + if (line < p2.line) return true; + if (line > p2.line) return false; + return column < p2.column; + } +}; + +extern Pos noPos; + +std::ostream & operator << (std::ostream & str, const Pos & pos); + + +struct Env; +struct Value; +class EvalState; +struct StaticEnv; +struct Expr; + + +/* An attribute path is a sequence of attribute names. */ +struct AttrName +{ + Symbol symbol; + Expr *expr; + AttrName(const Symbol & s) : symbol(s) {}; + AttrName(Expr *e) : expr(e) {}; +}; + +typedef std::vector<AttrName> AttrPath; + +string showAttrPath(const AttrPath & attrPath); + + +/* Abstract syntax of Nix expressions. */ + +struct Expr +{ + virtual ~Expr() { }; + virtual void show(std::ostream & str); + virtual void bindVars(const StaticEnv & env); + virtual void eval(EvalState & state, Env & env, Value & v); + virtual Value * maybeThunk(EvalState & state, Env & env); + virtual void setName(Symbol & name); +}; + +std::ostream & operator << (std::ostream & str, Expr & e); + +#define COMMON_METHODS \ + void show(std::ostream & str); \ + void eval(EvalState & state, Env & env, Value & v); \ + void bindVars(const StaticEnv & env); + +struct ExprInt : Expr +{ + NixInt n; + Value v; + ExprInt(NixInt n) : n(n) { mkInt(v, n); }; + COMMON_METHODS + Value * maybeThunk(EvalState & state, Env & env); +}; + +struct ExprString : Expr +{ + Symbol s; + Value v; + ExprString(const Symbol & s) : s(s) { mkString(v, s); }; + COMMON_METHODS + Value * maybeThunk(EvalState & state, Env & env); +}; + +/* Temporary class used during parsing of indented strings. */ +struct ExprIndStr : Expr +{ + string s; + ExprIndStr(const string & s) : s(s) { }; +}; + +struct ExprPath : Expr +{ + string s; + Value v; + ExprPath(const string & s) : s(s) { mkPathNoCopy(v, this->s.c_str()); }; + COMMON_METHODS + Value * maybeThunk(EvalState & state, Env & env); +}; + +struct ExprVar : Expr +{ + Pos pos; + Symbol name; + + /* Whether the variable comes from an environment (e.g. a rec, let + or function argument) or from a "with". */ + bool fromWith; + + /* In the former case, the value is obtained by going `level' + levels up from the current environment and getting the + `displ'th value in that environment. In the latter case, the + value is obtained by getting the attribute named `name' from + the set stored in the environment that is `level' levels up + from the current one.*/ + unsigned int level; + unsigned int displ; + + ExprVar(const Pos & pos, const Symbol & name) : pos(pos), name(name) { }; + COMMON_METHODS + Value * maybeThunk(EvalState & state, Env & env); +}; + +struct ExprSelect : Expr +{ + Expr * e, * def; + AttrPath attrPath; + ExprSelect(Expr * e, const AttrPath & attrPath, Expr * def) : e(e), def(def), attrPath(attrPath) { }; + ExprSelect(Expr * e, const Symbol & name) : e(e), def(0) { attrPath.push_back(AttrName(name)); }; + COMMON_METHODS +}; + +struct ExprOpHasAttr : Expr +{ + Expr * e; + AttrPath attrPath; + ExprOpHasAttr(Expr * e, const AttrPath & attrPath) : e(e), attrPath(attrPath) { }; + COMMON_METHODS +}; + +struct ExprAttrs : Expr +{ + bool recursive; + struct AttrDef { + bool inherited; + Expr * e; + Pos pos; + unsigned int displ; // displacement + AttrDef(Expr * e, const Pos & pos, bool inherited=false) : inherited(inherited), e(e), pos(pos) { }; + AttrDef() { }; + }; + typedef std::map<Symbol, AttrDef> AttrDefs; + AttrDefs attrs; + struct DynamicAttrDef { + Expr * nameExpr; + Expr * valueExpr; + Pos pos; + DynamicAttrDef(Expr * nameExpr, Expr * valueExpr, const Pos & pos) : nameExpr(nameExpr), valueExpr(valueExpr), pos(pos) { }; + }; + typedef std::vector<DynamicAttrDef> DynamicAttrDefs; + DynamicAttrDefs dynamicAttrs; + ExprAttrs() : recursive(false) { }; + COMMON_METHODS +}; + +struct ExprList : Expr +{ + std::vector<Expr *> elems; + ExprList() { }; + COMMON_METHODS +}; + +struct Formal +{ + Symbol name; + Expr * def; + Formal(const Symbol & name, Expr * def) : name(name), def(def) { }; +}; + +struct Formals +{ + typedef std::list<Formal> Formals_; + Formals_ formals; + std::set<Symbol> argNames; // used during parsing + bool ellipsis; +}; + +struct ExprLambda : Expr +{ + Pos pos; + Symbol name; + Symbol arg; + bool matchAttrs; + Formals * formals; + Expr * body; + ExprLambda(const Pos & pos, const Symbol & arg, bool matchAttrs, Formals * formals, Expr * body) + : pos(pos), arg(arg), matchAttrs(matchAttrs), formals(formals), body(body) + { + if (!arg.empty() && formals && formals->argNames.find(arg) != formals->argNames.end()) + throw ParseError(format("duplicate formal function argument `%1%' at %2%") + % arg % pos); + }; + void setName(Symbol & name); + string showNamePos() const; + COMMON_METHODS +}; + +struct ExprLet : Expr +{ + ExprAttrs * attrs; + Expr * body; + ExprLet(ExprAttrs * attrs, Expr * body) : attrs(attrs), body(body) { }; + COMMON_METHODS +}; + +struct ExprWith : Expr +{ + Pos pos; + Expr * attrs, * body; + unsigned int prevWith; + ExprWith(const Pos & pos, Expr * attrs, Expr * body) : pos(pos), attrs(attrs), body(body) { }; + COMMON_METHODS +}; + +struct ExprIf : Expr +{ + Expr * cond, * then, * else_; + ExprIf(Expr * cond, Expr * then, Expr * else_) : cond(cond), then(then), else_(else_) { }; + COMMON_METHODS +}; + +struct ExprAssert : Expr +{ + Pos pos; + Expr * cond, * body; + ExprAssert(const Pos & pos, Expr * cond, Expr * body) : pos(pos), cond(cond), body(body) { }; + COMMON_METHODS +}; + +struct ExprOpNot : Expr +{ + Expr * e; + ExprOpNot(Expr * e) : e(e) { }; + COMMON_METHODS +}; + +struct ExprBuiltin : Expr +{ + Symbol name; + ExprBuiltin(const Symbol & name) : name(name) { }; + COMMON_METHODS +}; + +#define MakeBinOp(name, s) \ + struct Expr##name : Expr \ + { \ + Expr * e1, * e2; \ + Expr##name(Expr * e1, Expr * e2) : e1(e1), e2(e2) { }; \ + void show(std::ostream & str) \ + { \ + str << *e1 << " " s " " << *e2; \ + } \ + void bindVars(const StaticEnv & env) \ + { \ + e1->bindVars(env); e2->bindVars(env); \ + } \ + void eval(EvalState & state, Env & env, Value & v); \ + }; + +MakeBinOp(App, "") +MakeBinOp(OpEq, "==") +MakeBinOp(OpNEq, "!=") +MakeBinOp(OpAnd, "&&") +MakeBinOp(OpOr, "||") +MakeBinOp(OpImpl, "->") +MakeBinOp(OpUpdate, "//") +MakeBinOp(OpConcatLists, "++") + +struct ExprConcatStrings : Expr +{ + bool forceString; + vector<Expr *> * es; + ExprConcatStrings(bool forceString, vector<Expr *> * es) + : forceString(forceString), es(es) { }; + COMMON_METHODS +}; + +struct ExprPos : Expr +{ + Pos pos; + ExprPos(const Pos & pos) : pos(pos) { }; + COMMON_METHODS +}; + + +/* Static environments are used to map variable names onto (level, + displacement) pairs used to obtain the value of the variable at + runtime. */ +struct StaticEnv +{ + bool isWith; + const StaticEnv * up; + typedef std::map<Symbol, unsigned int> Vars; + Vars vars; + StaticEnv(bool isWith, const StaticEnv * up) : isWith(isWith), up(up) { }; +}; + + + +} diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y new file mode 100644 index 000000000000..8a084fea0697 --- /dev/null +++ b/src/libexpr/parser.y @@ -0,0 +1,650 @@ +%glr-parser +%pure-parser +%locations +%error-verbose +%defines +/* %no-lines */ +%parse-param { void * scanner } +%parse-param { nix::ParseData * data } +%lex-param { void * scanner } +%lex-param { nix::ParseData * data } +%expect 1 +%expect-rr 1 + +%code requires { + +#ifndef BISON_HEADER +#define BISON_HEADER + +#include "util.hh" + +#include "nixexpr.hh" +#include "eval.hh" + +namespace nix { + + struct ParseData + { + EvalState & state; + SymbolTable & symbols; + Expr * result; + Path basePath; + Symbol path; + string error; + Symbol sLetBody; + ParseData(EvalState & state) + : state(state) + , symbols(state.symbols) + , sLetBody(symbols.create("<let-body>")) + { }; + }; + +} + +#define YY_DECL int yylex \ + (YYSTYPE * yylval_param, YYLTYPE * yylloc_param, yyscan_t yyscanner, nix::ParseData * data) + +#endif + +} + +%{ + +#include "parser-tab.hh" +#include "lexer-tab.hh" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +YY_DECL; + +using namespace nix; + + +namespace nix { + + +static void dupAttr(const AttrPath & attrPath, const Pos & pos, const Pos & prevPos) +{ + throw ParseError(format("attribute `%1%' at %2% already defined at %3%") + % showAttrPath(attrPath) % pos % prevPos); +} + + +static void dupAttr(Symbol attr, const Pos & pos, const Pos & prevPos) +{ + throw ParseError(format("attribute `%1%' at %2% already defined at %3%") + % attr % pos % prevPos); +} + + +static void addAttr(ExprAttrs * attrs, AttrPath & attrPath, + Expr * e, const Pos & pos) +{ + AttrPath::iterator i; + // All attrpaths have at least one attr + assert(!attrPath.empty()); + for (i = attrPath.begin(); i + 1 < attrPath.end(); i++) { + if (i->symbol.set()) { + ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol); + if (j != attrs->attrs.end()) { + if (!j->second.inherited) { + ExprAttrs * attrs2 = dynamic_cast<ExprAttrs *>(j->second.e); + if (!attrs2) dupAttr(attrPath, pos, j->second.pos); + attrs = attrs2; + } else + dupAttr(attrPath, pos, j->second.pos); + } else { + ExprAttrs * nested = new ExprAttrs; + attrs->attrs[i->symbol] = ExprAttrs::AttrDef(nested, pos); + attrs = nested; + } + } else { + ExprAttrs *nested = new ExprAttrs; + attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, nested, pos)); + attrs = nested; + } + } + if (i->symbol.set()) { + ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol); + if (j != attrs->attrs.end()) { + dupAttr(attrPath, pos, j->second.pos); + } else { + attrs->attrs[i->symbol] = ExprAttrs::AttrDef(e, pos); + e->setName(i->symbol); + } + } else { + attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, e, pos)); + } +} + + +static void addFormal(const Pos & pos, Formals * formals, const Formal & formal) +{ + if (formals->argNames.find(formal.name) != formals->argNames.end()) + throw ParseError(format("duplicate formal function argument `%1%' at %2%") + % formal.name % pos); + formals->formals.push_front(formal); + formals->argNames.insert(formal.name); +} + + +static Expr * stripIndentation(SymbolTable & symbols, vector<Expr *> & es) +{ + if (es.empty()) return new ExprString(symbols.create("")); + + /* Figure out the minimum indentation. Note that by design + whitespace-only final lines are not taken into account. (So + the " " in "\n ''" is ignored, but the " " in "\n foo''" is.) */ + bool atStartOfLine = true; /* = seen only whitespace in the current line */ + unsigned int minIndent = 1000000; + unsigned int curIndent = 0; + foreach (vector<Expr *>::iterator, i, es) { + ExprIndStr * e = dynamic_cast<ExprIndStr *>(*i); + if (!e) { + /* Anti-quotations end the current start-of-line whitespace. */ + if (atStartOfLine) { + atStartOfLine = false; + if (curIndent < minIndent) minIndent = curIndent; + } + continue; + } + for (unsigned int j = 0; j < e->s.size(); ++j) { + if (atStartOfLine) { + if (e->s[j] == ' ') + curIndent++; + else if (e->s[j] == '\n') { + /* Empty line, doesn't influence minimum + indentation. */ + curIndent = 0; + } else { + atStartOfLine = false; + if (curIndent < minIndent) minIndent = curIndent; + } + } else if (e->s[j] == '\n') { + atStartOfLine = true; + curIndent = 0; + } + } + } + + /* Strip spaces from each line. */ + vector<Expr *> * es2 = new vector<Expr *>; + atStartOfLine = true; + unsigned int curDropped = 0; + unsigned int n = es.size(); + for (vector<Expr *>::iterator i = es.begin(); i != es.end(); ++i, --n) { + ExprIndStr * e = dynamic_cast<ExprIndStr *>(*i); + if (!e) { + atStartOfLine = false; + curDropped = 0; + es2->push_back(*i); + continue; + } + + string s2; + for (unsigned int j = 0; j < e->s.size(); ++j) { + if (atStartOfLine) { + if (e->s[j] == ' ') { + if (curDropped++ >= minIndent) + s2 += e->s[j]; + } + else if (e->s[j] == '\n') { + curDropped = 0; + s2 += e->s[j]; + } else { + atStartOfLine = false; + curDropped = 0; + s2 += e->s[j]; + } + } else { + s2 += e->s[j]; + if (e->s[j] == '\n') atStartOfLine = true; + } + } + + /* Remove the last line if it is empty and consists only of + spaces. */ + if (n == 1) { + string::size_type p = s2.find_last_of('\n'); + if (p != string::npos && s2.find_first_not_of(' ', p + 1) == string::npos) + s2 = string(s2, 0, p + 1); + } + + es2->push_back(new ExprString(symbols.create(s2))); + } + + /* If this is a single string, then don't do a concatenation. */ + return es2->size() == 1 && dynamic_cast<ExprString *>((*es2)[0]) ? (*es2)[0] : new ExprConcatStrings(true, es2); +} + + +void backToString(yyscan_t scanner); +void backToIndString(yyscan_t scanner); + + +static Pos makeCurPos(const YYLTYPE & loc, ParseData * data) +{ + return Pos(data->path, loc.first_line, loc.first_column); +} + +#define CUR_POS makeCurPos(*yylocp, data) + + +} + + +void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * error) +{ + data->error = (format("%1%, at %2%") + % error % makeCurPos(*loc, data)).str(); +} + + +%} + +%union { + // !!! We're probably leaking stuff here. + nix::Expr * e; + nix::ExprList * list; + nix::ExprAttrs * attrs; + nix::Formals * formals; + nix::Formal * formal; + nix::NixInt n; + const char * id; // !!! -> Symbol + char * path; + char * uri; + std::vector<nix::AttrName> * attrNames; + std::vector<nix::Expr *> * string_parts; +} + +%type <e> start expr expr_function expr_if expr_op +%type <e> expr_app expr_select expr_simple +%type <list> expr_list +%type <attrs> binds +%type <formals> formals +%type <formal> formal +%type <attrNames> attrs attrpath +%type <string_parts> string_parts_interpolated ind_string_parts +%type <e> string_parts string_attr +%type <id> attr +%token <id> ID ATTRPATH +%token <e> STR IND_STR +%token <n> INT +%token <path> PATH SPATH +%token <uri> URI +%token IF THEN ELSE ASSERT WITH LET IN REC INHERIT EQ NEQ AND OR IMPL OR_KW +%token DOLLAR_CURLY /* == ${ */ +%token IND_STRING_OPEN IND_STRING_CLOSE +%token ELLIPSIS + +%nonassoc IMPL +%left OR +%left AND +%nonassoc EQ NEQ +%left '<' '>' LEQ GEQ +%right UPDATE +%left NOT +%left '+' '-' +%left '*' '/' +%right CONCAT +%nonassoc '?' +%nonassoc '~' +%nonassoc NEGATE + +%% + +start: expr { data->result = $1; }; + +expr: expr_function; + +expr_function + : ID ':' expr_function + { $$ = new ExprLambda(CUR_POS, data->symbols.create($1), false, 0, $3); } + | '{' formals '}' ':' expr_function + { $$ = new ExprLambda(CUR_POS, data->symbols.create(""), true, $2, $5); } + | '{' formals '}' '@' ID ':' expr_function + { $$ = new ExprLambda(CUR_POS, data->symbols.create($5), true, $2, $7); } + | ID '@' '{' formals '}' ':' expr_function + { $$ = new ExprLambda(CUR_POS, data->symbols.create($1), true, $4, $7); } + | ASSERT expr ';' expr_function + { $$ = new ExprAssert(CUR_POS, $2, $4); } + | WITH expr ';' expr_function + { $$ = new ExprWith(CUR_POS, $2, $4); } + | LET binds IN expr_function + { if (!$2->dynamicAttrs.empty()) + throw ParseError(format("dynamic attributes not allowed in let at %1%") + % CUR_POS); + $$ = new ExprLet($2, $4); + } + | expr_if + ; + +expr_if + : IF expr THEN expr ELSE expr { $$ = new ExprIf($2, $4, $6); } + | expr_op + ; + +expr_op + : '!' expr_op %prec NOT { $$ = new ExprOpNot($2); } +| '-' expr_op %prec NEGATE { $$ = new ExprApp(new ExprApp(new ExprBuiltin(data->symbols.create("sub")), new ExprInt(0)), $2); } + | expr_op EQ expr_op { $$ = new ExprOpEq($1, $3); } + | expr_op NEQ expr_op { $$ = new ExprOpNEq($1, $3); } + | expr_op '<' expr_op { $$ = new ExprApp(new ExprApp(new ExprBuiltin(data->symbols.create("lessThan")), $1), $3); } + | expr_op LEQ expr_op { $$ = new ExprOpNot(new ExprApp(new ExprApp(new ExprBuiltin(data->symbols.create("lessThan")), $3), $1)); } + | expr_op '>' expr_op { $$ = new ExprApp(new ExprApp(new ExprBuiltin(data->symbols.create("lessThan")), $3), $1); } + | expr_op GEQ expr_op { $$ = new ExprOpNot(new ExprApp(new ExprApp(new ExprBuiltin(data->symbols.create("lessThan")), $1), $3)); } + | expr_op AND expr_op { $$ = new ExprOpAnd($1, $3); } + | expr_op OR expr_op { $$ = new ExprOpOr($1, $3); } + | expr_op IMPL expr_op { $$ = new ExprOpImpl($1, $3); } + | expr_op UPDATE expr_op { $$ = new ExprOpUpdate($1, $3); } + | expr_op '?' attrpath { $$ = new ExprOpHasAttr($1, *$3); } + | expr_op '+' expr_op + { vector<Expr *> * l = new vector<Expr *>; + l->push_back($1); + l->push_back($3); + $$ = new ExprConcatStrings(false, l); + } + | expr_op '-' expr_op { $$ = new ExprApp(new ExprApp(new ExprBuiltin(data->symbols.create("sub")), $1), $3); } + | expr_op '*' expr_op { $$ = new ExprApp(new ExprApp(new ExprBuiltin(data->symbols.create("mul")), $1), $3); } + | expr_op '/' expr_op { $$ = new ExprApp(new ExprApp(new ExprBuiltin(data->symbols.create("div")), $1), $3); } + | expr_op CONCAT expr_op { $$ = new ExprOpConcatLists($1, $3); } + | expr_app + ; + +expr_app + : expr_app expr_select + { $$ = new ExprApp($1, $2); } + | expr_select { $$ = $1; } + ; + +expr_select + : expr_simple '.' attrpath + { $$ = new ExprSelect($1, *$3, 0); } + | expr_simple '.' attrpath OR_KW expr_select + { $$ = new ExprSelect($1, *$3, $5); } + | /* Backwards compatibility: because Nixpkgs has a rarely used + function named ‘or’, allow stuff like ‘map or [...]’. */ + expr_simple OR_KW + { $$ = new ExprApp($1, new ExprVar(CUR_POS, data->symbols.create("or"))); } + | expr_simple { $$ = $1; } + ; + +expr_simple + : ID { + if (strcmp($1, "__curPos") == 0) + $$ = new ExprPos(CUR_POS); + else + $$ = new ExprVar(CUR_POS, data->symbols.create($1)); + } + | INT { $$ = new ExprInt($1); } + | '"' string_parts '"' { $$ = $2; } + | IND_STRING_OPEN ind_string_parts IND_STRING_CLOSE { + $$ = stripIndentation(data->symbols, *$2); + } + | PATH { $$ = new ExprPath(absPath($1, data->basePath)); } + | SPATH { + string path($1 + 1, strlen($1) - 2); + Path path2 = data->state.findFile(path); + /* The file wasn't found in the search path. However, we can't + throw an error here, because the expression might never be + evaluated. So return an expression that lazily calls + ‘throw’. */ + $$ = path2 == "" + ? (Expr * ) new ExprApp( + new ExprBuiltin(data->symbols.create("throw")), + new ExprString(data->symbols.create( + (format("file `%1%' was not found in the Nix search path (add it using $NIX_PATH or -I)") % path).str()))) + : (Expr * ) new ExprPath(path2); + } + | URI { $$ = new ExprString(data->symbols.create($1)); } + | '(' expr ')' { $$ = $2; } + /* Let expressions `let {..., body = ...}' are just desugared + into `(rec {..., body = ...}).body'. */ + | LET '{' binds '}' + { $3->recursive = true; $$ = new ExprSelect($3, data->symbols.create("body")); } + | REC '{' binds '}' + { $3->recursive = true; $$ = $3; } + | '{' binds '}' + { $$ = $2; } + | '[' expr_list ']' { $$ = $2; } + ; + +string_parts + : STR + | string_parts_interpolated { $$ = new ExprConcatStrings(true, $1); } + | { $$ = new ExprString(data->symbols.create("")); } + ; + +string_parts_interpolated + : string_parts_interpolated STR { $$ = $1; $1->push_back($2); } + | string_parts_interpolated DOLLAR_CURLY expr '}' { backToString(scanner); $$ = $1; $1->push_back($3); } + | STR DOLLAR_CURLY expr '}' + { + backToString(scanner); + $$ = new vector<Expr *>; + $$->push_back($1); + $$->push_back($3); + } + | DOLLAR_CURLY expr '}' + { + backToString(scanner); + $$ = new vector<Expr *>; + $$->push_back($2); + } + ; + +ind_string_parts + : ind_string_parts IND_STR { $$ = $1; $1->push_back($2); } + | ind_string_parts DOLLAR_CURLY expr '}' { backToIndString(scanner); $$ = $1; $1->push_back($3); } + | { $$ = new vector<Expr *>; } + ; + +binds + : binds attrpath '=' expr ';' { $$ = $1; addAttr($$, *$2, $4, makeCurPos(@2, data)); } + | binds INHERIT attrs ';' + { $$ = $1; + foreach (AttrPath::iterator, i, *$3) { + if ($$->attrs.find(i->symbol) != $$->attrs.end()) + dupAttr(i->symbol, makeCurPos(@3, data), $$->attrs[i->symbol].pos); + Pos pos = makeCurPos(@3, data); + $$->attrs[i->symbol] = ExprAttrs::AttrDef(new ExprVar(CUR_POS, i->symbol), pos, true); + } + } + | binds INHERIT '(' expr ')' attrs ';' + { $$ = $1; + /* !!! Should ensure sharing of the expression in $4. */ + foreach (AttrPath::iterator, i, *$6) { + if ($$->attrs.find(i->symbol) != $$->attrs.end()) + dupAttr(i->symbol, makeCurPos(@6, data), $$->attrs[i->symbol].pos); + $$->attrs[i->symbol] = ExprAttrs::AttrDef(new ExprSelect($4, i->symbol), makeCurPos(@6, data)); + } + } + | { $$ = new ExprAttrs; } + ; + +attrs + : attrs attr { $$ = $1; $1->push_back(AttrName(data->symbols.create($2))); } + | attrs string_attr + { $$ = $1; + ExprString *str = dynamic_cast<ExprString *>($2); + if (str) { + $$->push_back(AttrName(str->s)); + delete str; + } else + throw ParseError(format("dynamic attributes not allowed in inherit at %1%") + % makeCurPos(@2, data)); + } + | { $$ = new AttrPath; } + ; + +attrpath + : attrpath '.' attr { $$ = $1; $1->push_back(AttrName(data->symbols.create($3))); } + | attrpath '.' string_attr + { $$ = $1; + ExprString *str = dynamic_cast<ExprString *>($3); + if (str) { + $$->push_back(AttrName(str->s)); + delete str; + } else + $$->push_back(AttrName($3)); + } + | attr { $$ = new vector<AttrName>; $$->push_back(AttrName(data->symbols.create($1))); } + | string_attr + { $$ = new vector<AttrName>; + ExprString *str = dynamic_cast<ExprString *>($1); + if (str) { + $$->push_back(AttrName(str->s)); + delete str; + } else + $$->push_back(AttrName($1)); + } + ; + +attr + : ID { $$ = $1; } + | OR_KW { $$ = "or"; } + ; + +string_attr + : '"' string_parts '"' { $$ = $2; } + | DOLLAR_CURLY expr '}' { $$ = new ExprConcatStrings(true, new vector<Expr*>(1, $2)); } + ; + +expr_list + : expr_list expr_select { $$ = $1; $1->elems.push_back($2); /* !!! dangerous */ } + | { $$ = new ExprList; } + ; + +formals + : formal ',' formals + { $$ = $3; addFormal(CUR_POS, $$, *$1); } + | formal + { $$ = new Formals; addFormal(CUR_POS, $$, *$1); $$->ellipsis = false; } + | + { $$ = new Formals; $$->ellipsis = false; } + | ELLIPSIS + { $$ = new Formals; $$->ellipsis = true; } + ; + +formal + : ID { $$ = new Formal(data->symbols.create($1), 0); } + | ID '?' expr { $$ = new Formal(data->symbols.create($1), $3); } + ; + +%% + + +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> +#include <unistd.h> + +#include <eval.hh> + + +namespace nix { + + +Expr * EvalState::parse(const char * text, + const Path & path, const Path & basePath, StaticEnv & staticEnv) +{ + yyscan_t scanner; + ParseData data(*this); + data.basePath = basePath; + data.path = data.symbols.create(path); + + yylex_init(&scanner); + yy_scan_string(text, scanner); + int res = yyparse(scanner, &data); + yylex_destroy(scanner); + + if (res) throw ParseError(data.error); + + data.result->bindVars(staticEnv); + + return data.result; +} + + +Path resolveExprPath(Path path) +{ + assert(path[0] == '/'); + + /* If `path' is a symlink, follow it. This is so that relative + path references work. */ + struct stat st; + while (true) { + if (lstat(path.c_str(), &st)) + throw SysError(format("getting status of `%1%'") % path); + if (!S_ISLNK(st.st_mode)) break; + path = absPath(readLink(path), dirOf(path)); + } + + /* If `path' refers to a directory, append `/default.nix'. */ + if (S_ISDIR(st.st_mode)) + path = canonPath(path + "/default.nix"); + + return path; +} + + +Expr * EvalState::parseExprFromFile(const Path & path) +{ + return parse(readFile(path).c_str(), path, dirOf(path), staticBaseEnv); +} + + +Expr * EvalState::parseExprFromString(const string & s, const Path & basePath, StaticEnv & staticEnv) +{ + return parse(s.c_str(), "(string)", basePath, staticEnv); +} + + +Expr * EvalState::parseExprFromString(const string & s, const Path & basePath) +{ + return parseExprFromString(s, basePath, staticBaseEnv); +} + + +void EvalState::addToSearchPath(const string & s) +{ + size_t pos = s.find('='); + string prefix; + Path path; + if (pos == string::npos) { + path = s; + } else { + prefix = string(s, 0, pos); + path = string(s, pos + 1); + } + + path = absPath(path); + if (pathExists(path)) { + debug(format("adding path `%1%' to the search path") % path); + searchPath.insert(searchPathInsertionPoint, std::pair<string, Path>(prefix, path)); + } +} + + +Path EvalState::findFile(const string & path) +{ + foreach (SearchPath::iterator, i, searchPath) { + Path res; + if (i->first.empty()) + res = i->second + "/" + path; + else { + if (path.compare(0, i->first.size(), i->first) != 0 || + (path.size() > i->first.size() && path[i->first.size()] != '/')) + continue; + res = i->second + + (path.size() == i->first.size() ? "" : "/" + string(path, i->first.size())); + } + if (pathExists(res)) return canonPath(res); + } + return ""; +} + + +} diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc new file mode 100644 index 000000000000..ca316f08af13 --- /dev/null +++ b/src/libexpr/primops.cc @@ -0,0 +1,1336 @@ +#include "eval.hh" +#include "misc.hh" +#include "globals.hh" +#include "store-api.hh" +#include "util.hh" +#include "archive.hh" +#include "value-to-xml.hh" +#include "value-to-json.hh" +#include "names.hh" +#include "eval-inline.hh" + +#include <sys/types.h> +#include <sys/stat.h> +#include <unistd.h> + +#include <algorithm> +#include <cstring> + + +namespace nix { + + +/************************************************************* + * Miscellaneous + *************************************************************/ + + +/* Decode a context string ‘!<name>!<path>’ into a pair <path, + name>. */ +std::pair<string, string> decodeContext(const string & s) +{ + if (s.at(0) == '!') { + size_t index = s.find("!", 1); + return std::pair<string, string>(string(s, index + 1), string(s, 1, index - 1)); + } else + return std::pair<string, string>(s, ""); +} + + +/* Load and evaluate an expression from path specified by the + argument. */ +static void prim_import(EvalState & state, Value * * args, Value & v) +{ + PathSet context; + Path path = state.coerceToPath(*args[0], context); + + foreach (PathSet::iterator, i, context) { + Path ctx = decodeContext(*i).first; + assert(isStorePath(ctx)); + if (!store->isValidPath(ctx)) + throw EvalError(format("cannot import `%1%', since path `%2%' is not valid") + % path % ctx); + if (isDerivation(ctx)) + try { + /* For performance, prefetch all substitute info. */ + PathSet willBuild, willSubstitute, unknown; + unsigned long long downloadSize, narSize; + queryMissing(*store, singleton<PathSet>(ctx), + willBuild, willSubstitute, unknown, downloadSize, narSize); + + /* !!! If using a substitute, we only need to fetch + the selected output of this derivation. */ + store->buildPaths(singleton<PathSet>(ctx)); + } catch (Error & e) { + throw ImportError(e.msg()); + } + } + + if (isStorePath(path) && store->isValidPath(path) && isDerivation(path)) { + Derivation drv = parseDerivation(readFile(path)); + Value & w = *state.allocValue(); + state.mkAttrs(w, 1 + drv.outputs.size()); + mkString(*state.allocAttr(w, state.sDrvPath), path, singleton<PathSet>("=" + path)); + state.mkList(*state.allocAttr(w, state.symbols.create("outputs")), drv.outputs.size()); + unsigned int outputs_index = 0; + + Value * outputsVal = w.attrs->find(state.symbols.create("outputs"))->value; + foreach (DerivationOutputs::iterator, i, drv.outputs) { + mkString(*state.allocAttr(w, state.symbols.create(i->first)), + i->second.path, singleton<PathSet>("!" + i->first + "!" + path)); + mkString(*(outputsVal->list.elems[outputs_index++] = state.allocValue()), + i->first); + } + w.attrs->sort(); + Value fun; + state.evalFile(state.findFile("nix/imported-drv-to-derivation.nix"), fun); + state.forceFunction(fun); + mkApp(v, fun, w); + state.forceAttrs(v); + } else { + state.evalFile(path, v); + } +} + + +/* Return a string representing the type of the expression. */ +static void prim_typeOf(EvalState & state, Value * * args, Value & v) +{ + state.forceValue(*args[0]); + string t; + switch (args[0]->type) { + case tInt: t = "int"; break; + case tBool: t = "bool"; break; + case tString: t = "string"; break; + case tPath: t = "path"; break; + case tNull: t = "null"; break; + case tAttrs: t = "set"; break; + case tList: t = "list"; break; + case tLambda: + case tPrimOp: + case tPrimOpApp: + t = "lambda"; + break; + default: abort(); + } + mkString(v, state.symbols.create(t)); +} + + +/* Determine whether the argument is the null value. */ +static void prim_isNull(EvalState & state, Value * * args, Value & v) +{ + state.forceValue(*args[0]); + mkBool(v, args[0]->type == tNull); +} + + +/* Determine whether the argument is a function. */ +static void prim_isFunction(EvalState & state, Value * * args, Value & v) +{ + state.forceValue(*args[0]); + mkBool(v, args[0]->type == tLambda); +} + + +/* Determine whether the argument is an integer. */ +static void prim_isInt(EvalState & state, Value * * args, Value & v) +{ + state.forceValue(*args[0]); + mkBool(v, args[0]->type == tInt); +} + + +/* Determine whether the argument is a string. */ +static void prim_isString(EvalState & state, Value * * args, Value & v) +{ + state.forceValue(*args[0]); + mkBool(v, args[0]->type == tString); +} + + +/* Determine whether the argument is a Boolean. */ +static void prim_isBool(EvalState & state, Value * * args, Value & v) +{ + state.forceValue(*args[0]); + mkBool(v, args[0]->type == tBool); +} + + +struct CompareValues +{ + bool operator () (const Value * v1, const Value * v2) const + { + if (v1->type != v2->type) + throw EvalError("cannot compare values of different types"); + switch (v1->type) { + case tInt: + return v1->integer < v2->integer; + case tString: + return strcmp(v1->string.s, v2->string.s) < 0; + case tPath: + return strcmp(v1->path, v2->path) < 0; + default: + throw EvalError(format("cannot compare %1% with %2%") % showType(*v1) % showType(*v2)); + } + } +}; + + +#if HAVE_BOEHMGC +typedef list<Value *, gc_allocator<Value *> > ValueList; +#else +typedef list<Value *> ValueList; +#endif + + +static void prim_genericClosure(EvalState & state, Value * * args, Value & v) +{ + startNest(nest, lvlDebug, "finding dependencies"); + + state.forceAttrs(*args[0]); + + /* Get the start set. */ + Bindings::iterator startSet = + args[0]->attrs->find(state.symbols.create("startSet")); + if (startSet == args[0]->attrs->end()) + throw EvalError("attribute `startSet' required"); + state.forceList(*startSet->value); + + ValueList workSet; + for (unsigned int n = 0; n < startSet->value->list.length; ++n) + workSet.push_back(startSet->value->list.elems[n]); + + /* Get the operator. */ + Bindings::iterator op = + args[0]->attrs->find(state.symbols.create("operator")); + if (op == args[0]->attrs->end()) + throw EvalError("attribute `operator' required"); + state.forceValue(*op->value); + + /* Construct the closure by applying the operator to element of + `workSet', adding the result to `workSet', continuing until + no new elements are found. */ + ValueList res; + // `doneKeys' doesn't need to be a GC root, because its values are + // reachable from res. + set<Value *, CompareValues> doneKeys; + while (!workSet.empty()) { + Value * e = *(workSet.begin()); + workSet.pop_front(); + + state.forceAttrs(*e); + + Bindings::iterator key = + e->attrs->find(state.symbols.create("key")); + if (key == e->attrs->end()) + throw EvalError("attribute `key' required"); + state.forceValue(*key->value); + + if (doneKeys.find(key->value) != doneKeys.end()) continue; + doneKeys.insert(key->value); + res.push_back(e); + + /* Call the `operator' function with `e' as argument. */ + Value call; + mkApp(call, *op->value, *e); + state.forceList(call); + + /* Add the values returned by the operator to the work set. */ + for (unsigned int n = 0; n < call.list.length; ++n) { + state.forceValue(*call.list.elems[n]); + workSet.push_back(call.list.elems[n]); + } + } + + /* Create the result list. */ + state.mkList(v, res.size()); + unsigned int n = 0; + foreach (ValueList::iterator, i, res) + v.list.elems[n++] = *i; +} + + +static void prim_abort(EvalState & state, Value * * args, Value & v) +{ + PathSet context; + throw Abort(format("evaluation aborted with the following error message: `%1%'") % + state.coerceToString(*args[0], context)); +} + + +static void prim_throw(EvalState & state, Value * * args, Value & v) +{ + PathSet context; + throw ThrownError(format("%1%") % state.coerceToString(*args[0], context)); +} + + +static void prim_addErrorContext(EvalState & state, Value * * args, Value & v) +{ + try { + state.forceValue(*args[1]); + v = *args[1]; + } catch (Error & e) { + PathSet context; + e.addPrefix(format("%1%\n") % state.coerceToString(*args[0], context)); + throw; + } +} + + +/* Try evaluating the argument. Success => {success=true; value=something;}, + * else => {success=false; value=false;} */ +static void prim_tryEval(EvalState & state, Value * * args, Value & v) +{ + state.mkAttrs(v, 2); + try { + state.forceValue(*args[0]); + v.attrs->push_back(Attr(state.sValue, args[0])); + mkBool(*state.allocAttr(v, state.symbols.create("success")), true); + } catch (AssertionError & e) { + mkBool(*state.allocAttr(v, state.sValue), false); + mkBool(*state.allocAttr(v, state.symbols.create("success")), false); + } + v.attrs->sort(); +} + + +/* Return an environment variable. Use with care. */ +static void prim_getEnv(EvalState & state, Value * * args, Value & v) +{ + string name = state.forceStringNoCtx(*args[0]); + mkString(v, getEnv(name)); +} + + +/* Evaluate the first expression and print it on standard error. Then + return the second expression. Useful for debugging. */ +static void prim_trace(EvalState & state, Value * * args, Value & v) +{ + state.forceValue(*args[0]); + if (args[0]->type == tString) + printMsg(lvlError, format("trace: %1%") % args[0]->string.s); + else + printMsg(lvlError, format("trace: %1%") % *args[0]); + state.forceValue(*args[1]); + v = *args[1]; +} + + +/************************************************************* + * Derivations + *************************************************************/ + + +/* Construct (as a unobservable side effect) a Nix derivation + expression that performs the derivation described by the argument + set. Returns the original set extended with the following + attributes: `outPath' containing the primary output path of the + derivation; `drvPath' containing the path of the Nix expression; + and `type' set to `derivation' to indicate that this is a + derivation. */ +static void prim_derivationStrict(EvalState & state, Value * * args, Value & v) +{ + startNest(nest, lvlVomit, "evaluating derivation"); + + state.forceAttrs(*args[0]); + + /* Figure out the name first (for stack backtraces). */ + Bindings::iterator attr = args[0]->attrs->find(state.sName); + if (attr == args[0]->attrs->end()) + throw EvalError("required attribute `name' missing"); + string drvName; + Pos & posDrvName(*attr->pos); + try { + drvName = state.forceStringNoCtx(*attr->value); + } catch (Error & e) { + e.addPrefix(format("while evaluating the derivation attribute `name' at %1%:\n") % posDrvName); + throw; + } + + /* Check whether null attributes should be ignored. */ + bool ignoreNulls = false; + attr = args[0]->attrs->find(state.sIgnoreNulls); + if (attr != args[0]->attrs->end()) + ignoreNulls = state.forceBool(*attr->value); + + /* Build the derivation expression by processing the attributes. */ + Derivation drv; + + PathSet context; + + string outputHash, outputHashAlgo; + bool outputHashRecursive = false; + + StringSet outputs; + outputs.insert("out"); + + foreach (Bindings::iterator, i, *args[0]->attrs) { + if (i->name == state.sIgnoreNulls) continue; + string key = i->name; + startNest(nest, lvlVomit, format("processing attribute `%1%'") % key); + + try { + + if (ignoreNulls) { + state.forceValue(*i->value); + if (i->value->type == tNull) continue; + } + + /* The `args' attribute is special: it supplies the + command-line arguments to the builder. */ + if (key == "args") { + state.forceList(*i->value); + for (unsigned int n = 0; n < i->value->list.length; ++n) { + string s = state.coerceToString(*i->value->list.elems[n], context, true); + drv.args.push_back(s); + } + } + + /* All other attributes are passed to the builder through + the environment. */ + else { + string s = state.coerceToString(*i->value, context, true); + drv.env[key] = s; + if (key == "builder") drv.builder = s; + else if (i->name == state.sSystem) drv.platform = s; + else if (i->name == state.sName) { + drvName = s; + printMsg(lvlVomit, format("derivation name is `%1%'") % drvName); + } + else if (key == "outputHash") outputHash = s; + else if (key == "outputHashAlgo") outputHashAlgo = s; + else if (key == "outputHashMode") { + if (s == "recursive") outputHashRecursive = true; + else if (s == "flat") outputHashRecursive = false; + else throw EvalError(format("invalid value `%1%' for `outputHashMode' attribute") % s); + } + else if (key == "outputs") { + Strings tmp = tokenizeString<Strings>(s); + outputs.clear(); + foreach (Strings::iterator, j, tmp) { + if (outputs.find(*j) != outputs.end()) + throw EvalError(format("duplicate derivation output `%1%'") % *j); + /* !!! Check whether *j is a valid attribute + name. */ + /* Derivations cannot be named ‘drv’, because + then we'd have an attribute ‘drvPath’ in + the resulting set. */ + if (*j == "drv") + throw EvalError(format("invalid derivation output name `drv'") % *j); + outputs.insert(*j); + } + if (outputs.empty()) + throw EvalError("derivation cannot have an empty set of outputs"); + } + } + + } catch (Error & e) { + e.addPrefix(format("while evaluating the attribute `%1%' of the derivation `%2%' at %3%:\n") + % key % drvName % posDrvName); + throw; + } + } + + /* Everything in the context of the strings in the derivation + attributes should be added as dependencies of the resulting + derivation. */ + foreach (PathSet::iterator, i, context) { + Path path = *i; + + /* Paths marked with `=' denote that the path of a derivation + is explicitly passed to the builder. Since that allows the + builder to gain access to every path in the dependency + graph of the derivation (including all outputs), all paths + in the graph must be added to this derivation's list of + inputs to ensure that they are available when the builder + runs. */ + if (path.at(0) == '=') { + /* !!! This doesn't work if readOnlyMode is set. */ + PathSet refs; computeFSClosure(*store, string(path, 1), refs); + foreach (PathSet::iterator, j, refs) { + drv.inputSrcs.insert(*j); + if (isDerivation(*j)) + drv.inputDrvs[*j] = store->queryDerivationOutputNames(*j); + } + } + + /* See prim_unsafeDiscardOutputDependency. */ + else if (path.at(0) == '~') + drv.inputSrcs.insert(string(path, 1)); + + /* Handle derivation outputs of the form ‘!<name>!<path>’. */ + else if (path.at(0) == '!') { + std::pair<string, string> ctx = decodeContext(path); + drv.inputDrvs[ctx.first].insert(ctx.second); + } + + /* Handle derivation contexts returned by + ‘builtins.storePath’. */ + else if (isDerivation(path)) + drv.inputDrvs[path] = store->queryDerivationOutputNames(path); + + /* Otherwise it's a source file. */ + else + drv.inputSrcs.insert(path); + } + + /* Do we have all required attributes? */ + if (drv.builder == "") + throw EvalError("required attribute `builder' missing"); + if (drv.platform == "") + throw EvalError("required attribute `system' missing"); + + /* Check whether the derivation name is valid. */ + checkStoreName(drvName); + if (isDerivation(drvName)) + throw EvalError(format("derivation names are not allowed to end in `%1%'") + % drvExtension); + + if (outputHash != "") { + /* Handle fixed-output derivations. */ + if (outputs.size() != 1 || *(outputs.begin()) != "out") + throw Error("multiple outputs are not supported in fixed-output derivations"); + + HashType ht = parseHashType(outputHashAlgo); + if (ht == htUnknown) + throw EvalError(format("unknown hash algorithm `%1%'") % outputHashAlgo); + Hash h = parseHash16or32(ht, outputHash); + outputHash = printHash(h); + if (outputHashRecursive) outputHashAlgo = "r:" + outputHashAlgo; + + Path outPath = makeFixedOutputPath(outputHashRecursive, ht, h, drvName); + drv.env["out"] = outPath; + drv.outputs["out"] = DerivationOutput(outPath, outputHashAlgo, outputHash); + } + + else { + /* Construct the "masked" store derivation, which is the final + one except that in the list of outputs, the output paths + are empty, and the corresponding environment variables have + an empty value. This ensures that changes in the set of + output names do get reflected in the hash. */ + foreach (StringSet::iterator, i, outputs) { + drv.env[*i] = ""; + drv.outputs[*i] = DerivationOutput("", "", ""); + } + + /* Use the masked derivation expression to compute the output + path. */ + Hash h = hashDerivationModulo(*store, drv); + + foreach (DerivationOutputs::iterator, i, drv.outputs) + if (i->second.path == "") { + Path outPath = makeOutputPath(i->first, h, drvName); + drv.env[i->first] = outPath; + i->second.path = outPath; + } + } + + /* Write the resulting term into the Nix store directory. */ + Path drvPath = writeDerivation(*store, drv, drvName, state.repair); + + printMsg(lvlChatty, format("instantiated `%1%' -> `%2%'") + % drvName % drvPath); + + /* Optimisation, but required in read-only mode! because in that + case we don't actually write store derivations, so we can't + read them later. */ + drvHashes[drvPath] = hashDerivationModulo(*store, drv); + + state.mkAttrs(v, 1 + drv.outputs.size()); + mkString(*state.allocAttr(v, state.sDrvPath), drvPath, singleton<PathSet>("=" + drvPath)); + foreach (DerivationOutputs::iterator, i, drv.outputs) { + mkString(*state.allocAttr(v, state.symbols.create(i->first)), + i->second.path, singleton<PathSet>("!" + i->first + "!" + drvPath)); + } + v.attrs->sort(); +} + + +/************************************************************* + * Paths + *************************************************************/ + + +/* Convert the argument to a path. !!! obsolete? */ +static void prim_toPath(EvalState & state, Value * * args, Value & v) +{ + PathSet context; + Path path = state.coerceToPath(*args[0], context); + mkString(v, canonPath(path), context); +} + + +/* Allow a valid store path to be used in an expression. This is + useful in some generated expressions such as in nix-push, which + generates a call to a function with an already existing store path + as argument. You don't want to use `toPath' here because it copies + the path to the Nix store, which yields a copy like + /nix/store/newhash-oldhash-oldname. In the past, `toPath' had + special case behaviour for store paths, but that created weird + corner cases. */ +static void prim_storePath(EvalState & state, Value * * args, Value & v) +{ + PathSet context; + Path path = state.coerceToPath(*args[0], context); + /* Resolve symlinks in ‘path’, unless ‘path’ itself is a symlink + directly in the store. The latter condition is necessary so + e.g. nix-push does the right thing. */ + if (!isStorePath(path)) path = canonPath(path, true); + if (!isInStore(path)) + throw EvalError(format("path `%1%' is not in the Nix store") % path); + Path path2 = toStorePath(path); + if (!settings.readOnlyMode) + store->ensurePath(path2); + context.insert(path2); + mkString(v, path, context); +} + + +static void prim_pathExists(EvalState & state, Value * * args, Value & v) +{ + PathSet context; + Path path = state.coerceToPath(*args[0], context); + if (!context.empty()) + throw EvalError(format("string `%1%' cannot refer to other paths") % path); + mkBool(v, pathExists(path)); +} + + +/* Return the base name of the given string, i.e., everything + following the last slash. */ +static void prim_baseNameOf(EvalState & state, Value * * args, Value & v) +{ + PathSet context; + mkString(v, baseNameOf(state.coerceToString(*args[0], context)), context); +} + + +/* Return the directory of the given path, i.e., everything before the + last slash. Return either a path or a string depending on the type + of the argument. */ +static void prim_dirOf(EvalState & state, Value * * args, Value & v) +{ + PathSet context; + Path dir = dirOf(state.coerceToPath(*args[0], context)); + if (args[0]->type == tPath) mkPath(v, dir.c_str()); else mkString(v, dir, context); +} + + +/* Return the contents of a file as a string. */ +static void prim_readFile(EvalState & state, Value * * args, Value & v) +{ + PathSet context; + Path path = state.coerceToPath(*args[0], context); + if (!context.empty()) + throw EvalError(format("string `%1%' cannot refer to other paths") % path); + mkString(v, readFile(path).c_str()); +} + + +/************************************************************* + * Creating files + *************************************************************/ + + +/* Convert the argument (which can be any Nix expression) to an XML + representation returned in a string. Not all Nix expressions can + be sensibly or completely represented (e.g., functions). */ +static void prim_toXML(EvalState & state, Value * * args, Value & v) +{ + std::ostringstream out; + PathSet context; + printValueAsXML(state, true, false, *args[0], out, context); + mkString(v, out.str(), context); +} + + +/* Convert the argument (which can be any Nix expression) to a JSON + string. Not all Nix expressions can be sensibly or completely + represented (e.g., functions). */ +static void prim_toJSON(EvalState & state, Value * * args, Value & v) +{ + std::ostringstream out; + PathSet context; + printValueAsJSON(state, true, *args[0], out, context); + mkString(v, out.str(), context); +} + + +/* Store a string in the Nix store as a source file that can be used + as an input by derivations. */ +static void prim_toFile(EvalState & state, Value * * args, Value & v) +{ + PathSet context; + string name = state.forceStringNoCtx(*args[0]); + string contents = state.forceString(*args[1], context); + + PathSet refs; + + foreach (PathSet::iterator, i, context) { + Path path = *i; + if (path.at(0) == '=') path = string(path, 1); + if (isDerivation(path)) + throw EvalError(format("in `toFile': the file `%1%' cannot refer to derivation outputs") % name); + refs.insert(path); + } + + Path storePath = settings.readOnlyMode + ? computeStorePathForText(name, contents, refs) + : store->addTextToStore(name, contents, refs, state.repair); + + /* Note: we don't need to add `context' to the context of the + result, since `storePath' itself has references to the paths + used in args[1]. */ + + mkString(v, storePath, singleton<PathSet>(storePath)); +} + + +struct FilterFromExpr : PathFilter +{ + EvalState & state; + Value & filter; + + FilterFromExpr(EvalState & state, Value & filter) + : state(state), filter(filter) + { + } + + bool operator () (const Path & path) + { + struct stat st; + if (lstat(path.c_str(), &st)) + throw SysError(format("getting attributes of path `%1%'") % path); + + /* Call the filter function. The first argument is the path, + the second is a string indicating the type of the file. */ + Value arg1; + mkString(arg1, path); + + Value fun2; + state.callFunction(filter, arg1, fun2); + + Value arg2; + mkString(arg2, + S_ISREG(st.st_mode) ? "regular" : + S_ISDIR(st.st_mode) ? "directory" : + S_ISLNK(st.st_mode) ? "symlink" : + "unknown" /* not supported, will fail! */); + + Value res; + state.callFunction(fun2, arg2, res); + + return state.forceBool(res); + } +}; + + +static void prim_filterSource(EvalState & state, Value * * args, Value & v) +{ + PathSet context; + Path path = state.coerceToPath(*args[1], context); + if (!context.empty()) + throw EvalError(format("string `%1%' cannot refer to other paths") % path); + + state.forceValue(*args[0]); + if (args[0]->type != tLambda) + throw TypeError(format("first argument in call to `filterSource' is not a function but %1%") % showType(*args[0])); + + FilterFromExpr filter(state, *args[0]); + + Path dstPath = settings.readOnlyMode + ? computeStorePathForPath(path, true, htSHA256, filter).first + : store->addToStore(path, true, htSHA256, filter, state.repair); + + mkString(v, dstPath, singleton<PathSet>(dstPath)); +} + + +/************************************************************* + * Sets + *************************************************************/ + + +/* Return the names of the attributes in a set as a sorted list of + strings. */ +static void prim_attrNames(EvalState & state, Value * * args, Value & v) +{ + state.forceAttrs(*args[0]); + + state.mkList(v, args[0]->attrs->size()); + + StringSet names; + foreach (Bindings::iterator, i, *args[0]->attrs) + names.insert(i->name); + + unsigned int n = 0; + foreach (StringSet::iterator, i, names) + mkString(*(v.list.elems[n++] = state.allocValue()), *i); +} + + +/* Dynamic version of the `.' operator. */ +void prim_getAttr(EvalState & state, Value * * args, Value & v) +{ + string attr = state.forceStringNoCtx(*args[0]); + state.forceAttrs(*args[1]); + // !!! Should we create a symbol here or just do a lookup? + Bindings::iterator i = args[1]->attrs->find(state.symbols.create(attr)); + if (i == args[1]->attrs->end()) + throw EvalError(format("attribute `%1%' missing") % attr); + // !!! add to stack trace? + if (state.countCalls && i->pos) state.attrSelects[*i->pos]++; + state.forceValue(*i->value); + v = *i->value; +} + + +/* Return position information of the specified attribute. */ +void prim_unsafeGetAttrPos(EvalState & state, Value * * args, Value & v) +{ + string attr = state.forceStringNoCtx(*args[0]); + state.forceAttrs(*args[1]); + Bindings::iterator i = args[1]->attrs->find(state.symbols.create(attr)); + if (i == args[1]->attrs->end()) + mkNull(v); + else + state.mkPos(v, i->pos); +} + + +/* Dynamic version of the `?' operator. */ +static void prim_hasAttr(EvalState & state, Value * * args, Value & v) +{ + string attr = state.forceStringNoCtx(*args[0]); + state.forceAttrs(*args[1]); + mkBool(v, args[1]->attrs->find(state.symbols.create(attr)) != args[1]->attrs->end()); +} + + +/* Determine whether the argument is a set. */ +static void prim_isAttrs(EvalState & state, Value * * args, Value & v) +{ + state.forceValue(*args[0]); + mkBool(v, args[0]->type == tAttrs); +} + + +static void prim_removeAttrs(EvalState & state, Value * * args, Value & v) +{ + state.forceAttrs(*args[0]); + state.forceList(*args[1]); + + /* Get the attribute names to be removed. */ + std::set<Symbol> names; + for (unsigned int i = 0; i < args[1]->list.length; ++i) { + state.forceStringNoCtx(*args[1]->list.elems[i]); + names.insert(state.symbols.create(args[1]->list.elems[i]->string.s)); + } + + /* Copy all attributes not in that set. Note that we don't need + to sort v.attrs because it's a subset of an already sorted + vector. */ + state.mkAttrs(v, args[0]->attrs->size()); + foreach (Bindings::iterator, i, *args[0]->attrs) { + if (names.find(i->name) == names.end()) + v.attrs->push_back(*i); + } +} + + +/* Builds a set from a list specifying (name, value) pairs. To be + precise, a list [{name = "name1"; value = value1;} ... {name = + "nameN"; value = valueN;}] is transformed to {name1 = value1; + ... nameN = valueN;}. In case of duplicate occurences of the same + name, the first takes precedence. */ +static void prim_listToAttrs(EvalState & state, Value * * args, Value & v) +{ + state.forceList(*args[0]); + + state.mkAttrs(v, args[0]->list.length); + + std::set<Symbol> seen; + + for (unsigned int i = 0; i < args[0]->list.length; ++i) { + Value & v2(*args[0]->list.elems[i]); + state.forceAttrs(v2); + + Bindings::iterator j = v2.attrs->find(state.sName); + if (j == v2.attrs->end()) + throw TypeError("`name' attribute missing in a call to `listToAttrs'"); + string name = state.forceStringNoCtx(*j->value); + + Symbol sym = state.symbols.create(name); + if (seen.find(sym) == seen.end()) { + Bindings::iterator j2 = v2.attrs->find(state.symbols.create(state.sValue)); + if (j2 == v2.attrs->end()) + throw TypeError("`value' attribute missing in a call to `listToAttrs'"); + + v.attrs->push_back(Attr(sym, j2->value, j2->pos)); + seen.insert(sym); + } + } + + v.attrs->sort(); +} + + +/* Return the right-biased intersection of two sets as1 and as2, + i.e. a set that contains every attribute from as2 that is also a + member of as1. */ +static void prim_intersectAttrs(EvalState & state, Value * * args, Value & v) +{ + state.forceAttrs(*args[0]); + state.forceAttrs(*args[1]); + + state.mkAttrs(v, std::min(args[0]->attrs->size(), args[1]->attrs->size())); + + foreach (Bindings::iterator, i, *args[0]->attrs) { + Bindings::iterator j = args[1]->attrs->find(i->name); + if (j != args[1]->attrs->end()) + v.attrs->push_back(*j); + } +} + + +/* Return a set containing the names of the formal arguments expected + by the function `f'. The value of each attribute is a Boolean + denoting whether has a default value. For instance, + + functionArgs ({ x, y ? 123}: ...) + => { x = false; y = true; } + + "Formal argument" here refers to the attributes pattern-matched by + the function. Plain lambdas are not included, e.g. + + functionArgs (x: ...) + => { } +*/ +static void prim_functionArgs(EvalState & state, Value * * args, Value & v) +{ + state.forceValue(*args[0]); + if (args[0]->type != tLambda) + throw TypeError("`functionArgs' requires a function"); + + if (!args[0]->lambda.fun->matchAttrs) { + state.mkAttrs(v, 0); + return; + } + + state.mkAttrs(v, args[0]->lambda.fun->formals->formals.size()); + foreach (Formals::Formals_::iterator, i, args[0]->lambda.fun->formals->formals) + // !!! should optimise booleans (allocate only once) + mkBool(*state.allocAttr(v, i->name), i->def); + v.attrs->sort(); +} + + +/************************************************************* + * Lists + *************************************************************/ + + +/* Determine whether the argument is a list. */ +static void prim_isList(EvalState & state, Value * * args, Value & v) +{ + state.forceValue(*args[0]); + mkBool(v, args[0]->type == tList); +} + + +static void elemAt(EvalState & state, Value & list, int n, Value & v) +{ + state.forceList(list); + if (n < 0 || n >= list.list.length) + throw Error(format("list index %1% is out of bounds") % n); + state.forceValue(*list.list.elems[n]); + v = *list.list.elems[n]; +} + + +/* Return the n-1'th element of a list. */ +static void prim_elemAt(EvalState & state, Value * * args, Value & v) +{ + elemAt(state, *args[0], state.forceInt(*args[1]), v); +} + + +/* Return the first element of a list. */ +static void prim_head(EvalState & state, Value * * args, Value & v) +{ + elemAt(state, *args[0], 0, v); +} + + +/* Return a list consisting of everything but the the first element of + a list. Warning: this function takes O(n) time, so you probably + don't want to use it! */ +static void prim_tail(EvalState & state, Value * * args, Value & v) +{ + state.forceList(*args[0]); + if (args[0]->list.length == 0) + throw Error("`tail' called on an empty list"); + state.mkList(v, args[0]->list.length - 1); + for (unsigned int n = 0; n < v.list.length; ++n) + v.list.elems[n] = args[0]->list.elems[n + 1]; +} + + +/* Apply a function to every element of a list. */ +static void prim_map(EvalState & state, Value * * args, Value & v) +{ + state.forceFunction(*args[0]); + state.forceList(*args[1]); + + state.mkList(v, args[1]->list.length); + + for (unsigned int n = 0; n < v.list.length; ++n) + mkApp(*(v.list.elems[n] = state.allocValue()), + *args[0], *args[1]->list.elems[n]); +} + + +/* Filter a list using a predicate; that is, return a list containing + every element from the list for which the predicate function + returns true. */ +static void prim_filter(EvalState & state, Value * * args, Value & v) +{ + state.forceFunction(*args[0]); + state.forceList(*args[1]); + + // FIXME: putting this on the stack is risky. + Value * vs[args[1]->list.length]; + unsigned int k = 0; + + bool same = true; + for (unsigned int n = 0; n < args[1]->list.length; ++n) { + Value res; + state.callFunction(*args[0], *args[1]->list.elems[n], res); + if (state.forceBool(res)) + vs[k++] = args[1]->list.elems[n]; + else + same = false; + } + + if (same) + v = *args[1]; + else { + state.mkList(v, k); + for (unsigned int n = 0; n < k; ++n) v.list.elems[n] = vs[n]; + } +} + + +/* Return true if a list contains a given element. */ +static void prim_elem(EvalState & state, Value * * args, Value & v) +{ + bool res = false; + state.forceList(*args[1]); + for (unsigned int n = 0; n < args[1]->list.length; ++n) + if (state.eqValues(*args[0], *args[1]->list.elems[n])) { + res = true; + break; + } + mkBool(v, res); +} + + +/* Concatenate a list of lists. */ +static void prim_concatLists(EvalState & state, Value * * args, Value & v) +{ + state.forceList(*args[0]); + state.concatLists(v, args[0]->list.length, args[0]->list.elems); +} + + +/* Return the length of a list. This is an O(1) time operation. */ +static void prim_length(EvalState & state, Value * * args, Value & v) +{ + state.forceList(*args[0]); + mkInt(v, args[0]->list.length); +} + + +/************************************************************* + * Integer arithmetic + *************************************************************/ + + +static void prim_add(EvalState & state, Value * * args, Value & v) +{ + mkInt(v, state.forceInt(*args[0]) + state.forceInt(*args[1])); +} + + +static void prim_sub(EvalState & state, Value * * args, Value & v) +{ + mkInt(v, state.forceInt(*args[0]) - state.forceInt(*args[1])); +} + + +static void prim_mul(EvalState & state, Value * * args, Value & v) +{ + mkInt(v, state.forceInt(*args[0]) * state.forceInt(*args[1])); +} + + +static void prim_div(EvalState & state, Value * * args, Value & v) +{ + NixInt i2 = state.forceInt(*args[1]); + if (i2 == 0) throw EvalError("division by zero"); + mkInt(v, state.forceInt(*args[0]) / i2); +} + + +static void prim_lessThan(EvalState & state, Value * * args, Value & v) +{ + state.forceValue(*args[0]); + state.forceValue(*args[1]); + CompareValues comp; + mkBool(v, comp(args[0], args[1])); +} + + +/************************************************************* + * String manipulation + *************************************************************/ + + +/* Convert the argument to a string. Paths are *not* copied to the + store, so `toString /foo/bar' yields `"/foo/bar"', not + `"/nix/store/whatever..."'. */ +static void prim_toString(EvalState & state, Value * * args, Value & v) +{ + PathSet context; + string s = state.coerceToString(*args[0], context, true, false); + mkString(v, s, context); +} + + +/* `substring start len str' returns the substring of `str' starting + at character position `min(start, stringLength str)' inclusive and + ending at `min(start + len, stringLength str)'. `start' must be + non-negative. */ +static void prim_substring(EvalState & state, Value * * args, Value & v) +{ + int start = state.forceInt(*args[0]); + int len = state.forceInt(*args[1]); + PathSet context; + string s = state.coerceToString(*args[2], context); + + if (start < 0) throw EvalError("negative start position in `substring'"); + + mkString(v, start >= s.size() ? "" : string(s, start, len), context); +} + + +static void prim_stringLength(EvalState & state, Value * * args, Value & v) +{ + PathSet context; + string s = state.coerceToString(*args[0], context); + mkInt(v, s.size()); +} + + +static void prim_unsafeDiscardStringContext(EvalState & state, Value * * args, Value & v) +{ + PathSet context; + string s = state.coerceToString(*args[0], context); + mkString(v, s, PathSet()); +} + + +/* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a + builder without causing the derivation to be built (for instance, + in the derivation that builds NARs in nix-push, when doing + source-only deployment). This primop marks the string context so + that builtins.derivation adds the path to drv.inputSrcs rather than + drv.inputDrvs. */ +static void prim_unsafeDiscardOutputDependency(EvalState & state, Value * * args, Value & v) +{ + PathSet context; + string s = state.coerceToString(*args[0], context); + + PathSet context2; + foreach (PathSet::iterator, i, context) { + Path p = *i; + if (p.at(0) == '=') p = "~" + string(p, 1); + context2.insert(p); + } + + mkString(v, s, context2); +} + + +/* Return the cryptographic hash of a string in base-16. */ +static void prim_hashString(EvalState & state, Value * * args, Value & v) +{ + string type = state.forceStringNoCtx(*args[0]); + HashType ht = parseHashType(type); + if (ht == htUnknown) + throw Error(format("unknown hash type `%1%'") % type); + + PathSet context; // discarded + string s = state.forceString(*args[1], context); + + mkString(v, printHash(hashString(ht, s)), context); +}; + + +/************************************************************* + * Versions + *************************************************************/ + + +static void prim_parseDrvName(EvalState & state, Value * * args, Value & v) +{ + string name = state.forceStringNoCtx(*args[0]); + DrvName parsed(name); + state.mkAttrs(v, 2); + mkString(*state.allocAttr(v, state.sName), parsed.name); + mkString(*state.allocAttr(v, state.symbols.create("version")), parsed.version); + v.attrs->sort(); +} + + +static void prim_compareVersions(EvalState & state, Value * * args, Value & v) +{ + string version1 = state.forceStringNoCtx(*args[0]); + string version2 = state.forceStringNoCtx(*args[1]); + mkInt(v, compareVersions(version1, version2)); +} + + +/************************************************************* + * Primop registration + *************************************************************/ + + +void EvalState::createBaseEnv() +{ + baseEnv.up = 0; + + /* Add global constants such as `true' to the base environment. */ + Value v; + + /* `builtins' must be first! */ + mkAttrs(v, 128); + addConstant("builtins", v); + + mkBool(v, true); + addConstant("true", v); + + mkBool(v, false); + addConstant("false", v); + + mkNull(v); + addConstant("null", v); + + mkInt(v, time(0)); + addConstant("__currentTime", v); + + mkString(v, settings.thisSystem.c_str()); + addConstant("__currentSystem", v); + + mkString(v, nixVersion.c_str()); + addConstant("__nixVersion", v); + + /* Language version. This should be increased every time a new + language feature gets added. It's not necessary to increase it + when primops get added, because you can just use `builtins ? + primOp' to check. */ + mkInt(v, 2); + addConstant("__langVersion", v); + + // Miscellaneous + addPrimOp("import", 1, prim_import); + addPrimOp("__typeOf", 1, prim_typeOf); + addPrimOp("isNull", 1, prim_isNull); + addPrimOp("__isFunction", 1, prim_isFunction); + addPrimOp("__isString", 1, prim_isString); + addPrimOp("__isInt", 1, prim_isInt); + addPrimOp("__isBool", 1, prim_isBool); + addPrimOp("__genericClosure", 1, prim_genericClosure); + addPrimOp("abort", 1, prim_abort); + addPrimOp("throw", 1, prim_throw); + addPrimOp("__addErrorContext", 2, prim_addErrorContext); + addPrimOp("__tryEval", 1, prim_tryEval); + addPrimOp("__getEnv", 1, prim_getEnv); + addPrimOp("__trace", 2, prim_trace); + + // Paths + addPrimOp("__toPath", 1, prim_toPath); + addPrimOp("__storePath", 1, prim_storePath); + addPrimOp("__pathExists", 1, prim_pathExists); + addPrimOp("baseNameOf", 1, prim_baseNameOf); + addPrimOp("dirOf", 1, prim_dirOf); + addPrimOp("__readFile", 1, prim_readFile); + + // Creating files + addPrimOp("__toXML", 1, prim_toXML); + addPrimOp("__toJSON", 1, prim_toJSON); + addPrimOp("__toFile", 2, prim_toFile); + addPrimOp("__filterSource", 2, prim_filterSource); + + // Sets + addPrimOp("__attrNames", 1, prim_attrNames); + addPrimOp("__getAttr", 2, prim_getAttr); + addPrimOp("__unsafeGetAttrPos", 2, prim_unsafeGetAttrPos); + addPrimOp("__hasAttr", 2, prim_hasAttr); + addPrimOp("__isAttrs", 1, prim_isAttrs); + addPrimOp("removeAttrs", 2, prim_removeAttrs); + addPrimOp("__listToAttrs", 1, prim_listToAttrs); + addPrimOp("__intersectAttrs", 2, prim_intersectAttrs); + addPrimOp("__functionArgs", 1, prim_functionArgs); + + // Lists + addPrimOp("__isList", 1, prim_isList); + addPrimOp("__elemAt", 2, prim_elemAt); + addPrimOp("__head", 1, prim_head); + addPrimOp("__tail", 1, prim_tail); + addPrimOp("map", 2, prim_map); + addPrimOp("__filter", 2, prim_filter); + addPrimOp("__elem", 2, prim_elem); + addPrimOp("__concatLists", 1, prim_concatLists); + addPrimOp("__length", 1, prim_length); + + // Integer arithmetic + addPrimOp("__add", 2, prim_add); + addPrimOp("__sub", 2, prim_sub); + addPrimOp("__mul", 2, prim_mul); + addPrimOp("__div", 2, prim_div); + addPrimOp("__lessThan", 2, prim_lessThan); + + // String manipulation + addPrimOp("toString", 1, prim_toString); + addPrimOp("__substring", 3, prim_substring); + addPrimOp("__stringLength", 1, prim_stringLength); + addPrimOp("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext); + addPrimOp("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency); + addPrimOp("__hashString", 2, prim_hashString); + + // Versions + addPrimOp("__parseDrvName", 1, prim_parseDrvName); + addPrimOp("__compareVersions", 2, prim_compareVersions); + + // Derivations + addPrimOp("derivationStrict", 1, prim_derivationStrict); + + /* Add a wrapper around the derivation primop that computes the + `drvPath' and `outPath' attributes lazily. */ + string path = findFile("nix/derivation.nix"); + assert(!path.empty()); + sDerivationNix = symbols.create(path); + evalFile(path, v); + addConstant("derivation", v); + + /* Now that we've added all primops, sort the `builtins' set, + because attribute lookups expect it to be sorted. */ + baseEnv.values[0]->attrs->sort(); +} + + +} diff --git a/src/libexpr/symbol-table.hh b/src/libexpr/symbol-table.hh new file mode 100644 index 000000000000..08e31d496585 --- /dev/null +++ b/src/libexpr/symbol-table.hh @@ -0,0 +1,95 @@ +#pragma once + +#include "config.h" + +#include <map> + +#if HAVE_TR1_UNORDERED_SET +#include <tr1/unordered_set> +#endif + +#include "types.hh" + +namespace nix { + +/* Symbol table used by the parser and evaluator to represent and look + up identifiers and attributes efficiently. SymbolTable::create() + converts a string into a symbol. Symbols have the property that + they can be compared efficiently (using a pointer equality test), + because the symbol table stores only one copy of each string. */ + +class Symbol +{ +private: + const string * s; // pointer into SymbolTable + Symbol(const string * s) : s(s) { }; + friend class SymbolTable; + +public: + Symbol() : s(0) { }; + + bool operator == (const Symbol & s2) const + { + return s == s2.s; + } + + bool operator != (const Symbol & s2) const + { + return s != s2.s; + } + + bool operator < (const Symbol & s2) const + { + return s < s2.s; + } + + operator const string & () const + { + return *s; + } + + bool set() const + { + return s; + } + + bool empty() const + { + return s->empty(); + } + + friend std::ostream & operator << (std::ostream & str, const Symbol & sym); +}; + +inline std::ostream & operator << (std::ostream & str, const Symbol & sym) +{ + str << *sym.s; + return str; +} + +class SymbolTable +{ +private: +#if HAVE_TR1_UNORDERED_SET + typedef std::tr1::unordered_set<string> Symbols; +#else + typedef std::set<string> Symbols; +#endif + Symbols symbols; + +public: + Symbol create(const string & s) + { + std::pair<Symbols::iterator, bool> res = symbols.insert(s); + return Symbol(&*res.first); + } + + unsigned int size() const + { + return symbols.size(); + } + + size_t totalSize() const; +}; + +} diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc new file mode 100644 index 000000000000..a2004df5c8c5 --- /dev/null +++ b/src/libexpr/value-to-json.cc @@ -0,0 +1,86 @@ +#include "value-to-json.hh" +#include "eval-inline.hh" +#include "util.hh" + +#include <cstdlib> + + +namespace nix { + + +void escapeJSON(std::ostream & str, const string & s) +{ + str << "\""; + foreach (string::const_iterator, i, s) + if (*i == '\"' || *i == '\\') str << "\\" << *i; + else if (*i == '\n') str << "\\n"; + else if (*i == '\r') str << "\\r"; + else if (*i == '\t') str << "\\t"; + else str << *i; + str << "\""; +} + + +void printValueAsJSON(EvalState & state, bool strict, + Value & v, std::ostream & str, PathSet & context) +{ + checkInterrupt(); + + if (strict) state.forceValue(v); + + switch (v.type) { + + case tInt: + str << v.integer; + break; + + case tBool: + str << (v.boolean ? "true" : "false"); + break; + + case tString: + copyContext(v, context); + escapeJSON(str, v.string.s); + break; + + case tPath: + escapeJSON(str, state.copyPathToStore(context, v.path)); + break; + + case tNull: + str << "null"; + break; + + case tAttrs: { + Bindings::iterator i = v.attrs->find(state.sOutPath); + if (i == v.attrs->end()) { + JSONObject json(str); + StringSet names; + foreach (Bindings::iterator, i, *v.attrs) + names.insert(i->name); + foreach (StringSet::iterator, i, names) { + Attr & a(*v.attrs->find(state.symbols.create(*i))); + json.attr(*i); + printValueAsJSON(state, strict, *a.value, str, context); + } + } else + printValueAsJSON(state, strict, *i->value, str, context); + break; + } + + case tList: { + JSONList json(str); + for (unsigned int n = 0; n < v.list.length; ++n) { + json.elem(); + printValueAsJSON(state, strict, *v.list.elems[n], str, context); + } + break; + } + + default: + throw TypeError(format("cannot convert %1% to JSON") % showType(v)); + } +} + + +} diff --git a/src/libexpr/value-to-json.hh b/src/libexpr/value-to-json.hh new file mode 100644 index 000000000000..e3a97efe4269 --- /dev/null +++ b/src/libexpr/value-to-json.hh @@ -0,0 +1,64 @@ +#pragma once + +#include "nixexpr.hh" +#include "eval.hh" + +#include <string> +#include <map> + +namespace nix { + +void printValueAsJSON(EvalState & state, bool strict, + Value & v, std::ostream & out, PathSet & context); + +void escapeJSON(std::ostream & str, const string & s); + +struct JSONObject +{ + std::ostream & str; + bool first; + JSONObject(std::ostream & str) : str(str), first(true) + { + str << "{"; + } + ~JSONObject() + { + str << "}"; + } + void attr(const string & s) + { + if (!first) str << ","; else first = false; + escapeJSON(str, s); + str << ":"; + } + void attr(const string & s, const string & t) + { + attr(s); + escapeJSON(str, t); + } +}; + +struct JSONList +{ + std::ostream & str; + bool first; + JSONList(std::ostream & str) : str(str), first(true) + { + str << "["; + } + ~JSONList() + { + str << "]"; + } + void elem() + { + if (!first) str << ","; else first = false; + } + void elem(const string & s) + { + elem(); + escapeJSON(str, s); + } +}; + +} diff --git a/src/libexpr/value-to-xml.cc b/src/libexpr/value-to-xml.cc new file mode 100644 index 000000000000..3934a83eec25 --- /dev/null +++ b/src/libexpr/value-to-xml.cc @@ -0,0 +1,163 @@ +#include "value-to-xml.hh" +#include "xml-writer.hh" +#include "eval-inline.hh" +#include "util.hh" + +#include <cstdlib> + + +namespace nix { + + +static XMLAttrs singletonAttrs(const string & name, const string & value) +{ + XMLAttrs attrs; + attrs[name] = value; + return attrs; +} + + +static void printValueAsXML(EvalState & state, bool strict, bool location, + Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen); + + +static void posToXML(XMLAttrs & xmlAttrs, const Pos & pos) +{ + xmlAttrs["path"] = pos.file; + xmlAttrs["line"] = (format("%1%") % pos.line).str(); + xmlAttrs["column"] = (format("%1%") % pos.column).str(); +} + + +static void showAttrs(EvalState & state, bool strict, bool location, + Bindings & attrs, XMLWriter & doc, PathSet & context, PathSet & drvsSeen) +{ + StringSet names; + + foreach (Bindings::iterator, i, attrs) + names.insert(i->name); + + foreach (StringSet::iterator, i, names) { + Attr & a(*attrs.find(state.symbols.create(*i))); + + XMLAttrs xmlAttrs; + xmlAttrs["name"] = *i; + if (location && a.pos != &noPos) posToXML(xmlAttrs, *a.pos); + + XMLOpenElement _(doc, "attr", xmlAttrs); + printValueAsXML(state, strict, location, + *a.value, doc, context, drvsSeen); + } +} + + +static void printValueAsXML(EvalState & state, bool strict, bool location, + Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen) +{ + checkInterrupt(); + + if (strict) state.forceValue(v); + + switch (v.type) { + + case tInt: + doc.writeEmptyElement("int", singletonAttrs("value", (format("%1%") % v.integer).str())); + break; + + case tBool: + doc.writeEmptyElement("bool", singletonAttrs("value", v.boolean ? "true" : "false")); + break; + + case tString: + /* !!! show the context? */ + copyContext(v, context); + doc.writeEmptyElement("string", singletonAttrs("value", v.string.s)); + break; + + case tPath: + doc.writeEmptyElement("path", singletonAttrs("value", v.path)); + break; + + case tNull: + doc.writeEmptyElement("null"); + break; + + case tAttrs: + if (state.isDerivation(v)) { + XMLAttrs xmlAttrs; + + Bindings::iterator a = v.attrs->find(state.symbols.create("derivation")); + + Path drvPath; + a = v.attrs->find(state.sDrvPath); + if (a != v.attrs->end()) { + if (strict) state.forceValue(*a->value); + if (a->value->type == tString) + xmlAttrs["drvPath"] = drvPath = a->value->string.s; + } + + a = v.attrs->find(state.sOutPath); + if (a != v.attrs->end()) { + if (strict) state.forceValue(*a->value); + if (a->value->type == tString) + xmlAttrs["outPath"] = a->value->string.s; + } + + XMLOpenElement _(doc, "derivation", xmlAttrs); + + if (drvPath != "" && drvsSeen.find(drvPath) == drvsSeen.end()) { + drvsSeen.insert(drvPath); + showAttrs(state, strict, location, *v.attrs, doc, context, drvsSeen); + } else + doc.writeEmptyElement("repeated"); + } + + else { + XMLOpenElement _(doc, "attrs"); + showAttrs(state, strict, location, *v.attrs, doc, context, drvsSeen); + } + + break; + + case tList: { + XMLOpenElement _(doc, "list"); + for (unsigned int n = 0; n < v.list.length; ++n) + printValueAsXML(state, strict, location, *v.list.elems[n], doc, context, drvsSeen); + break; + } + + case tLambda: { + XMLAttrs xmlAttrs; + if (location) posToXML(xmlAttrs, v.lambda.fun->pos); + XMLOpenElement _(doc, "function", xmlAttrs); + + if (v.lambda.fun->matchAttrs) { + XMLAttrs attrs; + if (!v.lambda.fun->arg.empty()) attrs["name"] = v.lambda.fun->arg; + if (v.lambda.fun->formals->ellipsis) attrs["ellipsis"] = "1"; + XMLOpenElement _(doc, "attrspat", attrs); + foreach (Formals::Formals_::iterator, i, v.lambda.fun->formals->formals) + doc.writeEmptyElement("attr", singletonAttrs("name", i->name)); + } else + doc.writeEmptyElement("varpat", singletonAttrs("name", v.lambda.fun->arg)); + + break; + } + + default: + doc.writeEmptyElement("unevaluated"); + } +} + + +void printValueAsXML(EvalState & state, bool strict, bool location, + Value & v, std::ostream & out, PathSet & context) +{ + XMLWriter doc(true, out); + XMLOpenElement root(doc, "expr"); + PathSet drvsSeen; + printValueAsXML(state, strict, location, v, doc, context, drvsSeen); +} + + +} diff --git a/src/libexpr/value-to-xml.hh b/src/libexpr/value-to-xml.hh new file mode 100644 index 000000000000..97657327edba --- /dev/null +++ b/src/libexpr/value-to-xml.hh @@ -0,0 +1,14 @@ +#pragma once + +#include "nixexpr.hh" +#include "eval.hh" + +#include <string> +#include <map> + +namespace nix { + +void printValueAsXML(EvalState & state, bool strict, bool location, + Value & v, std::ostream & out, PathSet & context); + +} diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh new file mode 100644 index 000000000000..2feb2f9492ca --- /dev/null +++ b/src/libexpr/value.hh @@ -0,0 +1,162 @@ +#pragma once + +#include "symbol-table.hh" + +namespace nix { + + +typedef enum { + tInt = 1, + tBool, + tString, + tPath, + tNull, + tAttrs, + tList, + tThunk, + tApp, + tLambda, + tBlackhole, + tPrimOp, + tPrimOpApp, +} ValueType; + + +class Bindings; +struct Env; +struct Expr; +struct ExprLambda; +struct PrimOp; +struct PrimOp; +class Symbol; + + +typedef long NixInt; + + +struct Value +{ + ValueType type; + union + { + NixInt integer; + bool boolean; + + /* Strings in the evaluator carry a so-called `context' which + is a list of strings representing store paths. This is to + allow users to write things like + + "--with-freetype2-library=" + freetype + "/lib" + + where `freetype' is a derivation (or a source to be copied + to the store). If we just concatenated the strings without + keeping track of the referenced store paths, then if the + string is used as a derivation attribute, the derivation + will not have the correct dependencies in its inputDrvs and + inputSrcs. + + The semantics of the context is as follows: when a string + with context C is used as a derivation attribute, then the + derivations in C will be added to the inputDrvs of the + derivation, and the other store paths in C will be added to + the inputSrcs of the derivations. + + For canonicity, the store paths should be in sorted order. */ + struct { + const char * s; + const char * * context; // must be in sorted order + } string; + + const char * path; + Bindings * attrs; + struct { + unsigned int length; + Value * * elems; + } list; + struct { + Env * env; + Expr * expr; + } thunk; + struct { + Value * left, * right; + } app; + struct { + Env * env; + ExprLambda * fun; + } lambda; + PrimOp * primOp; + struct { + Value * left, * right; + } primOpApp; + }; +}; + + +/* After overwriting an app node, be sure to clear pointers in the + Value to ensure that the target isn't kept alive unnecessarily. */ +static inline void clearValue(Value & v) +{ + v.app.right = 0; +} + + +static inline void mkInt(Value & v, NixInt n) +{ + clearValue(v); + v.type = tInt; + v.integer = n; +} + + +static inline void mkBool(Value & v, bool b) +{ + clearValue(v); + v.type = tBool; + v.boolean = b; +} + + +static inline void mkNull(Value & v) +{ + v.type = tNull; + v.app.left = v.app.right = 00; // scrub +} + + +static inline void mkApp(Value & v, Value & left, Value & right) +{ + v.type = tApp; + v.app.left = &left; + v.app.right = &right; +} + + +static inline void mkStringNoCopy(Value & v, const char * s) +{ + v.type = tString; + v.string.s = s; + v.string.context = 0; +} + + +static inline void mkString(Value & v, const Symbol & s) +{ + mkStringNoCopy(v, ((const string &) s).c_str()); +} + + +void mkString(Value & v, const char * s); + + +static inline void mkPathNoCopy(Value & v, const char * s) +{ + clearValue(v); + v.type = tPath; + v.path = s; +} + + +void mkPath(Value & v, const char * s); + + +} diff --git a/src/libmain/local.mk b/src/libmain/local.mk new file mode 100644 index 000000000000..71a07d1979ab --- /dev/null +++ b/src/libmain/local.mk @@ -0,0 +1,11 @@ +libraries += libmain + +libmain_NAME = libnixmain + +libmain_DIR := $(d) + +libmain_SOURCES := $(wildcard $(d)/*.cc) + +libmain_LIBS = libstore libutil libformat + +libmain_ALLOW_UNDEFINED = 1 diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc new file mode 100644 index 000000000000..fb70cb076732 --- /dev/null +++ b/src/libmain/shared.cc @@ -0,0 +1,297 @@ +#include "config.h" + +#include "shared.hh" +#include "globals.hh" +#include "store-api.hh" +#include "util.hh" +#include "misc.hh" + +#include <iostream> +#include <cctype> +#include <exception> + +#include <sys/time.h> +#include <sys/stat.h> +#include <unistd.h> +#include <signal.h> + + +namespace nix { + + +volatile sig_atomic_t blockInt = 0; + + +static void sigintHandler(int signo) +{ + if (!blockInt) { + _isInterrupted = 1; + blockInt = 1; + } +} + + +static bool gcWarning = true; + +void printGCWarning() +{ + if (!gcWarning) return; + static bool haveWarned = false; + warnOnce(haveWarned, + "you did not specify `--add-root'; " + "the result might be removed by the garbage collector"); +} + + +void printMissing(StoreAPI & store, const PathSet & paths) +{ + unsigned long long downloadSize, narSize; + PathSet willBuild, willSubstitute, unknown; + queryMissing(store, paths, willBuild, willSubstitute, unknown, downloadSize, narSize); + printMissing(willBuild, willSubstitute, unknown, downloadSize, narSize); +} + + +void printMissing(const PathSet & willBuild, + const PathSet & willSubstitute, const PathSet & unknown, + unsigned long long downloadSize, unsigned long long narSize) +{ + if (!willBuild.empty()) { + printMsg(lvlInfo, format("these derivations will be built:")); + foreach (PathSet::iterator, i, willBuild) + printMsg(lvlInfo, format(" %1%") % *i); + } + + if (!willSubstitute.empty()) { + printMsg(lvlInfo, format("these paths will be fetched (%.2f MiB download, %.2f MiB unpacked):") + % (downloadSize / (1024.0 * 1024.0)) + % (narSize / (1024.0 * 1024.0))); + foreach (PathSet::iterator, i, willSubstitute) + printMsg(lvlInfo, format(" %1%") % *i); + } + + if (!unknown.empty()) { + printMsg(lvlInfo, format("don't know how to build these paths%1%:") + % (settings.readOnlyMode ? " (may be caused by read-only store access)" : "")); + foreach (PathSet::iterator, i, unknown) + printMsg(lvlInfo, format(" %1%") % *i); + } +} + + +static void setLogType(string lt) +{ + if (lt == "pretty") logType = ltPretty; + else if (lt == "escapes") logType = ltEscapes; + else if (lt == "flat") logType = ltFlat; + else throw UsageError("unknown log type"); +} + + +string getArg(const string & opt, + Strings::iterator & i, const Strings::iterator & end) +{ + ++i; + if (i == end) throw UsageError(format("`%1%' requires an argument") % opt); + return *i; +} + + +void detectStackOverflow(); + + +/* Initialize and reorder arguments, then call the actual argument + processor. */ +static void initAndRun(int argc, char * * argv) +{ + settings.processEnvironment(); + settings.loadConfFile(); + + /* Catch SIGINT. */ + struct sigaction act; + act.sa_handler = sigintHandler; + sigfillset(&act.sa_mask); + act.sa_flags = 0; + if (sigaction(SIGINT, &act, 0)) + throw SysError("installing handler for SIGINT"); + if (sigaction(SIGTERM, &act, 0)) + throw SysError("installing handler for SIGTERM"); + if (sigaction(SIGHUP, &act, 0)) + throw SysError("installing handler for SIGHUP"); + + /* Ignore SIGPIPE. */ + act.sa_handler = SIG_IGN; + act.sa_flags = 0; + if (sigaction(SIGPIPE, &act, 0)) + throw SysError("ignoring SIGPIPE"); + + /* Reset SIGCHLD to its default. */ + act.sa_handler = SIG_DFL; + act.sa_flags = 0; + if (sigaction(SIGCHLD, &act, 0)) + throw SysError("resetting SIGCHLD"); + + /* Register a SIGSEGV handler to detect stack overflows. */ + detectStackOverflow(); + + /* There is no privacy in the Nix system ;-) At least not for + now. In particular, store objects should be readable by + everybody. */ + umask(0022); + + /* Initialise the PRNG. */ + struct timeval tv; + gettimeofday(&tv, 0); + srandom(tv.tv_usec); + + /* Process the NIX_LOG_TYPE environment variable. */ + string lt = getEnv("NIX_LOG_TYPE"); + if (lt != "") setLogType(lt); + + /* Put the arguments in a vector. */ + Strings args, remaining; + while (argc--) args.push_back(*argv++); + args.erase(args.begin()); + + /* Expand compound dash options (i.e., `-qlf' -> `-q -l -f'), and + ignore options for the ATerm library. */ + for (Strings::iterator i = args.begin(); i != args.end(); ++i) { + string arg = *i; + if (arg.length() > 2 && arg[0] == '-' && arg[1] != '-' && !isdigit(arg[1])) { + for (unsigned int j = 1; j < arg.length(); j++) + if (isalpha(arg[j])) + remaining.push_back((string) "-" + arg[j]); + else { + remaining.push_back(string(arg, j)); + break; + } + } else remaining.push_back(arg); + } + args = remaining; + remaining.clear(); + + /* Process default options. */ + for (Strings::iterator i = args.begin(); i != args.end(); ++i) { + string arg = *i; + if (arg == "--verbose" || arg == "-v") verbosity = (Verbosity) (verbosity + 1); + else if (arg == "--quiet") verbosity = verbosity > lvlError ? (Verbosity) (verbosity - 1) : lvlError; + else if (arg == "--log-type") { + string s = getArg(arg, i, args.end()); + setLogType(s); + } + else if (arg == "--no-build-output" || arg == "-Q") + settings.buildVerbosity = lvlVomit; + else if (arg == "--print-build-trace") + settings.printBuildTrace = true; + else if (arg == "--help") { + printHelp(); + return; + } + else if (arg == "--version") { + std::cout << format("%1% (Nix) %2%") % programId % nixVersion << std::endl; + return; + } + else if (arg == "--keep-failed" || arg == "-K") + settings.keepFailed = true; + else if (arg == "--keep-going" || arg == "-k") + settings.keepGoing = true; + else if (arg == "--fallback") + settings.set("build-fallback", "true"); + else if (arg == "--max-jobs" || arg == "-j") + settings.set("build-max-jobs", getArg(arg, i, args.end())); + else if (arg == "--cores") + settings.set("build-cores", getArg(arg, i, args.end())); + else if (arg == "--readonly-mode") + settings.readOnlyMode = true; + else if (arg == "--max-silent-time") + settings.set("build-max-silent-time", getArg(arg, i, args.end())); + else if (arg == "--timeout") + settings.set("build-timeout", getArg(arg, i, args.end())); + else if (arg == "--no-build-hook") + settings.useBuildHook = false; + else if (arg == "--show-trace") + settings.showTrace = true; + else if (arg == "--no-gc-warning") + gcWarning = false; + else if (arg == "--option") { + ++i; if (i == args.end()) throw UsageError("`--option' requires two arguments"); + string name = *i; + ++i; if (i == args.end()) throw UsageError("`--option' requires two arguments"); + string value = *i; + settings.set(name, value); + } + else remaining.push_back(arg); + } + + settings.update(); + + run(remaining); + + /* Close the Nix database. */ + store.reset((StoreAPI *) 0); +} + + +void showManPage(const string & name) +{ + string cmd = "man " + name; + if (system(cmd.c_str()) != 0) + throw Error(format("command `%1%' failed") % cmd); +} + + +int exitCode = 0; +char * * argvSaved = 0; + +} + + +static char buf[1024]; + +int main(int argc, char * * argv) +{ + using namespace nix; + + argvSaved = argv; + + /* Turn on buffering for cerr. */ +#if HAVE_PUBSETBUF + std::cerr.rdbuf()->pubsetbuf(buf, sizeof(buf)); +#endif + + std::ios::sync_with_stdio(false); + + try { + try { + initAndRun(argc, argv); + } catch (...) { + /* Subtle: we have to make sure that any `interrupted' + condition is discharged before we reach printMsg() + below, since otherwise it will throw an (uncaught) + exception. */ + blockInt = 1; /* ignore further SIGINTs */ + _isInterrupted = 0; + throw; + } + } catch (UsageError & e) { + printMsg(lvlError, + format( + "error: %1%\n" + "Try `%2% --help' for more information.") + % e.what() % programId); + return 1; + } catch (BaseError & e) { + printMsg(lvlError, format("error: %1%%2%") % (settings.showTrace ? e.prefix() : "") % e.msg()); + if (e.prefix() != "" && !settings.showTrace) + printMsg(lvlError, "(use `--show-trace' to show detailed location information)"); + return e.status; + } catch (std::bad_alloc & e) { + printMsg(lvlError, "error: out of memory"); + return 1; + } catch (std::exception & e) { + printMsg(lvlError, format("error: %1%") % e.what()); + return 1; + } + + return exitCode; +} diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh new file mode 100644 index 000000000000..ff89e86389c2 --- /dev/null +++ b/src/libmain/shared.hh @@ -0,0 +1,58 @@ +#pragma once + +#include "util.hh" + +#include <signal.h> + + +/* These are not implemented here, but must be implemented by a + program linking against libmain. */ + +/* Main program. Called by main() after the ATerm library has been + initialised and some default arguments have been processed (and + removed from `args'). main() will catch all exceptions. */ +void run(nix::Strings args); + +/* Should print a help message to stdout and return. */ +void printHelp(); + +extern std::string programId; + + +namespace nix { + +MakeError(UsageError, nix::Error); + +class StoreAPI; + +/* Ugh. No better place to put this. */ +void printGCWarning(); + +void printMissing(StoreAPI & store, const PathSet & paths); + +void printMissing(const PathSet & willBuild, + const PathSet & willSubstitute, const PathSet & unknown, + unsigned long long downloadSize, unsigned long long narSize); + +template<class N> N getIntArg(const string & opt, + Strings::iterator & i, const Strings::iterator & end) +{ + ++i; + if (i == end) throw UsageError(format("`%1%' requires an argument") % opt); + N n; + if (!string2Int(*i, n)) + throw UsageError(format("`%1%' requires an integer argument") % opt); + return n; +} + +/* Show the manual page for the specified program. */ +void showManPage(const string & name); + +extern volatile ::sig_atomic_t blockInt; + +/* Exit code of the program. */ +extern int exitCode; + +extern char * * argvSaved; + +} diff --git a/src/libmain/stack.cc b/src/libmain/stack.cc new file mode 100644 index 000000000000..64df95547e0b --- /dev/null +++ b/src/libmain/stack.cc @@ -0,0 +1,72 @@ +#include "config.h" + +#include "types.hh" + +#include <cstring> +#include <cstddef> +#include <cstdlib> + +#include <unistd.h> +#include <signal.h> + +namespace nix { + + +static void sigsegvHandler(int signo, siginfo_t * info, void * ctx) +{ + /* Detect stack overflows by comparing the faulting address with + the stack pointer. Unfortunately, getting the stack pointer is + not portable. */ + bool haveSP = true; + char * sp; +#if defined(__x86_64__) && defined(REG_RSP) + sp = (char *) ((ucontext *) ctx)->uc_mcontext.gregs[REG_RSP]; +#elif defined(REG_ESP) + sp = (char *) ((ucontext *) ctx)->uc_mcontext.gregs[REG_ESP]; +#else + haveSP = false; +#endif + + if (haveSP) { + ptrdiff_t diff = (char *) info->si_addr - sp; + if (diff < 0) diff = -diff; + if (diff < 4096) { + char msg[] = "error: stack overflow (possible infinite recursion)\n"; + write(2, msg, strlen(msg)); + _exit(1); // maybe abort instead? + } + } + + /* Restore default behaviour (i.e. segfault and dump core). */ + struct sigaction act; + sigfillset(&act.sa_mask); + act.sa_handler = SIG_DFL; + act.sa_flags = 0; + if (sigaction(SIGSEGV, &act, 0)) abort(); +} + + +void detectStackOverflow() +{ +#if defined(SA_SIGINFO) && defined (SA_ONSTACK) + /* Install a SIGSEGV handler to detect stack overflows. This + requires an alternative stack, otherwise the signal cannot be + delivered when we're out of stack space. */ + stack_t stack; + stack.ss_size = 4096 * 4 + MINSIGSTKSZ; + stack.ss_sp = new char[stack.ss_size]; + if (!stack.ss_sp) throw Error("cannot allocate alternative stack"); + stack.ss_flags = 0; + if (sigaltstack(&stack, 0) == -1) throw SysError("cannot set alternative stack"); + + struct sigaction act; + sigfillset(&act.sa_mask); + act.sa_sigaction = sigsegvHandler; + act.sa_flags = SA_SIGINFO | SA_ONSTACK; + if (sigaction(SIGSEGV, &act, 0)) + throw SysError("resetting SIGCHLD"); +#endif +} + + +} diff --git a/src/libstore/build.cc b/src/libstore/build.cc new file mode 100644 index 000000000000..1b379752fa0e --- /dev/null +++ b/src/libstore/build.cc @@ -0,0 +1,3258 @@ +#include "config.h" + +#include "references.hh" +#include "pathlocks.hh" +#include "misc.hh" +#include "globals.hh" +#include "local-store.hh" +#include "util.hh" +#include "archive.hh" +#include "affinity.hh" + +#include <map> +#include <sstream> +#include <algorithm> +#include <boost/shared_ptr.hpp> +#include <boost/weak_ptr.hpp> +#include <boost/enable_shared_from_this.hpp> + +#include <time.h> +#include <sys/time.h> +#include <sys/wait.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <sys/utsname.h> +#include <fcntl.h> +#include <unistd.h> +#include <errno.h> +#include <stdio.h> +#include <cstring> + +#include <pwd.h> +#include <grp.h> + +#include <bzlib.h> + + +/* Includes required for chroot support. */ +#if HAVE_SYS_PARAM_H +#include <sys/param.h> +#endif +#if HAVE_SYS_MOUNT_H +#include <sys/mount.h> +#endif +#if HAVE_SCHED_H +#include <sched.h> +#endif + +/* In GNU libc 2.11, <sys/mount.h> does not define `MS_PRIVATE', but + <linux/fs.h> does. */ +#if !defined MS_PRIVATE && defined HAVE_LINUX_FS_H +#include <linux/fs.h> +#endif + +#define CHROOT_ENABLED HAVE_CHROOT && HAVE_UNSHARE && HAVE_SYS_MOUNT_H && defined(MS_BIND) && defined(MS_PRIVATE) && defined(CLONE_NEWNS) + +#if CHROOT_ENABLED +#include <sys/socket.h> +#include <sys/ioctl.h> +#include <net/if.h> +#include <netinet/ip.h> +#endif + + +#if HAVE_SYS_PERSONALITY_H +#include <sys/personality.h> +#define CAN_DO_LINUX32_BUILDS +#endif + + +namespace nix { + +using std::map; + + +static string pathNullDevice = "/dev/null"; + + +/* Forward definition. */ +class Worker; +struct HookInstance; + + +/* A pointer to a goal. */ +class Goal; +typedef boost::shared_ptr<Goal> GoalPtr; +typedef boost::weak_ptr<Goal> WeakGoalPtr; + +/* Set of goals. */ +typedef set<GoalPtr> Goals; +typedef set<WeakGoalPtr> WeakGoals; + +/* A map of paths to goals (and the other way around). */ +typedef map<Path, WeakGoalPtr> WeakGoalMap; + + + +class Goal : public boost::enable_shared_from_this<Goal> +{ +public: + typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode; + +protected: + + /* Backlink to the worker. */ + Worker & worker; + + /* Goals that this goal is waiting for. */ + Goals waitees; + + /* Goals waiting for this one to finish. Must use weak pointers + here to prevent cycles. */ + WeakGoals waiters; + + /* Number of goals we are/were waiting for that have failed. */ + unsigned int nrFailed; + + /* Number of substitution goals we are/were waiting for that + failed because there are no substituters. */ + unsigned int nrNoSubstituters; + + /* Number of substitution goals we are/were waiting for that + failed because othey had unsubstitutable references. */ + unsigned int nrIncompleteClosure; + + /* Name of this goal for debugging purposes. */ + string name; + + /* Whether the goal is finished. */ + ExitCode exitCode; + + Goal(Worker & worker) : worker(worker) + { + nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; + exitCode = ecBusy; + } + + virtual ~Goal() + { + trace("goal destroyed"); + } + +public: + virtual void work() = 0; + + void addWaitee(GoalPtr waitee); + + virtual void waiteeDone(GoalPtr waitee, ExitCode result); + + virtual void handleChildOutput(int fd, const string & data) + { + abort(); + } + + virtual void handleEOF(int fd) + { + abort(); + } + + void trace(const format & f); + + string getName() + { + return name; + } + + ExitCode getExitCode() + { + return exitCode; + } + + /* Cancel the goal. It should wake up its waiters, get rid of any + running child processes that are being monitored by the worker + (important!), etc. */ + virtual void cancel(bool timeout) = 0; + +protected: + void amDone(ExitCode result); +}; + + +/* A mapping used to remember for each child process to what goal it + belongs, and file descriptors for receiving log data and output + path creation commands. */ +struct Child +{ + WeakGoalPtr goal; + set<int> fds; + bool respectTimeouts; + bool inBuildSlot; + time_t lastOutput; /* time we last got output on stdout/stderr */ + time_t timeStarted; +}; + +typedef map<pid_t, Child> Children; + + +/* The worker class. */ +class Worker +{ +private: + + /* Note: the worker should only have strong pointers to the + top-level goals. */ + + /* The top-level goals of the worker. */ + Goals topGoals; + + /* Goals that are ready to do some work. */ + WeakGoals awake; + + /* Goals waiting for a build slot. */ + WeakGoals wantingToBuild; + + /* Child processes currently running. */ + Children children; + + /* Number of build slots occupied. This includes local builds and + substitutions but not remote builds via the build hook. */ + unsigned int nrLocalBuilds; + + /* Maps used to prevent multiple instantiations of a goal for the + same derivation / path. */ + WeakGoalMap derivationGoals; + WeakGoalMap substitutionGoals; + + /* Goals waiting for busy paths to be unlocked. */ + WeakGoals waitingForAnyGoal; + + /* Goals sleeping for a few seconds (polling a lock). */ + WeakGoals waitingForAWhile; + + /* Last time the goals in `waitingForAWhile' where woken up. */ + time_t lastWokenUp; + +public: + + /* Set if at least one derivation had a BuildError (i.e. permanent + failure). */ + bool permanentFailure; + + LocalStore & store; + + boost::shared_ptr<HookInstance> hook; + + Worker(LocalStore & store); + ~Worker(); + + /* Make a goal (with caching). */ + GoalPtr makeDerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, bool repair = false); + GoalPtr makeSubstitutionGoal(const Path & storePath, bool repair = false); + + /* Remove a dead goal. */ + void removeGoal(GoalPtr goal); + + /* Wake up a goal (i.e., there is something for it to do). */ + void wakeUp(GoalPtr goal); + + /* Return the number of local build and substitution processes + currently running (but not remote builds via the build + hook). */ + unsigned int getNrLocalBuilds(); + + /* Registers a running child process. `inBuildSlot' means that + the process counts towards the jobs limit. */ + void childStarted(GoalPtr goal, pid_t pid, + const set<int> & fds, bool inBuildSlot, bool respectTimeouts); + + /* Unregisters a running child process. `wakeSleepers' should be + false if there is no sense in waking up goals that are sleeping + because they can't run yet (e.g., there is no free build slot, + or the hook would still say `postpone'). */ + void childTerminated(pid_t pid, bool wakeSleepers = true); + + /* Put `goal' to sleep until a build slot becomes available (which + might be right away). */ + void waitForBuildSlot(GoalPtr goal); + + /* Wait for any goal to finish. Pretty indiscriminate way to + wait for some resource that some other goal is holding. */ + void waitForAnyGoal(GoalPtr goal); + + /* Wait for a few seconds and then retry this goal. Used when + waiting for a lock held by another process. This kind of + polling is inefficient, but POSIX doesn't really provide a way + to wait for multiple locks in the main select() loop. */ + void waitForAWhile(GoalPtr goal); + + /* Loop until the specified top-level goals have finished. */ + void run(const Goals & topGoals); + + /* Wait for input to become available. */ + void waitForInput(); + + unsigned int exitStatus(); +}; + + +////////////////////////////////////////////////////////////////////// + + +void Goal::addWaitee(GoalPtr waitee) +{ + waitees.insert(waitee); + waitee->waiters.insert(shared_from_this()); +} + + +void Goal::waiteeDone(GoalPtr waitee, ExitCode result) +{ + assert(waitees.find(waitee) != waitees.end()); + waitees.erase(waitee); + + trace(format("waitee `%1%' done; %2% left") % + waitee->name % waitees.size()); + + if (result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure) ++nrFailed; + + if (result == ecNoSubstituters) ++nrNoSubstituters; + + if (result == ecIncompleteClosure) ++nrIncompleteClosure; + + if (waitees.empty() || (result == ecFailed && !settings.keepGoing)) { + + /* If we failed and keepGoing is not set, we remove all + remaining waitees. */ + foreach (Goals::iterator, i, waitees) { + GoalPtr goal = *i; + WeakGoals waiters2; + foreach (WeakGoals::iterator, j, goal->waiters) + if (j->lock() != shared_from_this()) waiters2.insert(*j); + goal->waiters = waiters2; + } + waitees.clear(); + + worker.wakeUp(shared_from_this()); + } +} + + +void Goal::amDone(ExitCode result) +{ + trace("done"); + assert(exitCode == ecBusy); + assert(result == ecSuccess || result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure); + exitCode = result; + foreach (WeakGoals::iterator, i, waiters) { + GoalPtr goal = i->lock(); + if (goal) goal->waiteeDone(shared_from_this(), result); + } + waiters.clear(); + worker.removeGoal(shared_from_this()); +} + + +void Goal::trace(const format & f) +{ + debug(format("%1%: %2%") % name % f); +} + + + +////////////////////////////////////////////////////////////////////// + + +/* Common initialisation performed in child processes. */ +static void commonChildInit(Pipe & logPipe) +{ + restoreAffinity(); + + /* Put the child in a separate session (and thus a separate + process group) so that it has no controlling terminal (meaning + that e.g. ssh cannot open /dev/tty) and it doesn't receive + terminal signals. */ + if (setsid() == -1) + throw SysError(format("creating a new session")); + + /* Dup the write side of the logger pipe into stderr. */ + if (dup2(logPipe.writeSide, STDERR_FILENO) == -1) + throw SysError("cannot pipe standard error into log file"); + + /* Dup stderr to stdout. */ + if (dup2(STDERR_FILENO, STDOUT_FILENO) == -1) + throw SysError("cannot dup stderr into stdout"); + + /* Reroute stdin to /dev/null. */ + int fdDevNull = open(pathNullDevice.c_str(), O_RDWR); + if (fdDevNull == -1) + throw SysError(format("cannot open `%1%'") % pathNullDevice); + if (dup2(fdDevNull, STDIN_FILENO) == -1) + throw SysError("cannot dup null device into stdin"); + close(fdDevNull); +} + + +/* Convert a string list to an array of char pointers. Careful: the + string list should outlive the array. */ +const char * * strings2CharPtrs(const Strings & ss) +{ + const char * * arr = new const char * [ss.size() + 1]; + const char * * p = arr; + foreach (Strings::const_iterator, i, ss) *p++ = i->c_str(); + *p = 0; + return arr; +} + + +/* Restore default handling of SIGPIPE, otherwise some programs will + randomly say "Broken pipe". */ +static void restoreSIGPIPE() +{ + struct sigaction act, oact; + act.sa_handler = SIG_DFL; + act.sa_flags = 0; + if (sigaction(SIGPIPE, &act, &oact)) throw SysError("resetting SIGPIPE"); +} + + +////////////////////////////////////////////////////////////////////// + + +class UserLock +{ +private: + /* POSIX locks suck. If we have a lock on a file, and we open and + close that file again (without closing the original file + descriptor), we lose the lock. So we have to be *very* careful + not to open a lock file on which we are holding a lock. */ + static PathSet lockedPaths; /* !!! not thread-safe */ + + Path fnUserLock; + AutoCloseFD fdUserLock; + + string user; + uid_t uid; + gid_t gid; + +public: + UserLock(); + ~UserLock(); + + void acquire(); + void release(); + + void kill(); + + string getUser() { return user; } + uid_t getUID() { return uid; } + uid_t getGID() { return gid; } + + bool enabled() { return uid != 0; } + +}; + + +PathSet UserLock::lockedPaths; + + +UserLock::UserLock() +{ + uid = gid = 0; +} + + +UserLock::~UserLock() +{ + release(); +} + + +void UserLock::acquire() +{ + assert(uid == 0); + + assert(settings.buildUsersGroup != ""); + + /* Get the members of the build-users-group. */ + struct group * gr = getgrnam(settings.buildUsersGroup.c_str()); + if (!gr) + throw Error(format("the group `%1%' specified in `build-users-group' does not exist") + % settings.buildUsersGroup); + gid = gr->gr_gid; + + /* Copy the result of getgrnam. */ + Strings users; + for (char * * p = gr->gr_mem; *p; ++p) { + debug(format("found build user `%1%'") % *p); + users.push_back(*p); + } + + if (users.empty()) + throw Error(format("the build users group `%1%' has no members") + % settings.buildUsersGroup); + + /* Find a user account that isn't currently in use for another + build. */ + foreach (Strings::iterator, i, users) { + debug(format("trying user `%1%'") % *i); + + struct passwd * pw = getpwnam(i->c_str()); + if (!pw) + throw Error(format("the user `%1%' in the group `%2%' does not exist") + % *i % settings.buildUsersGroup); + + createDirs(settings.nixStateDir + "/userpool"); + + fnUserLock = (format("%1%/userpool/%2%") % settings.nixStateDir % pw->pw_uid).str(); + + if (lockedPaths.find(fnUserLock) != lockedPaths.end()) + /* We already have a lock on this one. */ + continue; + + AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT, 0600); + if (fd == -1) + throw SysError(format("opening user lock `%1%'") % fnUserLock); + closeOnExec(fd); + + if (lockFile(fd, ltWrite, false)) { + fdUserLock = fd.borrow(); + lockedPaths.insert(fnUserLock); + user = *i; + uid = pw->pw_uid; + + /* Sanity check... */ + if (uid == getuid() || uid == geteuid()) + throw Error(format("the Nix user should not be a member of `%1%'") + % settings.buildUsersGroup); + + return; + } + } + + throw Error(format("all build users are currently in use; " + "consider creating additional users and adding them to the `%1%' group") + % settings.buildUsersGroup); +} + + +void UserLock::release() +{ + if (uid == 0) return; + fdUserLock.close(); /* releases lock */ + assert(lockedPaths.find(fnUserLock) != lockedPaths.end()); + lockedPaths.erase(fnUserLock); + fnUserLock = ""; + uid = 0; +} + + +void UserLock::kill() +{ + assert(enabled()); + killUser(uid); +} + + +////////////////////////////////////////////////////////////////////// + + +struct HookInstance +{ + /* Pipes for talking to the build hook. */ + Pipe toHook; + + /* Pipe for the hook's standard output/error. */ + Pipe fromHook; + + /* Pipe for the builder's standard output/error. */ + Pipe builderOut; + + /* The process ID of the hook. */ + Pid pid; + + HookInstance(); + + ~HookInstance(); +}; + + +HookInstance::HookInstance() +{ + debug("starting build hook"); + + Path buildHook = absPath(getEnv("NIX_BUILD_HOOK")); + + /* Create a pipe to get the output of the child. */ + fromHook.create(); + + /* Create the communication pipes. */ + toHook.create(); + + /* Create a pipe to get the output of the builder. */ + builderOut.create(); + + /* Fork the hook. */ + pid = maybeVfork(); + switch (pid) { + + case -1: + throw SysError("unable to fork"); + + case 0: + try { /* child */ + + commonChildInit(fromHook); + + if (chdir("/") == -1) throw SysError("changing into `/"); + + /* Dup the communication pipes. */ + if (dup2(toHook.readSide, STDIN_FILENO) == -1) + throw SysError("dupping to-hook read side"); + + /* Use fd 4 for the builder's stdout/stderr. */ + if (dup2(builderOut.writeSide, 4) == -1) + throw SysError("dupping builder's stdout/stderr"); + + execl(buildHook.c_str(), buildHook.c_str(), settings.thisSystem.c_str(), + (format("%1%") % settings.maxSilentTime).str().c_str(), + (format("%1%") % settings.printBuildTrace).str().c_str(), + (format("%1%") % settings.buildTimeout).str().c_str(), + NULL); + + throw SysError(format("executing `%1%'") % buildHook); + + } catch (std::exception & e) { + writeToStderr("build hook error: " + string(e.what()) + "\n"); + } + _exit(1); + } + + /* parent */ + pid.setSeparatePG(true); + pid.setKillSignal(SIGTERM); + fromHook.writeSide.close(); + toHook.readSide.close(); +} + + +HookInstance::~HookInstance() +{ + try { + pid.kill(); + } catch (...) { + ignoreException(); + } +} + + +////////////////////////////////////////////////////////////////////// + + +typedef map<string, string> HashRewrites; + + +string rewriteHashes(string s, const HashRewrites & rewrites) +{ + foreach (HashRewrites::const_iterator, i, rewrites) { + assert(i->first.size() == i->second.size()); + size_t j = 0; + while ((j = s.find(i->first, j)) != string::npos) { + debug(format("rewriting @ %1%") % j); + s.replace(j, i->second.size(), i->second); + } + } + return s; +} + + +////////////////////////////////////////////////////////////////////// + + +typedef enum {rpAccept, rpDecline, rpPostpone} HookReply; + +class SubstitutionGoal; + +class DerivationGoal : public Goal +{ +private: + /* The path of the derivation. */ + Path drvPath; + + /* The specific outputs that we need to build. Empty means all of + them. */ + StringSet wantedOutputs; + + /* Whether additional wanted outputs have been added. */ + bool needRestart; + + /* Whether to retry substituting the outputs after building the + inputs. */ + bool retrySubstitution; + + /* The derivation stored at drvPath. */ + Derivation drv; + + /* The remainder is state held during the build. */ + + /* Locks on the output paths. */ + PathLocks outputLocks; + + /* All input paths (that is, the union of FS closures of the + immediate input paths). */ + PathSet inputPaths; + + /* Referenceable paths (i.e., input and output paths). */ + PathSet allPaths; + + /* Outputs that are already valid. */ + PathSet validPaths; + + /* Outputs that are corrupt or not valid. */ + PathSet missingPaths; + + /* Paths that have been subject to hash rewriting. */ + PathSet rewrittenPaths; + + /* User selected for running the builder. */ + UserLock buildUser; + + /* The process ID of the builder. */ + Pid pid; + + /* The temporary directory. */ + Path tmpDir; + + /* File descriptor for the log file. */ + FILE * fLogFile; + BZFILE * bzLogFile; + AutoCloseFD fdLogFile; + + /* Number of bytes received from the builder's stdout/stderr. */ + unsigned long logSize; + + /* Pipe for the builder's standard output/error. */ + Pipe builderOut; + + /* The build hook. */ + boost::shared_ptr<HookInstance> hook; + + /* Whether we're currently doing a chroot build. */ + bool useChroot; + + Path chrootRootDir; + + /* RAII object to delete the chroot directory. */ + boost::shared_ptr<AutoDelete> autoDelChroot; + + /* All inputs that are regular files. */ + PathSet regularInputPaths; + + /* Whether this is a fixed-output derivation. */ + bool fixedOutput; + + typedef void (DerivationGoal::*GoalState)(); + GoalState state; + + /* Stuff we need to pass to initChild(). */ + typedef map<Path, Path> DirsInChroot; // maps target path to source path + DirsInChroot dirsInChroot; + typedef map<string, string> Environment; + Environment env; + + /* Hash rewriting. */ + HashRewrites rewritesToTmp, rewritesFromTmp; + PathSet redirectedOutputs; + + /* Whether we're repairing. If set, a derivation will be rebuilt + if its outputs are valid but corrupt or missing. */ + bool repair; + map<Path, Path> redirectedBadOutputs; + + /* Set of inodes seen during calls to canonicalisePathMetaData() + for this build's outputs. This needs to be shared between + outputs to allow hard links between outputs. */ + InodesSeen inodesSeen; + + /* Magic exit code denoting that setting up the child environment + failed. (It's possible that the child actually returns the + exit code, but ah well.) */ + const static int childSetupFailed = 189; + +public: + DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, Worker & worker, bool repair = false); + ~DerivationGoal(); + + void cancel(bool timeout); + + void work(); + + Path getDrvPath() + { + return drvPath; + } + + /* Add wanted outputs to an already existing derivation goal. */ + void addWantedOutputs(const StringSet & outputs); + +private: + /* The states. */ + void init(); + void haveDerivation(); + void outputsSubstituted(); + void closureRepaired(); + void inputsRealised(); + void tryToBuild(); + void buildDone(); + + /* Is the build hook willing to perform the build? */ + HookReply tryBuildHook(); + + /* Start building a derivation. */ + void startBuilder(); + + /* Initialise the builder's process. */ + void initChild(); + + friend int childEntry(void *); + + /* Must be called after the output paths have become valid (either + due to a successful build or hook, or because they already + were). */ + void computeClosure(); + + /* Open a log file and a pipe to it. */ + Path openLogFile(); + + /* Close the log file. */ + void closeLogFile(); + + /* Delete the temporary directory, if we have one. */ + void deleteTmpDir(bool force); + + /* Callback used by the worker to write to the log. */ + void handleChildOutput(int fd, const string & data); + void handleEOF(int fd); + + /* Return the set of (in)valid paths. */ + PathSet checkPathValidity(bool returnValid, bool checkHash); + + /* Abort the goal if `path' failed to build. */ + bool pathFailed(const Path & path); + + /* Forcibly kill the child process, if any. */ + void killChild(); + + Path addHashRewrite(const Path & path); + + void repairClosure(); +}; + + +DerivationGoal::DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, Worker & worker, bool repair) + : Goal(worker) + , wantedOutputs(wantedOutputs) + , needRestart(false) + , retrySubstitution(false) + , fLogFile(0) + , bzLogFile(0) + , useChroot(false) + , repair(repair) +{ + this->drvPath = drvPath; + state = &DerivationGoal::init; + name = (format("building of `%1%'") % drvPath).str(); + trace("created"); +} + + +DerivationGoal::~DerivationGoal() +{ + /* Careful: we should never ever throw an exception from a + destructor. */ + try { + killChild(); + deleteTmpDir(false); + closeLogFile(); + } catch (...) { + ignoreException(); + } +} + + +void DerivationGoal::killChild() +{ + if (pid != -1) { + worker.childTerminated(pid); + + if (buildUser.enabled()) { + /* If we're using a build user, then there is a tricky + race condition: if we kill the build user before the + child has done its setuid() to the build user uid, then + it won't be killed, and we'll potentially lock up in + pid.wait(). So also send a conventional kill to the + child. */ + ::kill(-pid, SIGKILL); /* ignore the result */ + buildUser.kill(); + pid.wait(true); + } else + pid.kill(); + + assert(pid == -1); + } + + hook.reset(); +} + + +void DerivationGoal::cancel(bool timeout) +{ + if (settings.printBuildTrace && timeout) + printMsg(lvlError, format("@ build-failed %1% - timeout") % drvPath); + killChild(); + amDone(ecFailed); +} + + +void DerivationGoal::work() +{ + (this->*state)(); +} + + +void DerivationGoal::addWantedOutputs(const StringSet & outputs) +{ + /* If we already want all outputs, there is nothing to do. */ + if (wantedOutputs.empty()) return; + + if (outputs.empty()) { + wantedOutputs.clear(); + needRestart = true; + } else + foreach (StringSet::const_iterator, i, outputs) + if (wantedOutputs.find(*i) == wantedOutputs.end()) { + wantedOutputs.insert(*i); + needRestart = true; + } +} + + +void DerivationGoal::init() +{ + trace("init"); + + if (settings.readOnlyMode) + throw Error(format("cannot build derivation `%1%' - no write access to the Nix store") % drvPath); + + /* The first thing to do is to make sure that the derivation + exists. If it doesn't, it may be created through a + substitute. */ + addWaitee(worker.makeSubstitutionGoal(drvPath)); + + state = &DerivationGoal::haveDerivation; +} + + +void DerivationGoal::haveDerivation() +{ + trace("loading derivation"); + + if (nrFailed != 0) { + printMsg(lvlError, format("cannot build missing derivation `%1%'") % drvPath); + amDone(ecFailed); + return; + } + + /* `drvPath' should already be a root, but let's be on the safe + side: if the user forgot to make it a root, we wouldn't want + things being garbage collected while we're busy. */ + worker.store.addTempRoot(drvPath); + + assert(worker.store.isValidPath(drvPath)); + + /* Get the derivation. */ + drv = derivationFromPath(worker.store, drvPath); + + foreach (DerivationOutputs::iterator, i, drv.outputs) + worker.store.addTempRoot(i->second.path); + + /* Check what outputs paths are not already valid. */ + PathSet invalidOutputs = checkPathValidity(false, repair); + + /* If they are all valid, then we're done. */ + if (invalidOutputs.size() == 0 && !repair) { + amDone(ecSuccess); + return; + } + + /* Check whether any output previously failed to build. If so, + don't bother. */ + foreach (PathSet::iterator, i, invalidOutputs) + if (pathFailed(*i)) return; + + /* We are first going to try to create the invalid output paths + through substitutes. If that doesn't work, we'll build + them. */ + if (settings.useSubstitutes && !willBuildLocally(drv)) + foreach (PathSet::iterator, i, invalidOutputs) + addWaitee(worker.makeSubstitutionGoal(*i, repair)); + + if (waitees.empty()) /* to prevent hang (no wake-up event) */ + outputsSubstituted(); + else + state = &DerivationGoal::outputsSubstituted; +} + + +void DerivationGoal::outputsSubstituted() +{ + trace("all outputs substituted (maybe)"); + + if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) + throw Error(format("some substitutes for the outputs of derivation `%1%' failed (usually happens due to networking issues); try `--fallback' to build derivation from source ") % drvPath); + + /* If the substitutes form an incomplete closure, then we should + build the dependencies of this derivation, but after that, we + can still use the substitutes for this derivation itself. */ + if (nrIncompleteClosure > 0 && !retrySubstitution) retrySubstitution = true; + + nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; + + if (needRestart) { + needRestart = false; + haveDerivation(); + return; + } + + if (checkPathValidity(false, repair).size() == 0) { + if (repair) repairClosure(); else amDone(ecSuccess); + return; + } + + /* Otherwise, at least one of the output paths could not be + produced using a substitute. So we have to build instead. */ + + /* Make sure checkPathValidity() from now on checks all + outputs. */ + wantedOutputs = PathSet(); + + /* The inputs must be built before we can build this goal. */ + foreach (DerivationInputs::iterator, i, drv.inputDrvs) + addWaitee(worker.makeDerivationGoal(i->first, i->second, repair)); + + foreach (PathSet::iterator, i, drv.inputSrcs) + addWaitee(worker.makeSubstitutionGoal(*i)); + + if (waitees.empty()) /* to prevent hang (no wake-up event) */ + inputsRealised(); + else + state = &DerivationGoal::inputsRealised; +} + + +void DerivationGoal::repairClosure() +{ + /* If we're repairing, we now know that our own outputs are valid. + Now check whether the other paths in the outputs closure are + good. If not, then start derivation goals for the derivations + that produced those outputs. */ + + /* Get the output closure. */ + PathSet outputClosure; + foreach (DerivationOutputs::iterator, i, drv.outputs) + computeFSClosure(worker.store, i->second.path, outputClosure); + + /* Filter out our own outputs (which we have already checked). */ + foreach (DerivationOutputs::iterator, i, drv.outputs) + outputClosure.erase(i->second.path); + + /* Get all dependencies of this derivation so that we know which + derivation is responsible for which path in the output + closure. */ + PathSet inputClosure; + computeFSClosure(worker.store, drvPath, inputClosure); + std::map<Path, Path> outputsToDrv; + foreach (PathSet::iterator, i, inputClosure) + if (isDerivation(*i)) { + Derivation drv = derivationFromPath(worker.store, *i); + foreach (DerivationOutputs::iterator, j, drv.outputs) + outputsToDrv[j->second.path] = *i; + } + + /* Check each path (slow!). */ + PathSet broken; + foreach (PathSet::iterator, i, outputClosure) { + if (worker.store.pathContentsGood(*i)) continue; + printMsg(lvlError, format("found corrupted or missing path `%1%' in the output closure of `%2%'") % *i % drvPath); + Path drvPath2 = outputsToDrv[*i]; + if (drvPath2 == "") + addWaitee(worker.makeSubstitutionGoal(*i, true)); + else + addWaitee(worker.makeDerivationGoal(drvPath2, PathSet(), true)); + } + + if (waitees.empty()) { + amDone(ecSuccess); + return; + } + + state = &DerivationGoal::closureRepaired; +} + + +void DerivationGoal::closureRepaired() +{ + trace("closure repaired"); + if (nrFailed > 0) + throw Error(format("some paths in the output closure of derivation `%1%' could not be repaired") % drvPath); + amDone(ecSuccess); +} + + +void DerivationGoal::inputsRealised() +{ + trace("all inputs realised"); + + if (nrFailed != 0) { + printMsg(lvlError, + format("cannot build derivation `%1%': " + "%2% dependencies couldn't be built") + % drvPath % nrFailed); + amDone(ecFailed); + return; + } + + if (retrySubstitution) { + haveDerivation(); + return; + } + + /* Gather information necessary for computing the closure and/or + running the build hook. */ + + /* The outputs are referenceable paths. */ + foreach (DerivationOutputs::iterator, i, drv.outputs) { + debug(format("building path `%1%'") % i->second.path); + allPaths.insert(i->second.path); + } + + /* Determine the full set of input paths. */ + + /* First, the input derivations. */ + foreach (DerivationInputs::iterator, i, drv.inputDrvs) { + /* Add the relevant output closures of the input derivation + `*i' as input paths. Only add the closures of output paths + that are specified as inputs. */ + assert(worker.store.isValidPath(i->first)); + Derivation inDrv = derivationFromPath(worker.store, i->first); + foreach (StringSet::iterator, j, i->second) + if (inDrv.outputs.find(*j) != inDrv.outputs.end()) + computeFSClosure(worker.store, inDrv.outputs[*j].path, inputPaths); + else + throw Error( + format("derivation `%1%' requires non-existent output `%2%' from input derivation `%3%'") + % drvPath % *j % i->first); + } + + /* Second, the input sources. */ + foreach (PathSet::iterator, i, drv.inputSrcs) + computeFSClosure(worker.store, *i, inputPaths); + + debug(format("added input paths %1%") % showPaths(inputPaths)); + + allPaths.insert(inputPaths.begin(), inputPaths.end()); + + /* Is this a fixed-output derivation? */ + fixedOutput = true; + foreach (DerivationOutputs::iterator, i, drv.outputs) + if (i->second.hash == "") fixedOutput = false; + + /* Okay, try to build. Note that here we don't wait for a build + slot to become available, since we don't need one if there is a + build hook. */ + state = &DerivationGoal::tryToBuild; + worker.wakeUp(shared_from_this()); +} + + +PathSet outputPaths(const DerivationOutputs & outputs) +{ + PathSet paths; + foreach (DerivationOutputs::const_iterator, i, outputs) + paths.insert(i->second.path); + return paths; +} + + +static string get(const StringPairs & map, const string & key) +{ + StringPairs::const_iterator i = map.find(key); + return i == map.end() ? (string) "" : i->second; +} + + +static bool canBuildLocally(const string & platform) +{ + return platform == settings.thisSystem +#ifdef CAN_DO_LINUX32_BUILDS + || (platform == "i686-linux" && settings.thisSystem == "x86_64-linux") +#endif + ; +} + + +bool willBuildLocally(const Derivation & drv) +{ + return get(drv.env, "preferLocalBuild") == "1" && canBuildLocally(drv.platform); +} + + +void DerivationGoal::tryToBuild() +{ + trace("trying to build"); + + /* Check for the possibility that some other goal in this process + has locked the output since we checked in haveDerivation(). + (It can't happen between here and the lockPaths() call below + because we're not allowing multi-threading.) If so, put this + goal to sleep until another goal finishes, then try again. */ + foreach (DerivationOutputs::iterator, i, drv.outputs) + if (pathIsLockedByMe(i->second.path)) { + debug(format("putting derivation `%1%' to sleep because `%2%' is locked by another goal") + % drvPath % i->second.path); + worker.waitForAnyGoal(shared_from_this()); + return; + } + + /* Obtain locks on all output paths. The locks are automatically + released when we exit this function or Nix crashes. If we + can't acquire the lock, then continue; hopefully some other + goal can start a build, and if not, the main loop will sleep a + few seconds and then retry this goal. */ + if (!outputLocks.lockPaths(outputPaths(drv.outputs), "", false)) { + worker.waitForAWhile(shared_from_this()); + return; + } + + /* Now check again whether the outputs are valid. This is because + another process may have started building in parallel. After + it has finished and released the locks, we can (and should) + reuse its results. (Strictly speaking the first check can be + omitted, but that would be less efficient.) Note that since we + now hold the locks on the output paths, no other process can + build this derivation, so no further checks are necessary. */ + validPaths = checkPathValidity(true, repair); + if (validPaths.size() == drv.outputs.size()) { + debug(format("skipping build of derivation `%1%', someone beat us to it") + % drvPath); + outputLocks.setDeletion(true); + amDone(ecSuccess); + return; + } + + missingPaths = outputPaths(drv.outputs); + foreach (PathSet::iterator, i, validPaths) missingPaths.erase(*i); + + /* If any of the outputs already exist but are not valid, delete + them. */ + foreach (DerivationOutputs::iterator, i, drv.outputs) { + Path path = i->second.path; + if (worker.store.isValidPath(path)) continue; + if (!pathExists(path)) continue; + debug(format("removing unregistered path `%1%'") % path); + deletePath(path); + } + + /* Check again whether any output previously failed to build, + because some other process may have tried and failed before we + acquired the lock. */ + foreach (DerivationOutputs::iterator, i, drv.outputs) + if (pathFailed(i->second.path)) return; + + /* Don't do a remote build if the derivation has the attribute + `preferLocalBuild' set. */ + bool buildLocally = willBuildLocally(drv); + + /* Is the build hook willing to accept this job? */ + if (!buildLocally) { + switch (tryBuildHook()) { + case rpAccept: + /* Yes, it has started doing so. Wait until we get + EOF from the hook. */ + state = &DerivationGoal::buildDone; + return; + case rpPostpone: + /* Not now; wait until at least one child finishes or + the wake-up timeout expires. */ + worker.waitForAWhile(shared_from_this()); + outputLocks.unlock(); + return; + case rpDecline: + /* We should do it ourselves. */ + break; + } + } + + /* Make sure that we are allowed to start a build. If this + derivation prefers to be done locally, do it even if + maxBuildJobs is 0. */ + unsigned int curBuilds = worker.getNrLocalBuilds(); + if (curBuilds >= settings.maxBuildJobs && !(buildLocally && curBuilds == 0)) { + worker.waitForBuildSlot(shared_from_this()); + outputLocks.unlock(); + return; + } + + try { + + /* Okay, we have to build. */ + startBuilder(); + + } catch (BuildError & e) { + printMsg(lvlError, e.msg()); + outputLocks.unlock(); + buildUser.release(); + if (settings.printBuildTrace) + printMsg(lvlError, format("@ build-failed %1% - %2% %3%") + % drvPath % 0 % e.msg()); + worker.permanentFailure = true; + amDone(ecFailed); + return; + } + + /* This state will be reached when we get EOF on the child's + log pipe. */ + state = &DerivationGoal::buildDone; +} + + +void replaceValidPath(const Path & storePath, const Path tmpPath) +{ + /* We can't atomically replace storePath (the original) with + tmpPath (the replacement), so we have to move it out of the + way first. We'd better not be interrupted here, because if + we're repairing (say) Glibc, we end up with a broken system. */ + Path oldPath = (format("%1%.old-%2%-%3%") % storePath % getpid() % rand()).str(); + if (pathExists(storePath)) + rename(storePath.c_str(), oldPath.c_str()); + if (rename(tmpPath.c_str(), storePath.c_str()) == -1) + throw SysError(format("moving `%1%' to `%2%'") % tmpPath % storePath); + if (pathExists(oldPath)) + deletePath(oldPath); +} + + +void DerivationGoal::buildDone() +{ + trace("build done"); + + /* Since we got an EOF on the logger pipe, the builder is presumed + to have terminated. In fact, the builder could also have + simply have closed its end of the pipe --- just don't do that + :-) */ + int status; + pid_t savedPid; + if (hook) { + savedPid = hook->pid; + status = hook->pid.wait(true); + } else { + /* !!! this could block! security problem! solution: kill the + child */ + savedPid = pid; + status = pid.wait(true); + } + + debug(format("builder process for `%1%' finished") % drvPath); + + /* So the child is gone now. */ + worker.childTerminated(savedPid); + + /* Close the read side of the logger pipe. */ + if (hook) { + hook->builderOut.readSide.close(); + hook->fromHook.readSide.close(); + } + else builderOut.readSide.close(); + + /* Close the log file. */ + closeLogFile(); + + /* When running under a build user, make sure that all processes + running under that uid are gone. This is to prevent a + malicious user from leaving behind a process that keeps files + open and modifies them after they have been chown'ed to + root. */ + if (buildUser.enabled()) buildUser.kill(); + + try { + + /* Some cleanup per path. We do this here and not in + computeClosure() for convenience when the build has + failed. */ + foreach (PathSet::iterator, i, missingPaths) { + Path path = *i; + + /* If the output was already valid, just skip (discard) it. */ + if (validPaths.find(path) != validPaths.end()) continue; + + if (useChroot && pathExists(chrootRootDir + path)) { + /* Move output paths from the chroot to the Nix store. */ + if (repair) + replaceValidPath(path, chrootRootDir + path); + else + if (rename((chrootRootDir + path).c_str(), path.c_str()) == -1) + throw SysError(format("moving build output `%1%' from the chroot to the Nix store") % path); + } + + Path redirected; + if (repair && (redirected = redirectedBadOutputs[path]) != "" && pathExists(redirected)) + replaceValidPath(path, redirected); + + if (!pathExists(path)) continue; + + struct stat st; + if (lstat(path.c_str(), &st) == -1) + throw SysError(format("getting attributes of path `%1%'") % path); + +#ifndef __CYGWIN__ + /* Check that the output is not group or world writable, + as that means that someone else can have interfered + with the build. Also, the output should be owned by + the build user. */ + if ((!S_ISLNK(st.st_mode) && (st.st_mode & (S_IWGRP | S_IWOTH))) || + (buildUser.enabled() && st.st_uid != buildUser.getUID())) + throw BuildError(format("suspicious ownership or permission on `%1%'; rejecting this build output") % path); +#endif + + /* Apply hash rewriting if necessary. */ + if (!rewritesFromTmp.empty()) { + printMsg(lvlError, format("warning: rewriting hashes in `%1%'; cross fingers") % path); + + /* Canonicalise first. This ensures that the path + we're rewriting doesn't contain a hard link to + /etc/shadow or something like that. */ + canonicalisePathMetaData(path, buildUser.enabled() ? buildUser.getUID() : -1, inodesSeen); + + /* FIXME: this is in-memory. */ + StringSink sink; + dumpPath(path, sink); + deletePath(path); + sink.s = rewriteHashes(sink.s, rewritesFromTmp); + StringSource source(sink.s); + restorePath(path, source); + + rewrittenPaths.insert(path); + } + } + + /* Check the exit status. */ + if (!statusOk(status)) { + deleteTmpDir(false); + if (WIFEXITED(status) && WEXITSTATUS(status) == childSetupFailed) + throw Error(format("failed to set up the build environment for `%1%'") % drvPath); + throw BuildError(format("builder for `%1%' %2%") + % drvPath % statusToString(status)); + } + + /* Delete the chroot (if we were using one). */ + autoDelChroot.reset(); /* this runs the destructor */ + + /* Delete redirected outputs (when doing hash rewriting). */ + foreach (PathSet::iterator, i, redirectedOutputs) + deletePath(*i); + + /* Compute the FS closure of the outputs and register them as + being valid. */ + computeClosure(); + + deleteTmpDir(true); + + /* It is now safe to delete the lock files, since all future + lockers will see that the output paths are valid; they will + not create new lock files with the same names as the old + (unlinked) lock files. */ + outputLocks.setDeletion(true); + outputLocks.unlock(); + + } catch (BuildError & e) { + printMsg(lvlError, e.msg()); + outputLocks.unlock(); + buildUser.release(); + + /* When using a build hook, the hook will return a remote + build failure using exit code 100. Anything else is a hook + problem. */ + bool hookError = hook && + (!WIFEXITED(status) || WEXITSTATUS(status) != 100); + + if (settings.printBuildTrace) { + if (hook && hookError) + printMsg(lvlError, format("@ hook-failed %1% - %2% %3%") + % drvPath % status % e.msg()); + else + printMsg(lvlError, format("@ build-failed %1% - %2% %3%") + % drvPath % 1 % e.msg()); + } + + /* Register the outputs of this build as "failed" so we won't + try to build them again (negative caching). However, don't + do this for fixed-output derivations, since they're likely + to fail for transient reasons (e.g., fetchurl not being + able to access the network). Hook errors (like + communication problems with the remote machine) shouldn't + be cached either. */ + if (settings.cacheFailure && !hookError && !fixedOutput) + foreach (DerivationOutputs::iterator, i, drv.outputs) + worker.store.registerFailedPath(i->second.path); + + worker.permanentFailure = !hookError && !fixedOutput; + amDone(ecFailed); + return; + } + + /* Release the build user, if applicable. */ + buildUser.release(); + + if (settings.printBuildTrace) + printMsg(lvlError, format("@ build-succeeded %1% -") % drvPath); + + amDone(ecSuccess); +} + + +HookReply DerivationGoal::tryBuildHook() +{ + if (!settings.useBuildHook || getEnv("NIX_BUILD_HOOK") == "") return rpDecline; + + if (!worker.hook) + worker.hook = boost::shared_ptr<HookInstance>(new HookInstance); + + /* Tell the hook about system features (beyond the system type) + required from the build machine. (The hook could parse the + drv file itself, but this is easier.) */ + Strings features = tokenizeString<Strings>(get(drv.env, "requiredSystemFeatures")); + foreach (Strings::iterator, i, features) checkStoreName(*i); /* !!! abuse */ + + /* Send the request to the hook. */ + writeLine(worker.hook->toHook.writeSide, (format("%1% %2% %3% %4%") + % (worker.getNrLocalBuilds() < settings.maxBuildJobs ? "1" : "0") + % drv.platform % drvPath % concatStringsSep(",", features)).str()); + + /* Read the first line of input, which should be a word indicating + whether the hook wishes to perform the build. */ + string reply; + while (true) { + string s = readLine(worker.hook->fromHook.readSide); + if (string(s, 0, 2) == "# ") { + reply = string(s, 2); + break; + } + s += "\n"; + writeToStderr(s); + } + + debug(format("hook reply is `%1%'") % reply); + + if (reply == "decline" || reply == "postpone") + return reply == "decline" ? rpDecline : rpPostpone; + else if (reply != "accept") + throw Error(format("bad hook reply `%1%'") % reply); + + printMsg(lvlTalkative, format("using hook to build path(s) %1%") + % showPaths(outputPaths(drv.outputs))); + + hook = worker.hook; + worker.hook.reset(); + + /* Tell the hook all the inputs that have to be copied to the + remote system. This unfortunately has to contain the entire + derivation closure to ensure that the validity invariant holds + on the remote system. (I.e., it's unfortunate that we have to + list it since the remote system *probably* already has it.) */ + PathSet allInputs; + allInputs.insert(inputPaths.begin(), inputPaths.end()); + computeFSClosure(worker.store, drvPath, allInputs); + + string s; + foreach (PathSet::iterator, i, allInputs) s += *i + " "; + writeLine(hook->toHook.writeSide, s); + + /* Tell the hooks the outputs that have to be copied back from the + remote system. */ + s = ""; + foreach (DerivationOutputs::iterator, i, drv.outputs) + s += i->second.path + " "; + writeLine(hook->toHook.writeSide, s); + + hook->toHook.writeSide.close(); + + /* Create the log file and pipe. */ + Path logFile = openLogFile(); + + set<int> fds; + fds.insert(hook->fromHook.readSide); + fds.insert(hook->builderOut.readSide); + worker.childStarted(shared_from_this(), hook->pid, fds, false, false); + + if (settings.printBuildTrace) + printMsg(lvlError, format("@ build-started %1% - %2% %3%") + % drvPath % drv.platform % logFile); + + return rpAccept; +} + + +void chmod_(const Path & path, mode_t mode) +{ + if (chmod(path.c_str(), mode) == -1) + throw SysError(format("setting permissions on `%1%'") % path); +} + + +int childEntry(void * arg) +{ + ((DerivationGoal *) arg)->initChild(); + return 1; +} + + +void DerivationGoal::startBuilder() +{ + startNest(nest, lvlInfo, format(repair ? "repairing path(s) %1%" : "building path(s) %1%") % showPaths(missingPaths)); + + /* Right platform? */ + if (!canBuildLocally(drv.platform)) { + if (settings.printBuildTrace) + printMsg(lvlError, format("@ unsupported-platform %1% %2%") % drvPath % drv.platform); + throw Error( + format("a `%1%' is required to build `%3%', but I am a `%2%'") + % drv.platform % settings.thisSystem % drvPath); + } + + /* Construct the environment passed to the builder. */ + + /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when + PATH is not set. We don't want this, so we fill it in with some dummy + value. */ + env["PATH"] = "/path-not-set"; + + /* Set HOME to a non-existing path to prevent certain programs from using + /etc/passwd (or NIS, or whatever) to locate the home directory (for + example, wget looks for ~/.wgetrc). I.e., these tools use /etc/passwd + if HOME is not set, but they will just assume that the settings file + they are looking for does not exist if HOME is set but points to some + non-existing path. */ + Path homeDir = "/homeless-shelter"; + env["HOME"] = homeDir; + + /* Tell the builder where the Nix store is. Usually they + shouldn't care, but this is useful for purity checking (e.g., + the compiler or linker might only want to accept paths to files + in the store or in the build directory). */ + env["NIX_STORE"] = settings.nixStore; + + /* The maximum number of cores to utilize for parallel building. */ + env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str(); + + /* Add all bindings specified in the derivation. */ + foreach (StringPairs::iterator, i, drv.env) + env[i->first] = i->second; + + /* Create a temporary directory where the build will take + place. */ + tmpDir = createTempDir("", "nix-build-" + storePathToName(drvPath), false, false, 0700); + + /* For convenience, set an environment pointing to the top build + directory. */ + env["NIX_BUILD_TOP"] = tmpDir; + + /* Also set TMPDIR and variants to point to this directory. */ + env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDir; + + /* Explicitly set PWD to prevent problems with chroot builds. In + particular, dietlibc cannot figure out the cwd because the + inode of the current directory doesn't appear in .. (because + getdents returns the inode of the mount point). */ + env["PWD"] = tmpDir; + + /* Compatibility hack with Nix <= 0.7: if this is a fixed-output + derivation, tell the builder, so that for instance `fetchurl' + can skip checking the output. On older Nixes, this environment + variable won't be set, so `fetchurl' will do the check. */ + if (fixedOutput) env["NIX_OUTPUT_CHECKED"] = "1"; + + /* *Only* if this is a fixed-output derivation, propagate the + values of the environment variables specified in the + `impureEnvVars' attribute to the builder. This allows for + instance environment variables for proxy configuration such as + `http_proxy' to be easily passed to downloaders like + `fetchurl'. Passing such environment variables from the caller + to the builder is generally impure, but the output of + fixed-output derivations is by definition pure (since we + already know the cryptographic hash of the output). */ + if (fixedOutput) { + Strings varNames = tokenizeString<Strings>(get(drv.env, "impureEnvVars")); + foreach (Strings::iterator, i, varNames) env[*i] = getEnv(*i); + } + + /* The `exportReferencesGraph' feature allows the references graph + to be passed to a builder. This attribute should be a list of + pairs [name1 path1 name2 path2 ...]. The references graph of + each `pathN' will be stored in a text file `nameN' in the + temporary build directory. The text files have the format used + by `nix-store --register-validity'. However, the deriver + fields are left empty. */ + string s = get(drv.env, "exportReferencesGraph"); + Strings ss = tokenizeString<Strings>(s); + if (ss.size() % 2 != 0) + throw BuildError(format("odd number of tokens in `exportReferencesGraph': `%1%'") % s); + for (Strings::iterator i = ss.begin(); i != ss.end(); ) { + string fileName = *i++; + checkStoreName(fileName); /* !!! abuse of this function */ + + /* Check that the store path is valid. */ + Path storePath = *i++; + if (!isInStore(storePath)) + throw BuildError(format("`exportReferencesGraph' contains a non-store path `%1%'") + % storePath); + storePath = toStorePath(storePath); + if (!worker.store.isValidPath(storePath)) + throw BuildError(format("`exportReferencesGraph' contains an invalid path `%1%'") + % storePath); + + /* If there are derivations in the graph, then include their + outputs as well. This is useful if you want to do things + like passing all build-time dependencies of some path to a + derivation that builds a NixOS DVD image. */ + PathSet paths, paths2; + computeFSClosure(worker.store, storePath, paths); + paths2 = paths; + + foreach (PathSet::iterator, j, paths2) { + if (isDerivation(*j)) { + Derivation drv = derivationFromPath(worker.store, *j); + foreach (DerivationOutputs::iterator, k, drv.outputs) + computeFSClosure(worker.store, k->second.path, paths); + } + } + + /* Write closure info to `fileName'. */ + writeFile(tmpDir + "/" + fileName, + worker.store.makeValidityRegistration(paths, false, false)); + } + + + /* If `build-users-group' is not empty, then we have to build as + one of the members of that group. */ + if (settings.buildUsersGroup != "") { + buildUser.acquire(); + assert(buildUser.getUID() != 0); + assert(buildUser.getGID() != 0); + + /* Make sure that no other processes are executing under this + uid. */ + buildUser.kill(); + + /* Change ownership of the temporary build directory. */ + if (chown(tmpDir.c_str(), buildUser.getUID(), buildUser.getGID()) == -1) + throw SysError(format("cannot change ownership of `%1%'") % tmpDir); + + /* Check that the Nix store has the appropriate permissions, + i.e., owned by root and mode 1775 (sticky bit on so that + the builder can create its output but not mess with the + outputs of other processes). */ + struct stat st; + if (stat(settings.nixStore.c_str(), &st) == -1) + throw SysError(format("cannot stat `%1%'") % settings.nixStore); + if (!(st.st_mode & S_ISVTX) || + ((st.st_mode & S_IRWXG) != S_IRWXG) || + (st.st_gid != buildUser.getGID())) + throw Error(format( + "builder does not have write permission to `%2%'; " + "try `chgrp %1% %2%; chmod 1775 %2%'") + % buildUser.getGID() % settings.nixStore); + } + + + /* Are we doing a chroot build? Note that fixed-output + derivations are never done in a chroot, mainly so that + functions like fetchurl (which needs a proper /etc/resolv.conf) + work properly. Purity checking for fixed-output derivations + is somewhat pointless anyway. */ + useChroot = settings.useChroot; + + if (fixedOutput) useChroot = false; + + /* Hack to allow derivations to disable chroot builds. */ + if (get(drv.env, "__noChroot") == "1") useChroot = false; + + if (useChroot) { +#if CHROOT_ENABLED + /* Create a temporary directory in which we set up the chroot + environment using bind-mounts. We put it in the Nix store + to ensure that we can create hard-links to non-directory + inputs in the fake Nix store in the chroot (see below). */ + chrootRootDir = drvPath + ".chroot"; + if (pathExists(chrootRootDir)) deletePath(chrootRootDir); + + /* Clean up the chroot directory automatically. */ + autoDelChroot = boost::shared_ptr<AutoDelete>(new AutoDelete(chrootRootDir)); + + printMsg(lvlChatty, format("setting up chroot environment in `%1%'") % chrootRootDir); + + /* Create a writable /tmp in the chroot. Many builders need + this. (Of course they should really respect $TMPDIR + instead.) */ + Path chrootTmpDir = chrootRootDir + "/tmp"; + createDirs(chrootTmpDir); + chmod_(chrootTmpDir, 01777); + + /* Create a /etc/passwd with entries for the build user and the + nobody account. The latter is kind of a hack to support + Samba-in-QEMU. */ + createDirs(chrootRootDir + "/etc"); + + writeFile(chrootRootDir + "/etc/passwd", + (format( + "nixbld:x:%1%:%2%:Nix build user:/:/noshell\n" + "nobody:x:65534:65534:Nobody:/:/noshell\n") + % (buildUser.enabled() ? buildUser.getUID() : getuid()) + % (buildUser.enabled() ? buildUser.getGID() : getgid())).str()); + + /* Declare the build user's group so that programs get a consistent + view of the system (e.g., "id -gn"). */ + writeFile(chrootRootDir + "/etc/group", + (format("nixbld:!:%1%:\n") + % (buildUser.enabled() ? buildUser.getGID() : getgid())).str()); + + /* Create /etc/hosts with localhost entry. */ + writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n"); + + /* Bind-mount a user-configurable set of directories from the + host file system. */ + foreach (StringSet::iterator, i, settings.dirsInChroot) { + size_t p = i->find('='); + if (p == string::npos) + dirsInChroot[*i] = *i; + else + dirsInChroot[string(*i, 0, p)] = string(*i, p + 1); + } + dirsInChroot[tmpDir] = tmpDir; + + /* Make the closure of the inputs available in the chroot, + rather than the whole Nix store. This prevents any access + to undeclared dependencies. Directories are bind-mounted, + while other inputs are hard-linked (since only directories + can be bind-mounted). !!! As an extra security + precaution, make the fake Nix store only writable by the + build user. */ + createDirs(chrootRootDir + settings.nixStore); + chmod_(chrootRootDir + settings.nixStore, 01777); + + foreach (PathSet::iterator, i, inputPaths) { + struct stat st; + if (lstat(i->c_str(), &st)) + throw SysError(format("getting attributes of path `%1%'") % *i); + if (S_ISDIR(st.st_mode)) + dirsInChroot[*i] = *i; + else { + Path p = chrootRootDir + *i; + if (link(i->c_str(), p.c_str()) == -1) { + /* Hard-linking fails if we exceed the maximum + link count on a file (e.g. 32000 of ext3), + which is quite possible after a `nix-store + --optimise'. */ + if (errno != EMLINK) + throw SysError(format("linking `%1%' to `%2%'") % p % *i); + StringSink sink; + dumpPath(*i, sink); + StringSource source(sink.s); + restorePath(p, source); + } + + regularInputPaths.insert(*i); + } + } + + /* If we're repairing, it's possible that we're rebuilding a + path that is in settings.dirsInChroot (typically the + dependencies of /bin/sh). Throw them out. */ + if (repair) + foreach (DerivationOutputs::iterator, i, drv.outputs) + dirsInChroot.erase(i->second.path); + +#else + throw Error("chroot builds are not supported on this platform"); +#endif + } + + else { + + if (pathExists(homeDir)) + throw Error(format("directory `%1%' exists; please remove it") % homeDir); + + /* We're not doing a chroot build, but we have some valid + output paths. Since we can't just overwrite or delete + them, we have to do hash rewriting: i.e. in the + environment/arguments passed to the build, we replace the + hashes of the valid outputs with unique dummy strings; + after the build, we discard the redirected outputs + corresponding to the valid outputs, and rewrite the + contents of the new outputs to replace the dummy strings + with the actual hashes. */ + if (validPaths.size() > 0) + //throw Error(format("derivation `%1%' is blocked by its output path(s) %2%") % drvPath % showPaths(validPaths)); + foreach (PathSet::iterator, i, validPaths) + redirectedOutputs.insert(addHashRewrite(*i)); + + /* If we're repairing, then we don't want to delete the + corrupt outputs in advance. So rewrite them as well. */ + if (repair) + foreach (PathSet::iterator, i, missingPaths) + if (worker.store.isValidPath(*i) && pathExists(*i)) + redirectedBadOutputs[*i] = addHashRewrite(*i); + } + + + /* Run the builder. */ + printMsg(lvlChatty, format("executing builder `%1%'") % + drv.builder); + + /* Create the log file. */ + Path logFile = openLogFile(); + + /* Create a pipe to get the output of the builder. */ + builderOut.create(); + + /* Fork a child to build the package. Note that while we + currently use forks to run and wait for the children, it + shouldn't be hard to use threads for this on systems where + fork() is unavailable or inefficient. + + If we're building in a chroot, then also set up private + namespaces for the build: + + - The PID namespace causes the build to start as PID 1. + Processes outside of the chroot are not visible to those on + the inside, but processes inside the chroot are visible from + the outside (though with different PIDs). + + - The private mount namespace ensures that all the bind mounts + we do will only show up in this process and its children, and + will disappear automatically when we're done. + + - The private network namespace ensures that the builder cannot + talk to the outside world (or vice versa). It only has a + private loopback interface. + + - The IPC namespace prevents the builder from communicating + with outside processes using SysV IPC mechanisms (shared + memory, message queues, semaphores). It also ensures that + all IPC objects are destroyed when the builder exits. + + - The UTS namespace ensures that builders see a hostname of + localhost rather than the actual hostname. + */ +#if CHROOT_ENABLED + if (useChroot) { + char stack[32 * 1024]; + pid = clone(childEntry, stack + sizeof(stack) - 8, + CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWNET | CLONE_NEWIPC | CLONE_NEWUTS | SIGCHLD, this); + } else +#endif + { + pid = fork(); + if (pid == 0) initChild(); + } + + if (pid == -1) throw SysError("unable to fork"); + + /* parent */ + pid.setSeparatePG(true); + builderOut.writeSide.close(); + worker.childStarted(shared_from_this(), pid, + singleton<set<int> >(builderOut.readSide), true, true); + + if (settings.printBuildTrace) { + printMsg(lvlError, format("@ build-started %1% - %2% %3%") + % drvPath % drv.platform % logFile); + } +} + + +void DerivationGoal::initChild() +{ + /* Warning: in the child we should absolutely not make any SQLite + calls! */ + + bool inSetup = true; + + try { /* child */ + +#if CHROOT_ENABLED + if (useChroot) { + /* Initialise the loopback interface. */ + AutoCloseFD fd(socket(PF_INET, SOCK_DGRAM, IPPROTO_IP)); + if (fd == -1) throw SysError("cannot open IP socket"); + + struct ifreq ifr; + strcpy(ifr.ifr_name, "lo"); + ifr.ifr_flags = IFF_UP | IFF_LOOPBACK | IFF_RUNNING; + if (ioctl(fd, SIOCSIFFLAGS, &ifr) == -1) + throw SysError("cannot set loopback interface flags"); + + fd.close(); + + /* Set the hostname etc. to fixed values. */ + char hostname[] = "localhost"; + sethostname(hostname, sizeof(hostname)); + char domainname[] = "(none)"; // kernel default + setdomainname(domainname, sizeof(domainname)); + + /* Make all filesystems private. This is necessary + because subtrees may have been mounted as "shared" + (MS_SHARED). (Systemd does this, for instance.) Even + though we have a private mount namespace, mounting + filesystems on top of a shared subtree still propagates + outside of the namespace. Making a subtree private is + local to the namespace, though, so setting MS_PRIVATE + does not affect the outside world. */ + Strings mounts = tokenizeString<Strings>(readFile("/proc/self/mountinfo", true), "\n"); + foreach (Strings::iterator, i, mounts) { + vector<string> fields = tokenizeString<vector<string> >(*i, " "); + string fs = decodeOctalEscaped(fields.at(4)); + if (mount(0, fs.c_str(), 0, MS_PRIVATE, 0) == -1) + throw SysError(format("unable to make filesystem `%1%' private") % fs); + } + + /* Bind-mount all the directories from the "host" + filesystem that we want in the chroot + environment. */ + foreach (DirsInChroot::iterator, i, dirsInChroot) { + struct stat st; + Path source = i->second; + Path target = chrootRootDir + i->first; + if (source == "/proc") continue; // backwards compatibility + debug(format("bind mounting `%1%' to `%2%'") % source % target); + if (stat(source.c_str(), &st) == -1) + throw SysError(format("getting attributes of path `%1%'") % source); + if (S_ISDIR(st.st_mode)) + createDirs(target); + else { + createDirs(dirOf(target)); + writeFile(target, ""); + } + if (mount(source.c_str(), target.c_str(), "", MS_BIND, 0) == -1) + throw SysError(format("bind mount from `%1%' to `%2%' failed") % source % target); + } + + /* Bind a new instance of procfs on /proc to reflect our + private PID namespace. */ + createDirs(chrootRootDir + "/proc"); + if (mount("none", (chrootRootDir + "/proc").c_str(), "proc", 0, 0) == -1) + throw SysError("mounting /proc"); + + /* Mount a new tmpfs on /dev/shm to ensure that whatever + the builder puts in /dev/shm is cleaned up automatically. */ + if (pathExists("/dev/shm")) + if (mount("none", (chrootRootDir + "/dev/shm").c_str(), "tmpfs", 0, 0) == -1) + throw SysError("mounting /dev/shm"); + + /* Do the chroot(). Below we do a chdir() to the + temporary build directory to make sure the current + directory is in the chroot. (Actually the order + doesn't matter, since due to the bind mount tmpDir and + tmpRootDit/tmpDir are the same directories.) */ + if (chroot(chrootRootDir.c_str()) == -1) + throw SysError(format("cannot change root directory to `%1%'") % chrootRootDir); + } +#endif + + commonChildInit(builderOut); + + if (chdir(tmpDir.c_str()) == -1) + throw SysError(format("changing into `%1%'") % tmpDir); + + /* Close all other file descriptors. */ + closeMostFDs(set<int>()); + +#ifdef CAN_DO_LINUX32_BUILDS + /* Change the personality to 32-bit if we're doing an + i686-linux build on an x86_64-linux machine. */ + struct utsname utsbuf; + uname(&utsbuf); + if (drv.platform == "i686-linux" && + (settings.thisSystem == "x86_64-linux" || + (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) { + if (personality(0x0008 | 0x8000000 /* == PER_LINUX32_3GB */) == -1) + throw SysError("cannot set i686-linux personality"); + } + + /* Impersonate a Linux 2.6 machine to get some determinism in + builds that depend on the kernel version. */ + if ((drv.platform == "i686-linux" || drv.platform == "x86_64-linux") && settings.impersonateLinux26) { + int cur = personality(0xffffffff); + if (cur != -1) personality(cur | 0x0020000 /* == UNAME26 */); + } +#endif + + /* Fill in the environment. */ + Strings envStrs; + foreach (Environment::const_iterator, i, env) + envStrs.push_back(rewriteHashes(i->first + "=" + i->second, rewritesToTmp)); + const char * * envArr = strings2CharPtrs(envStrs); + + Path program = drv.builder.c_str(); + std::vector<const char *> args; /* careful with c_str()! */ + string user; /* must be here for its c_str()! */ + + /* If we are running in `build-users' mode, then switch to the + user we allocated above. Make sure that we drop all root + privileges. Note that above we have closed all file + descriptors except std*, so that's safe. Also note that + setuid() when run as root sets the real, effective and + saved UIDs. */ + if (buildUser.enabled()) { + printMsg(lvlChatty, format("switching to user `%1%'") % buildUser.getUser()); + + if (setgroups(0, 0) == -1) + throw SysError("cannot clear the set of supplementary groups"); + + if (setgid(buildUser.getGID()) == -1 || + getgid() != buildUser.getGID() || + getegid() != buildUser.getGID()) + throw SysError("setgid failed"); + + if (setuid(buildUser.getUID()) == -1 || + getuid() != buildUser.getUID() || + geteuid() != buildUser.getUID()) + throw SysError("setuid failed"); + } + + /* Fill in the arguments. */ + string builderBasename = baseNameOf(drv.builder); + args.push_back(builderBasename.c_str()); + foreach (Strings::iterator, i, drv.args) + args.push_back(rewriteHashes(*i, rewritesToTmp).c_str()); + args.push_back(0); + + restoreSIGPIPE(); + + /* Execute the program. This should not return. */ + inSetup = false; + execve(program.c_str(), (char * *) &args[0], (char * *) envArr); + + throw SysError(format("executing `%1%'") % drv.builder); + + } catch (std::exception & e) { + writeToStderr("build error: " + string(e.what()) + "\n"); + _exit(inSetup ? childSetupFailed : 1); + } + + abort(); /* never reached */ +} + + +/* Parse a list of reference specifiers. Each element must either be + a store path, or the symbolic name of the output of the derivation + (such as `out'). */ +PathSet parseReferenceSpecifiers(const Derivation & drv, string attr) +{ + PathSet result; + Paths paths = tokenizeString<Paths>(attr); + foreach (Strings::iterator, i, paths) { + if (isStorePath(*i)) + result.insert(*i); + else if (drv.outputs.find(*i) != drv.outputs.end()) + result.insert(drv.outputs.find(*i)->second.path); + else throw BuildError( + format("derivation contains an illegal reference specifier `%1%'") + % *i); + } + return result; +} + + +void DerivationGoal::computeClosure() +{ + map<Path, PathSet> allReferences; + map<Path, HashResult> contentHashes; + + /* When using a build hook, the build hook can register the output + as valid (by doing `nix-store --import'). If so we don't have + to do anything here. */ + if (hook) { + bool allValid = true; + foreach (DerivationOutputs::iterator, i, drv.outputs) + if (!worker.store.isValidPath(i->second.path)) allValid = false; + if (allValid) return; + } + + /* Check whether the output paths were created, and grep each + output path to determine what other paths it references. Also make all + output paths read-only. */ + foreach (DerivationOutputs::iterator, i, drv.outputs) { + Path path = i->second.path; + if (missingPaths.find(path) == missingPaths.end()) continue; + + if (!pathExists(path)) { + throw BuildError( + format("builder for `%1%' failed to produce output path `%2%'") + % drvPath % path); + } + + struct stat st; + if (lstat(path.c_str(), &st) == -1) + throw SysError(format("getting attributes of path `%1%'") % path); + + startNest(nest, lvlTalkative, + format("scanning for references inside `%1%'") % path); + + /* Check that fixed-output derivations produced the right + outputs (i.e., the content hash should match the specified + hash). */ + if (i->second.hash != "") { + + bool recursive; HashType ht; Hash h; + i->second.parseHashInfo(recursive, ht, h); + + if (!recursive) { + /* The output path should be a regular file without + execute permission. */ + if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0) + throw BuildError( + format("output path `%1% should be a non-executable regular file") + % path); + } + + /* Check the hash. */ + Hash h2 = recursive ? hashPath(ht, path).first : hashFile(ht, path); + if (h != h2) + throw BuildError( + format("output path `%1%' should have %2% hash `%3%', instead has `%4%'") + % path % i->second.hashAlgo % printHash16or32(h) % printHash16or32(h2)); + } + + /* Get rid of all weird permissions. This also checks that + all files are owned by the build user, if applicable. */ + canonicalisePathMetaData(path, + buildUser.enabled() && rewrittenPaths.find(path) == rewrittenPaths.end() ? buildUser.getUID() : -1, inodesSeen); + + /* For this output path, find the references to other paths + contained in it. Compute the SHA-256 NAR hash at the same + time. The hash is stored in the database so that we can + verify later on whether nobody has messed with the store. */ + HashResult hash; + PathSet references = scanForReferences(path, allPaths, hash); + contentHashes[path] = hash; + + /* For debugging, print out the referenced and unreferenced + paths. */ + foreach (PathSet::iterator, i, inputPaths) { + PathSet::iterator j = references.find(*i); + if (j == references.end()) + debug(format("unreferenced input: `%1%'") % *i); + else + debug(format("referenced input: `%1%'") % *i); + } + + allReferences[path] = references; + + /* If the derivation specifies an `allowedReferences' + attribute (containing a list of paths that the output may + refer to), check that all references are in that list. !!! + allowedReferences should really be per-output. */ + if (drv.env.find("allowedReferences") != drv.env.end()) { + PathSet allowed = parseReferenceSpecifiers(drv, get(drv.env, "allowedReferences")); + foreach (PathSet::iterator, i, references) + if (allowed.find(*i) == allowed.end()) + throw BuildError(format("output is not allowed to refer to path `%1%'") % *i); + } + + worker.store.optimisePath(path); // FIXME: combine with scanForReferences() + + worker.store.markContentsGood(path); + } + + /* Register each output path as valid, and register the sets of + paths referenced by each of them. If there are cycles in the + outputs, this will fail. */ + ValidPathInfos infos; + foreach (PathSet::iterator, i, missingPaths) { + ValidPathInfo info; + info.path = *i; + info.hash = contentHashes[*i].first; + info.narSize = contentHashes[*i].second; + info.references = allReferences[*i]; + info.deriver = drvPath; + infos.push_back(info); + } + worker.store.registerValidPaths(infos); +} + + +string drvsLogDir = "drvs"; + + +Path DerivationGoal::openLogFile() +{ + logSize = 0; + + if (!settings.keepLog) return ""; + + string baseName = baseNameOf(drvPath); + + /* Create a log file. */ + Path dir = (format("%1%/%2%/%3%/") % settings.nixLogDir % drvsLogDir % string(baseName, 0, 2)).str(); + createDirs(dir); + + if (settings.compressLog) { + + Path logFileName = (format("%1%/%2%.bz2") % dir % string(baseName, 2)).str(); + AutoCloseFD fd = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC, 0666); + if (fd == -1) throw SysError(format("creating log file `%1%'") % logFileName); + closeOnExec(fd); + + if (!(fLogFile = fdopen(fd.borrow(), "w"))) + throw SysError(format("opening file `%1%'") % logFileName); + + int err; + if (!(bzLogFile = BZ2_bzWriteOpen(&err, fLogFile, 9, 0, 0))) + throw Error(format("cannot open compressed log file `%1%'") % logFileName); + + return logFileName; + + } else { + Path logFileName = (format("%1%/%2%") % dir % string(baseName, 2)).str(); + fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC, 0666); + if (fdLogFile == -1) throw SysError(format("creating log file `%1%'") % logFileName); + closeOnExec(fdLogFile); + return logFileName; + } +} + + +void DerivationGoal::closeLogFile() +{ + if (bzLogFile) { + int err; + BZ2_bzWriteClose(&err, bzLogFile, 0, 0, 0); + bzLogFile = 0; + if (err != BZ_OK) throw Error(format("cannot close compressed log file (BZip2 error = %1%)") % err); + } + + if (fLogFile) { + fclose(fLogFile); + fLogFile = 0; + } + + fdLogFile.close(); +} + + +void DerivationGoal::deleteTmpDir(bool force) +{ + if (tmpDir != "") { + if (settings.keepFailed && !force) { + printMsg(lvlError, + format("note: keeping build directory `%2%'") + % drvPath % tmpDir); + chmod(tmpDir.c_str(), 0755); + } + else + deletePath(tmpDir); + tmpDir = ""; + } +} + + +void DerivationGoal::handleChildOutput(int fd, const string & data) +{ + if ((hook && fd == hook->builderOut.readSide) || + (!hook && fd == builderOut.readSide)) + { + logSize += data.size(); + if (settings.maxLogSize && logSize > settings.maxLogSize) { + printMsg(lvlError, + format("%1% killed after writing more than %2% bytes of log output") + % getName() % settings.maxLogSize); + cancel(true); // not really a timeout, but close enough + return; + } + if (verbosity >= settings.buildVerbosity) + writeToStderr(data); + if (bzLogFile) { + int err; + BZ2_bzWrite(&err, bzLogFile, (unsigned char *) data.data(), data.size()); + if (err != BZ_OK) throw Error(format("cannot write to compressed log file (BZip2 error = %1%)") % err); + } else if (fdLogFile != -1) + writeFull(fdLogFile, (unsigned char *) data.data(), data.size()); + } + + if (hook && fd == hook->fromHook.readSide) + writeToStderr(data); +} + + +void DerivationGoal::handleEOF(int fd) +{ + worker.wakeUp(shared_from_this()); +} + + +PathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash) +{ + PathSet result; + foreach (DerivationOutputs::iterator, i, drv.outputs) { + if (!wantOutput(i->first, wantedOutputs)) continue; + bool good = + worker.store.isValidPath(i->second.path) && + (!checkHash || worker.store.pathContentsGood(i->second.path)); + if (good == returnValid) result.insert(i->second.path); + } + return result; +} + + +bool DerivationGoal::pathFailed(const Path & path) +{ + if (!settings.cacheFailure) return false; + + if (!worker.store.hasPathFailed(path)) return false; + + printMsg(lvlError, format("builder for `%1%' failed previously (cached)") % path); + + if (settings.printBuildTrace) + printMsg(lvlError, format("@ build-failed %1% - cached") % drvPath); + + worker.permanentFailure = true; + amDone(ecFailed); + + return true; +} + + +Path DerivationGoal::addHashRewrite(const Path & path) +{ + string h1 = string(path, settings.nixStore.size() + 1, 32); + string h2 = string(printHash32(hashString(htSHA256, "rewrite:" + drvPath + ":" + path)), 0, 32); + Path p = settings.nixStore + "/" + h2 + string(path, settings.nixStore.size() + 33); + if (pathExists(p)) deletePath(p); + assert(path.size() == p.size()); + rewritesToTmp[h1] = h2; + rewritesFromTmp[h2] = h1; + return p; +} + + +////////////////////////////////////////////////////////////////////// + + +class SubstitutionGoal : public Goal +{ + friend class Worker; + +private: + /* The store path that should be realised through a substitute. */ + Path storePath; + + /* The remaining substituters. */ + Paths subs; + + /* The current substituter. */ + Path sub; + + /* Whether any substituter can realise this path */ + bool hasSubstitute; + + /* Path info returned by the substituter's query info operation. */ + SubstitutablePathInfo info; + + /* Pipe for the substituter's standard output. */ + Pipe outPipe; + + /* Pipe for the substituter's standard error. */ + Pipe logPipe; + + /* The process ID of the builder. */ + Pid pid; + + /* Lock on the store path. */ + boost::shared_ptr<PathLocks> outputLock; + + /* Whether to try to repair a valid path. */ + bool repair; + + /* Location where we're downloading the substitute. Differs from + storePath when doing a repair. */ + Path destPath; + + typedef void (SubstitutionGoal::*GoalState)(); + GoalState state; + +public: + SubstitutionGoal(const Path & storePath, Worker & worker, bool repair = false); + ~SubstitutionGoal(); + + void cancel(bool timeout); + + void work(); + + /* The states. */ + void init(); + void tryNext(); + void gotInfo(); + void referencesValid(); + void tryToRun(); + void finished(); + + /* Callback used by the worker to write to the log. */ + void handleChildOutput(int fd, const string & data); + void handleEOF(int fd); + + Path getStorePath() { return storePath; } +}; + + +SubstitutionGoal::SubstitutionGoal(const Path & storePath, Worker & worker, bool repair) + : Goal(worker) + , hasSubstitute(false) + , repair(repair) +{ + this->storePath = storePath; + state = &SubstitutionGoal::init; + name = (format("substitution of `%1%'") % storePath).str(); + trace("created"); +} + + +SubstitutionGoal::~SubstitutionGoal() +{ + if (pid != -1) worker.childTerminated(pid); +} + + +void SubstitutionGoal::cancel(bool timeout) +{ + if (settings.printBuildTrace && timeout) + printMsg(lvlError, format("@ substituter-failed %1% timeout") % storePath); + if (pid != -1) { + pid_t savedPid = pid; + pid.kill(); + worker.childTerminated(savedPid); + } + amDone(ecFailed); +} + + +void SubstitutionGoal::work() +{ + (this->*state)(); +} + + +void SubstitutionGoal::init() +{ + trace("init"); + + worker.store.addTempRoot(storePath); + + /* If the path already exists we're done. */ + if (!repair && worker.store.isValidPath(storePath)) { + amDone(ecSuccess); + return; + } + + if (settings.readOnlyMode) + throw Error(format("cannot substitute path `%1%' - no write access to the Nix store") % storePath); + + subs = settings.substituters; + + tryNext(); +} + + +void SubstitutionGoal::tryNext() +{ + trace("trying next substituter"); + + if (subs.size() == 0) { + /* None left. Terminate this goal and let someone else deal + with it. */ + debug(format("path `%1%' is required, but there is no substituter that can build it") % storePath); + /* Hack: don't indicate failure if there were no substituters. + In that case the calling derivation should just do a + build. */ + amDone(hasSubstitute ? ecFailed : ecNoSubstituters); + return; + } + + sub = subs.front(); + subs.pop_front(); + + SubstitutablePathInfos infos; + PathSet dummy(singleton<PathSet>(storePath)); + worker.store.querySubstitutablePathInfos(sub, dummy, infos); + SubstitutablePathInfos::iterator k = infos.find(storePath); + if (k == infos.end()) { tryNext(); return; } + info = k->second; + hasSubstitute = true; + + /* To maintain the closure invariant, we first have to realise the + paths referenced by this one. */ + foreach (PathSet::iterator, i, info.references) + if (*i != storePath) /* ignore self-references */ + addWaitee(worker.makeSubstitutionGoal(*i)); + + if (waitees.empty()) /* to prevent hang (no wake-up event) */ + referencesValid(); + else + state = &SubstitutionGoal::referencesValid; +} + + +void SubstitutionGoal::referencesValid() +{ + trace("all references realised"); + + if (nrFailed > 0) { + debug(format("some references of path `%1%' could not be realised") % storePath); + amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed); + return; + } + + foreach (PathSet::iterator, i, info.references) + if (*i != storePath) /* ignore self-references */ + assert(worker.store.isValidPath(*i)); + + state = &SubstitutionGoal::tryToRun; + worker.wakeUp(shared_from_this()); +} + + +void SubstitutionGoal::tryToRun() +{ + trace("trying to run"); + + /* Make sure that we are allowed to start a build. Note that even + is maxBuildJobs == 0 (no local builds allowed), we still allow + a substituter to run. This is because substitutions cannot be + distributed to another machine via the build hook. */ + if (worker.getNrLocalBuilds() >= (settings.maxBuildJobs == 0 ? 1 : settings.maxBuildJobs)) { + worker.waitForBuildSlot(shared_from_this()); + return; + } + + /* Maybe a derivation goal has already locked this path + (exceedingly unlikely, since it should have used a substitute + first, but let's be defensive). */ + outputLock.reset(); // make sure this goal's lock is gone + if (pathIsLockedByMe(storePath)) { + debug(format("restarting substitution of `%1%' because it's locked by another goal") + % storePath); + worker.waitForAnyGoal(shared_from_this()); + return; /* restart in the tryToRun() state when another goal finishes */ + } + + /* Acquire a lock on the output path. */ + outputLock = boost::shared_ptr<PathLocks>(new PathLocks); + if (!outputLock->lockPaths(singleton<PathSet>(storePath), "", false)) { + worker.waitForAWhile(shared_from_this()); + return; + } + + /* Check again whether the path is invalid. */ + if (!repair && worker.store.isValidPath(storePath)) { + debug(format("store path `%1%' has become valid") % storePath); + outputLock->setDeletion(true); + amDone(ecSuccess); + return; + } + + printMsg(lvlInfo, format("fetching path `%1%'...") % storePath); + + outPipe.create(); + logPipe.create(); + + destPath = repair ? storePath + ".tmp" : storePath; + + /* Remove the (stale) output path if it exists. */ + if (pathExists(destPath)) + deletePath(destPath); + + worker.store.setSubstituterEnv(); + + /* Fill in the arguments. */ + Strings args; + args.push_back(baseNameOf(sub)); + args.push_back("--substitute"); + args.push_back(storePath); + args.push_back(destPath); + const char * * argArr = strings2CharPtrs(args); + + /* Fork the substitute program. */ + pid = maybeVfork(); + + switch (pid) { + + case -1: + throw SysError("unable to fork"); + + case 0: + try { /* child */ + + commonChildInit(logPipe); + + if (dup2(outPipe.writeSide, STDOUT_FILENO) == -1) + throw SysError("cannot dup output pipe into stdout"); + + execv(sub.c_str(), (char * *) argArr); + + throw SysError(format("executing `%1%'") % sub); + + } catch (std::exception & e) { + writeToStderr("substitute error: " + string(e.what()) + "\n"); + } + _exit(1); + } + + /* parent */ + pid.setSeparatePG(true); + pid.setKillSignal(SIGTERM); + outPipe.writeSide.close(); + logPipe.writeSide.close(); + worker.childStarted(shared_from_this(), + pid, singleton<set<int> >(logPipe.readSide), true, true); + + state = &SubstitutionGoal::finished; + + if (settings.printBuildTrace) + printMsg(lvlError, format("@ substituter-started %1% %2%") % storePath % sub); +} + + +void SubstitutionGoal::finished() +{ + trace("substitute finished"); + + /* Since we got an EOF on the logger pipe, the substitute is + presumed to have terminated. */ + pid_t savedPid = pid; + int status = pid.wait(true); + + /* So the child is gone now. */ + worker.childTerminated(savedPid); + + /* Close the read side of the logger pipe. */ + logPipe.readSide.close(); + + /* Get the hash info from stdout. */ + string dummy = readLine(outPipe.readSide); + string expectedHashStr = statusOk(status) ? readLine(outPipe.readSide) : ""; + outPipe.readSide.close(); + + /* Check the exit status and the build result. */ + HashResult hash; + try { + + if (!statusOk(status)) + throw SubstError(format("fetching path `%1%' %2%") + % storePath % statusToString(status)); + + if (!pathExists(destPath)) + throw SubstError(format("substitute did not produce path `%1%'") % destPath); + + hash = hashPath(htSHA256, destPath); + + /* Verify the expected hash we got from the substituer. */ + if (expectedHashStr != "") { + size_t n = expectedHashStr.find(':'); + if (n == string::npos) + throw Error(format("bad hash from substituter: %1%") % expectedHashStr); + HashType hashType = parseHashType(string(expectedHashStr, 0, n)); + if (hashType == htUnknown) + throw Error(format("unknown hash algorithm in `%1%'") % expectedHashStr); + Hash expectedHash = parseHash16or32(hashType, string(expectedHashStr, n + 1)); + Hash actualHash = hashType == htSHA256 ? hash.first : hashPath(hashType, destPath).first; + if (expectedHash != actualHash) + throw SubstError(format("hash mismatch in downloaded path `%1%': expected %2%, got %3%") + % storePath % printHash(expectedHash) % printHash(actualHash)); + } + + } catch (SubstError & e) { + + printMsg(lvlInfo, e.msg()); + + if (settings.printBuildTrace) { + printMsg(lvlError, format("@ substituter-failed %1% %2% %3%") + % storePath % status % e.msg()); + } + + /* Try the next substitute. */ + state = &SubstitutionGoal::tryNext; + worker.wakeUp(shared_from_this()); + return; + } + + canonicalisePathMetaData(destPath, -1); + + worker.store.optimisePath(destPath); // FIXME: combine with hashPath() + + if (repair) replaceValidPath(storePath, destPath); + + ValidPathInfo info2; + info2.path = storePath; + info2.hash = hash.first; + info2.narSize = hash.second; + info2.references = info.references; + info2.deriver = info.deriver; + worker.store.registerValidPath(info2); + + outputLock->setDeletion(true); + + worker.store.markContentsGood(storePath); + + printMsg(lvlChatty, + format("substitution of path `%1%' succeeded") % storePath); + + if (settings.printBuildTrace) + printMsg(lvlError, format("@ substituter-succeeded %1%") % storePath); + + amDone(ecSuccess); +} + + +void SubstitutionGoal::handleChildOutput(int fd, const string & data) +{ + assert(fd == logPipe.readSide); + if (verbosity >= settings.buildVerbosity) writeToStderr(data); + /* Don't write substitution output to a log file for now. We + probably should, though. */ +} + + +void SubstitutionGoal::handleEOF(int fd) +{ + if (fd == logPipe.readSide) worker.wakeUp(shared_from_this()); +} + + + +////////////////////////////////////////////////////////////////////// + + +static bool working = false; + + +Worker::Worker(LocalStore & store) + : store(store) +{ + /* Debugging: prevent recursive workers. */ + if (working) abort(); + working = true; + nrLocalBuilds = 0; + lastWokenUp = 0; + permanentFailure = false; +} + + +Worker::~Worker() +{ + working = false; + + /* Explicitly get rid of all strong pointers now. After this all + goals that refer to this worker should be gone. (Otherwise we + are in trouble, since goals may call childTerminated() etc. in + their destructors). */ + topGoals.clear(); +} + + +GoalPtr Worker::makeDerivationGoal(const Path & path, const StringSet & wantedOutputs, bool repair) +{ + GoalPtr goal = derivationGoals[path].lock(); + if (!goal) { + goal = GoalPtr(new DerivationGoal(path, wantedOutputs, *this, repair)); + derivationGoals[path] = goal; + wakeUp(goal); + } else + (dynamic_cast<DerivationGoal *>(goal.get()))->addWantedOutputs(wantedOutputs); + return goal; +} + + +GoalPtr Worker::makeSubstitutionGoal(const Path & path, bool repair) +{ + GoalPtr goal = substitutionGoals[path].lock(); + if (!goal) { + goal = GoalPtr(new SubstitutionGoal(path, *this, repair)); + substitutionGoals[path] = goal; + wakeUp(goal); + } + return goal; +} + + +static void removeGoal(GoalPtr goal, WeakGoalMap & goalMap) +{ + /* !!! inefficient */ + for (WeakGoalMap::iterator i = goalMap.begin(); + i != goalMap.end(); ) + if (i->second.lock() == goal) { + WeakGoalMap::iterator j = i; ++j; + goalMap.erase(i); + i = j; + } + else ++i; +} + + +void Worker::removeGoal(GoalPtr goal) +{ + nix::removeGoal(goal, derivationGoals); + nix::removeGoal(goal, substitutionGoals); + if (topGoals.find(goal) != topGoals.end()) { + topGoals.erase(goal); + /* If a top-level goal failed, then kill all other goals + (unless keepGoing was set). */ + if (goal->getExitCode() == Goal::ecFailed && !settings.keepGoing) + topGoals.clear(); + } + + /* Wake up goals waiting for any goal to finish. */ + foreach (WeakGoals::iterator, i, waitingForAnyGoal) { + GoalPtr goal = i->lock(); + if (goal) wakeUp(goal); + } + + waitingForAnyGoal.clear(); +} + + +void Worker::wakeUp(GoalPtr goal) +{ + goal->trace("woken up"); + awake.insert(goal); +} + + +unsigned Worker::getNrLocalBuilds() +{ + return nrLocalBuilds; +} + + +void Worker::childStarted(GoalPtr goal, + pid_t pid, const set<int> & fds, bool inBuildSlot, + bool respectTimeouts) +{ + Child child; + child.goal = goal; + child.fds = fds; + child.timeStarted = child.lastOutput = time(0); + child.inBuildSlot = inBuildSlot; + child.respectTimeouts = respectTimeouts; + children[pid] = child; + if (inBuildSlot) nrLocalBuilds++; +} + + +void Worker::childTerminated(pid_t pid, bool wakeSleepers) +{ + assert(pid != -1); /* common mistake */ + + Children::iterator i = children.find(pid); + assert(i != children.end()); + + if (i->second.inBuildSlot) { + assert(nrLocalBuilds > 0); + nrLocalBuilds--; + } + + children.erase(pid); + + if (wakeSleepers) { + + /* Wake up goals waiting for a build slot. */ + foreach (WeakGoals::iterator, i, wantingToBuild) { + GoalPtr goal = i->lock(); + if (goal) wakeUp(goal); + } + + wantingToBuild.clear(); + } +} + + +void Worker::waitForBuildSlot(GoalPtr goal) +{ + debug("wait for build slot"); + if (getNrLocalBuilds() < settings.maxBuildJobs) + wakeUp(goal); /* we can do it right away */ + else + wantingToBuild.insert(goal); +} + + +void Worker::waitForAnyGoal(GoalPtr goal) +{ + debug("wait for any goal"); + waitingForAnyGoal.insert(goal); +} + + +void Worker::waitForAWhile(GoalPtr goal) +{ + debug("wait for a while"); + waitingForAWhile.insert(goal); +} + + +void Worker::run(const Goals & _topGoals) +{ + foreach (Goals::iterator, i, _topGoals) topGoals.insert(*i); + + startNest(nest, lvlDebug, format("entered goal loop")); + + while (1) { + + checkInterrupt(); + + /* Call every wake goal. */ + while (!awake.empty() && !topGoals.empty()) { + WeakGoals awake2(awake); + awake.clear(); + foreach (WeakGoals::iterator, i, awake2) { + checkInterrupt(); + GoalPtr goal = i->lock(); + if (goal) goal->work(); + if (topGoals.empty()) break; + } + } + + if (topGoals.empty()) break; + + /* Wait for input. */ + if (!children.empty() || !waitingForAWhile.empty()) + waitForInput(); + else { + if (awake.empty() && settings.maxBuildJobs == 0) throw Error( + "unable to start any build; either increase `--max-jobs' " + "or enable distributed builds"); + assert(!awake.empty()); + } + } + + /* If --keep-going is not set, it's possible that the main goal + exited while some of its subgoals were still active. But if + --keep-going *is* set, then they must all be finished now. */ + assert(!settings.keepGoing || awake.empty()); + assert(!settings.keepGoing || wantingToBuild.empty()); + assert(!settings.keepGoing || children.empty()); +} + + +void Worker::waitForInput() +{ + printMsg(lvlVomit, "waiting for children"); + + /* Process output from the file descriptors attached to the + children, namely log output and output path creation commands. + We also use this to detect child termination: if we get EOF on + the logger pipe of a build, we assume that the builder has + terminated. */ + + bool useTimeout = false; + struct timeval timeout; + timeout.tv_usec = 0; + time_t before = time(0); + + /* If we're monitoring for silence on stdout/stderr, or if there + is a build timeout, then wait for input until the first + deadline for any child. */ + assert(sizeof(time_t) >= sizeof(long)); + time_t nearest = LONG_MAX; // nearest deadline + foreach (Children::iterator, i, children) { + if (!i->second.respectTimeouts) continue; + if (settings.maxSilentTime != 0) + nearest = std::min(nearest, i->second.lastOutput + settings.maxSilentTime); + if (settings.buildTimeout != 0) + nearest = std::min(nearest, i->second.timeStarted + settings.buildTimeout); + } + if (nearest != LONG_MAX) { + timeout.tv_sec = std::max((time_t) 1, nearest - before); + useTimeout = true; + printMsg(lvlVomit, format("sleeping %1% seconds") % timeout.tv_sec); + } + + /* If we are polling goals that are waiting for a lock, then wake + up after a few seconds at most. */ + if (!waitingForAWhile.empty()) { + useTimeout = true; + if (lastWokenUp == 0) + printMsg(lvlError, "waiting for locks or build slots..."); + if (lastWokenUp == 0 || lastWokenUp > before) lastWokenUp = before; + timeout.tv_sec = std::max((time_t) 1, (time_t) (lastWokenUp + settings.pollInterval - before)); + } else lastWokenUp = 0; + + using namespace std; + /* Use select() to wait for the input side of any logger pipe to + become `available'. Note that `available' (i.e., non-blocking) + includes EOF. */ + fd_set fds; + FD_ZERO(&fds); + int fdMax = 0; + foreach (Children::iterator, i, children) { + foreach (set<int>::iterator, j, i->second.fds) { + FD_SET(*j, &fds); + if (*j >= fdMax) fdMax = *j + 1; + } + } + + if (select(fdMax, &fds, 0, 0, useTimeout ? &timeout : 0) == -1) { + if (errno == EINTR) return; + throw SysError("waiting for input"); + } + + time_t after = time(0); + + /* Process all available file descriptors. */ + + /* Since goals may be canceled from inside the loop below (causing + them go be erased from the `children' map), we have to be + careful that we don't keep iterators alive across calls to + cancel(). */ + set<pid_t> pids; + foreach (Children::iterator, i, children) pids.insert(i->first); + + foreach (set<pid_t>::iterator, i, pids) { + checkInterrupt(); + Children::iterator j = children.find(*i); + if (j == children.end()) continue; // child destroyed + GoalPtr goal = j->second.goal.lock(); + assert(goal); + + set<int> fds2(j->second.fds); + foreach (set<int>::iterator, k, fds2) { + if (FD_ISSET(*k, &fds)) { + unsigned char buffer[4096]; + ssize_t rd = read(*k, buffer, sizeof(buffer)); + if (rd == -1) { + if (errno != EINTR) + throw SysError(format("reading from %1%") + % goal->getName()); + } else if (rd == 0) { + debug(format("%1%: got EOF") % goal->getName()); + goal->handleEOF(*k); + j->second.fds.erase(*k); + } else { + printMsg(lvlVomit, format("%1%: read %2% bytes") + % goal->getName() % rd); + string data((char *) buffer, rd); + goal->handleChildOutput(*k, data); + j->second.lastOutput = after; + } + } + } + + if (settings.maxSilentTime != 0 && + j->second.respectTimeouts && + after - j->second.lastOutput >= (time_t) settings.maxSilentTime) + { + printMsg(lvlError, + format("%1% timed out after %2% seconds of silence") + % goal->getName() % settings.maxSilentTime); + goal->cancel(true); + } + + if (settings.buildTimeout != 0 && + j->second.respectTimeouts && + after - j->second.timeStarted >= (time_t) settings.buildTimeout) + { + printMsg(lvlError, + format("%1% timed out after %2% seconds") + % goal->getName() % settings.buildTimeout); + goal->cancel(true); + } + } + + if (!waitingForAWhile.empty() && lastWokenUp + settings.pollInterval <= after) { + lastWokenUp = after; + foreach (WeakGoals::iterator, i, waitingForAWhile) { + GoalPtr goal = i->lock(); + if (goal) wakeUp(goal); + } + waitingForAWhile.clear(); + } +} + + +unsigned int Worker::exitStatus() +{ + return permanentFailure ? 100 : 1; +} + + +////////////////////////////////////////////////////////////////////// + + +void LocalStore::buildPaths(const PathSet & drvPaths, bool repair) +{ + startNest(nest, lvlDebug, + format("building %1%") % showPaths(drvPaths)); + + Worker worker(*this); + + Goals goals; + foreach (PathSet::const_iterator, i, drvPaths) { + DrvPathWithOutputs i2 = parseDrvPathWithOutputs(*i); + if (isDerivation(i2.first)) + goals.insert(worker.makeDerivationGoal(i2.first, i2.second, repair)); + else + goals.insert(worker.makeSubstitutionGoal(*i, repair)); + } + + worker.run(goals); + + PathSet failed; + foreach (Goals::iterator, i, goals) + if ((*i)->getExitCode() == Goal::ecFailed) { + DerivationGoal * i2 = dynamic_cast<DerivationGoal *>(i->get()); + if (i2) failed.insert(i2->getDrvPath()); + else failed.insert(dynamic_cast<SubstitutionGoal *>(i->get())->getStorePath()); + } + + if (!failed.empty()) + throw Error(format("build of %1% failed") % showPaths(failed), worker.exitStatus()); +} + + +void LocalStore::ensurePath(const Path & path) +{ + /* If the path is already valid, we're done. */ + if (isValidPath(path)) return; + + Worker worker(*this); + GoalPtr goal = worker.makeSubstitutionGoal(path); + Goals goals = singleton<Goals>(goal); + + worker.run(goals); + + if (goal->getExitCode() != Goal::ecSuccess) + throw Error(format("path `%1%' does not exist and cannot be created") % path, worker.exitStatus()); +} + + +void LocalStore::repairPath(const Path & path) +{ + Worker worker(*this); + GoalPtr goal = worker.makeSubstitutionGoal(path, true); + Goals goals = singleton<Goals>(goal); + + worker.run(goals); + + if (goal->getExitCode() != Goal::ecSuccess) + throw Error(format("cannot repair path `%1%'") % path, worker.exitStatus()); +} + + +} diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc new file mode 100644 index 000000000000..d91e42784ca5 --- /dev/null +++ b/src/libstore/derivations.cc @@ -0,0 +1,278 @@ +#include "derivations.hh" +#include "store-api.hh" +#include "globals.hh" +#include "util.hh" +#include "misc.hh" + + +namespace nix { + + +void DerivationOutput::parseHashInfo(bool & recursive, HashType & hashType, Hash & hash) const +{ + recursive = false; + string algo = hashAlgo; + + if (string(algo, 0, 2) == "r:") { + recursive = true; + algo = string(algo, 2); + } + + hashType = parseHashType(algo); + if (hashType == htUnknown) + throw Error(format("unknown hash algorithm `%1%'") % algo); + + hash = parseHash(hashType, this->hash); +} + + +Path writeDerivation(StoreAPI & store, + const Derivation & drv, const string & name, bool repair) +{ + PathSet references; + references.insert(drv.inputSrcs.begin(), drv.inputSrcs.end()); + foreach (DerivationInputs::const_iterator, i, drv.inputDrvs) + references.insert(i->first); + /* Note that the outputs of a derivation are *not* references + (that can be missing (of course) and should not necessarily be + held during a garbage collection). */ + string suffix = name + drvExtension; + string contents = unparseDerivation(drv); + return settings.readOnlyMode + ? computeStorePathForText(suffix, contents, references) + : store.addTextToStore(suffix, contents, references, repair); +} + + +static Path parsePath(std::istream & str) +{ + string s = parseString(str); + if (s.size() == 0 || s[0] != '/') + throw Error(format("bad path `%1%' in derivation") % s); + return s; +} + + +static StringSet parseStrings(std::istream & str, bool arePaths) +{ + StringSet res; + while (!endOfList(str)) + res.insert(arePaths ? parsePath(str) : parseString(str)); + return res; +} + + +Derivation parseDerivation(const string & s) +{ + Derivation drv; + std::istringstream str(s); + expect(str, "Derive(["); + + /* Parse the list of outputs. */ + while (!endOfList(str)) { + DerivationOutput out; + expect(str, "("); string id = parseString(str); + expect(str, ","); out.path = parsePath(str); + expect(str, ","); out.hashAlgo = parseString(str); + expect(str, ","); out.hash = parseString(str); + expect(str, ")"); + drv.outputs[id] = out; + } + + /* Parse the list of input derivations. */ + expect(str, ",["); + while (!endOfList(str)) { + expect(str, "("); + Path drvPath = parsePath(str); + expect(str, ",["); + drv.inputDrvs[drvPath] = parseStrings(str, false); + expect(str, ")"); + } + + expect(str, ",["); drv.inputSrcs = parseStrings(str, true); + expect(str, ","); drv.platform = parseString(str); + expect(str, ","); drv.builder = parseString(str); + + /* Parse the builder arguments. */ + expect(str, ",["); + while (!endOfList(str)) + drv.args.push_back(parseString(str)); + + /* Parse the environment variables. */ + expect(str, ",["); + while (!endOfList(str)) { + expect(str, "("); string name = parseString(str); + expect(str, ","); string value = parseString(str); + expect(str, ")"); + drv.env[name] = value; + } + + expect(str, ")"); + return drv; +} + + +static void printString(string & res, const string & s) +{ + res += '"'; + for (const char * i = s.c_str(); *i; i++) + if (*i == '\"' || *i == '\\') { res += "\\"; res += *i; } + else if (*i == '\n') res += "\\n"; + else if (*i == '\r') res += "\\r"; + else if (*i == '\t') res += "\\t"; + else res += *i; + res += '"'; +} + + +template<class ForwardIterator> +static void printStrings(string & res, ForwardIterator i, ForwardIterator j) +{ + res += '['; + bool first = true; + for ( ; i != j; ++i) { + if (first) first = false; else res += ','; + printString(res, *i); + } + res += ']'; +} + + +string unparseDerivation(const Derivation & drv) +{ + string s; + s.reserve(65536); + s += "Derive(["; + + bool first = true; + foreach (DerivationOutputs::const_iterator, i, drv.outputs) { + if (first) first = false; else s += ','; + s += '('; printString(s, i->first); + s += ','; printString(s, i->second.path); + s += ','; printString(s, i->second.hashAlgo); + s += ','; printString(s, i->second.hash); + s += ')'; + } + + s += "],["; + first = true; + foreach (DerivationInputs::const_iterator, i, drv.inputDrvs) { + if (first) first = false; else s += ','; + s += '('; printString(s, i->first); + s += ','; printStrings(s, i->second.begin(), i->second.end()); + s += ')'; + } + + s += "],"; + printStrings(s, drv.inputSrcs.begin(), drv.inputSrcs.end()); + + s += ','; printString(s, drv.platform); + s += ','; printString(s, drv.builder); + s += ','; printStrings(s, drv.args.begin(), drv.args.end()); + + s += ",["; + first = true; + foreach (StringPairs::const_iterator, i, drv.env) { + if (first) first = false; else s += ','; + s += '('; printString(s, i->first); + s += ','; printString(s, i->second); + s += ')'; + } + + s += "])"; + + return s; +} + + +bool isDerivation(const string & fileName) +{ + return hasSuffix(fileName, drvExtension); +} + + +bool isFixedOutputDrv(const Derivation & drv) +{ + return drv.outputs.size() == 1 && + drv.outputs.begin()->first == "out" && + drv.outputs.begin()->second.hash != ""; +} + + +DrvHashes drvHashes; + + +/* Returns the hash of a derivation modulo fixed-output + subderivations. A fixed-output derivation is a derivation with one + output (`out') for which an expected hash and hash algorithm are + specified (using the `outputHash' and `outputHashAlgo' + attributes). We don't want changes to such derivations to + propagate upwards through the dependency graph, changing output + paths everywhere. + + For instance, if we change the url in a call to the `fetchurl' + function, we do not want to rebuild everything depending on it + (after all, (the hash of) the file being downloaded is unchanged). + So the *output paths* should not change. On the other hand, the + *derivation paths* should change to reflect the new dependency + graph. + + That's what this function does: it returns a hash which is just the + hash of the derivation ATerm, except that any input derivation + paths have been replaced by the result of a recursive call to this + function, and that for fixed-output derivations we return a hash of + its output path. */ +Hash hashDerivationModulo(StoreAPI & store, Derivation drv) +{ + /* Return a fixed hash for fixed-output derivations. */ + if (isFixedOutputDrv(drv)) { + DerivationOutputs::const_iterator i = drv.outputs.begin(); + return hashString(htSHA256, "fixed:out:" + + i->second.hashAlgo + ":" + + i->second.hash + ":" + + i->second.path); + } + + /* For other derivations, replace the inputs paths with recursive + calls to this function.*/ + DerivationInputs inputs2; + foreach (DerivationInputs::const_iterator, i, drv.inputDrvs) { + Hash h = drvHashes[i->first]; + if (h.type == htUnknown) { + assert(store.isValidPath(i->first)); + Derivation drv2 = parseDerivation(readFile(i->first)); + h = hashDerivationModulo(store, drv2); + drvHashes[i->first] = h; + } + inputs2[printHash(h)] = i->second; + } + drv.inputDrvs = inputs2; + + return hashString(htSHA256, unparseDerivation(drv)); +} + + +DrvPathWithOutputs parseDrvPathWithOutputs(const string & s) +{ + size_t n = s.find("!"); + return n == s.npos + ? DrvPathWithOutputs(s, std::set<string>()) + : DrvPathWithOutputs(string(s, 0, n), tokenizeString<std::set<string> >(string(s, n + 1), ",")); +} + + +Path makeDrvPathWithOutputs(const Path & drvPath, const std::set<string> & outputs) +{ + return outputs.empty() + ? drvPath + : drvPath + "!" + concatStringsSep(",", outputs); +} + + +bool wantOutput(const string & output, const std::set<string> & wanted) +{ + return wanted.empty() || wanted.find(output) != wanted.end(); +} + + +} diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh new file mode 100644 index 000000000000..703410b92552 --- /dev/null +++ b/src/libstore/derivations.hh @@ -0,0 +1,93 @@ +#pragma once + +#include "types.hh" +#include "hash.hh" + +#include <map> + + +namespace nix { + + +/* Extension of derivations in the Nix store. */ +const string drvExtension = ".drv"; + + +/* Abstract syntax of derivations. */ + +struct DerivationOutput +{ + Path path; + string hashAlgo; /* hash used for expected hash computation */ + string hash; /* expected hash, may be null */ + DerivationOutput() + { + } + DerivationOutput(Path path, string hashAlgo, string hash) + { + this->path = path; + this->hashAlgo = hashAlgo; + this->hash = hash; + } + void parseHashInfo(bool & recursive, HashType & hashType, Hash & hash) const; +}; + +typedef std::map<string, DerivationOutput> DerivationOutputs; + +/* For inputs that are sub-derivations, we specify exactly which + output IDs we are interested in. */ +typedef std::map<Path, StringSet> DerivationInputs; + +typedef std::map<string, string> StringPairs; + +struct Derivation +{ + DerivationOutputs outputs; /* keyed on symbolic IDs */ + DerivationInputs inputDrvs; /* inputs that are sub-derivations */ + PathSet inputSrcs; /* inputs that are sources */ + string platform; + Path builder; + Strings args; + StringPairs env; +}; + + +class StoreAPI; + + +/* Write a derivation to the Nix store, and return its path. */ +Path writeDerivation(StoreAPI & store, + const Derivation & drv, const string & name, bool repair = false); + +/* Parse a derivation. */ +Derivation parseDerivation(const string & s); + +/* Print a derivation. */ +string unparseDerivation(const Derivation & drv); + +/* Check whether a file name ends with the extensions for + derivations. */ +bool isDerivation(const string & fileName); + +/* Return true iff this is a fixed-output derivation. */ +bool isFixedOutputDrv(const Derivation & drv); + +Hash hashDerivationModulo(StoreAPI & store, Derivation drv); + +/* Memoisation of hashDerivationModulo(). */ +typedef std::map<Path, Hash> DrvHashes; + +extern DrvHashes drvHashes; + +/* Split a string specifying a derivation and a set of outputs + (/nix/store/hash-foo!out1,out2,...) into the derivation path and + the outputs. */ +typedef std::pair<string, std::set<string> > DrvPathWithOutputs; +DrvPathWithOutputs parseDrvPathWithOutputs(const string & s); + +Path makeDrvPathWithOutputs(const Path & drvPath, const std::set<string> & outputs); + +bool wantOutput(const string & output, const std::set<string> & wanted); + + +} diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc new file mode 100644 index 000000000000..79bd7d56b3fe --- /dev/null +++ b/src/libstore/gc.cc @@ -0,0 +1,752 @@ +#include "globals.hh" +#include "misc.hh" +#include "local-store.hh" + +#include <boost/shared_ptr.hpp> + +#include <functional> +#include <queue> +#include <algorithm> + +#include <sys/types.h> +#include <sys/stat.h> +#include <errno.h> +#include <fcntl.h> +#include <unistd.h> + + +namespace nix { + + +static string gcLockName = "gc.lock"; +static string tempRootsDir = "temproots"; +static string gcRootsDir = "gcroots"; + + +/* Acquire the global GC lock. This is used to prevent new Nix + processes from starting after the temporary root files have been + read. To be precise: when they try to create a new temporary root + file, they will block until the garbage collector has finished / + yielded the GC lock. */ +int LocalStore::openGCLock(LockType lockType) +{ + Path fnGCLock = (format("%1%/%2%") + % settings.nixStateDir % gcLockName).str(); + + debug(format("acquiring global GC lock `%1%'") % fnGCLock); + + AutoCloseFD fdGCLock = open(fnGCLock.c_str(), O_RDWR | O_CREAT, 0600); + if (fdGCLock == -1) + throw SysError(format("opening global GC lock `%1%'") % fnGCLock); + closeOnExec(fdGCLock); + + if (!lockFile(fdGCLock, lockType, false)) { + printMsg(lvlError, format("waiting for the big garbage collector lock...")); + lockFile(fdGCLock, lockType, true); + } + + /* !!! Restrict read permission on the GC root. Otherwise any + process that can open the file for reading can DoS the + collector. */ + + return fdGCLock.borrow(); +} + + +void createSymlink(const Path & link, const Path & target) +{ + /* Create directories up to `gcRoot'. */ + createDirs(dirOf(link)); + + /* Create the new symlink. */ + Path tempLink = (format("%1%.tmp-%2%-%3%") + % link % getpid() % rand()).str(); + if (symlink(target.c_str(), tempLink.c_str()) == -1) + throw SysError(format("symlinking `%1%' to `%2%'") + % tempLink % target); + + /* Atomically replace the old one. */ + if (rename(tempLink.c_str(), link.c_str()) == -1) + throw SysError(format("cannot rename `%1%' to `%2%'") + % tempLink % link); +} + + +void LocalStore::syncWithGC() +{ + AutoCloseFD fdGCLock = openGCLock(ltRead); +} + + +void LocalStore::addIndirectRoot(const Path & path) +{ + string hash = printHash32(hashString(htSHA1, path)); + Path realRoot = canonPath((format("%1%/%2%/auto/%3%") + % settings.nixStateDir % gcRootsDir % hash).str()); + createSymlink(realRoot, path); +} + + +Path addPermRoot(StoreAPI & store, const Path & _storePath, + const Path & _gcRoot, bool indirect, bool allowOutsideRootsDir) +{ + Path storePath(canonPath(_storePath)); + Path gcRoot(canonPath(_gcRoot)); + assertStorePath(storePath); + + if (isInStore(gcRoot)) + throw Error(format( + "creating a garbage collector root (%1%) in the Nix store is forbidden " + "(are you running nix-build inside the store?)") % gcRoot); + + if (indirect) { + /* Don't clobber the the link if it already exists and doesn't + point to the Nix store. */ + if (pathExists(gcRoot) && (!isLink(gcRoot) || !isInStore(readLink(gcRoot)))) + throw Error(format("cannot create symlink `%1%'; already exists") % gcRoot); + createSymlink(gcRoot, storePath); + store.addIndirectRoot(gcRoot); + } + + else { + if (!allowOutsideRootsDir) { + Path rootsDir = canonPath((format("%1%/%2%") % settings.nixStateDir % gcRootsDir).str()); + + if (string(gcRoot, 0, rootsDir.size() + 1) != rootsDir + "/") + throw Error(format( + "path `%1%' is not a valid garbage collector root; " + "it's not in the directory `%2%'") + % gcRoot % rootsDir); + } + + createSymlink(gcRoot, storePath); + } + + /* Check that the root can be found by the garbage collector. + !!! This can be very slow on machines that have many roots. + Instead of reading all the roots, it would be more efficient to + check if the root is in a directory in or linked from the + gcroots directory. */ + if (settings.checkRootReachability) { + Roots roots = store.findRoots(); + if (roots.find(gcRoot) == roots.end()) + printMsg(lvlError, + format( + "warning: `%1%' is not in a directory where the garbage collector looks for roots; " + "therefore, `%2%' might be removed by the garbage collector") + % gcRoot % storePath); + } + + /* Grab the global GC root, causing us to block while a GC is in + progress. This prevents the set of permanent roots from + increasing while a GC is in progress. */ + store.syncWithGC(); + + return gcRoot; +} + + +/* The file to which we write our temporary roots. */ +static Path fnTempRoots; +static AutoCloseFD fdTempRoots; + + +void LocalStore::addTempRoot(const Path & path) +{ + /* Create the temporary roots file for this process. */ + if (fdTempRoots == -1) { + + while (1) { + Path dir = (format("%1%/%2%") % settings.nixStateDir % tempRootsDir).str(); + createDirs(dir); + + fnTempRoots = (format("%1%/%2%") + % dir % getpid()).str(); + + AutoCloseFD fdGCLock = openGCLock(ltRead); + + if (pathExists(fnTempRoots)) + /* It *must* be stale, since there can be no two + processes with the same pid. */ + unlink(fnTempRoots.c_str()); + + fdTempRoots = openLockFile(fnTempRoots, true); + + fdGCLock.close(); + + debug(format("acquiring read lock on `%1%'") % fnTempRoots); + lockFile(fdTempRoots, ltRead, true); + + /* Check whether the garbage collector didn't get in our + way. */ + struct stat st; + if (fstat(fdTempRoots, &st) == -1) + throw SysError(format("statting `%1%'") % fnTempRoots); + if (st.st_size == 0) break; + + /* The garbage collector deleted this file before we could + get a lock. (It won't delete the file after we get a + lock.) Try again. */ + } + + } + + /* Upgrade the lock to a write lock. This will cause us to block + if the garbage collector is holding our lock. */ + debug(format("acquiring write lock on `%1%'") % fnTempRoots); + lockFile(fdTempRoots, ltWrite, true); + + string s = path + '\0'; + writeFull(fdTempRoots, (const unsigned char *) s.data(), s.size()); + + /* Downgrade to a read lock. */ + debug(format("downgrading to read lock on `%1%'") % fnTempRoots); + lockFile(fdTempRoots, ltRead, true); +} + + +void removeTempRoots() +{ + if (fdTempRoots != -1) { + fdTempRoots.close(); + unlink(fnTempRoots.c_str()); + } +} + + +/* Automatically clean up the temporary roots file when we exit. */ +struct RemoveTempRoots +{ + ~RemoveTempRoots() + { + removeTempRoots(); + } +}; + +static RemoveTempRoots autoRemoveTempRoots __attribute__((unused)); + + +typedef boost::shared_ptr<AutoCloseFD> FDPtr; +typedef list<FDPtr> FDs; + + +static void readTempRoots(PathSet & tempRoots, FDs & fds) +{ + /* Read the `temproots' directory for per-process temporary root + files. */ + Strings tempRootFiles = readDirectory( + (format("%1%/%2%") % settings.nixStateDir % tempRootsDir).str()); + + foreach (Strings::iterator, i, tempRootFiles) { + Path path = (format("%1%/%2%/%3%") % settings.nixStateDir % tempRootsDir % *i).str(); + + debug(format("reading temporary root file `%1%'") % path); + FDPtr fd(new AutoCloseFD(open(path.c_str(), O_RDWR, 0666))); + if (*fd == -1) { + /* It's okay if the file has disappeared. */ + if (errno == ENOENT) continue; + throw SysError(format("opening temporary roots file `%1%'") % path); + } + + /* This should work, but doesn't, for some reason. */ + //FDPtr fd(new AutoCloseFD(openLockFile(path, false))); + //if (*fd == -1) continue; + + /* Try to acquire a write lock without blocking. This can + only succeed if the owning process has died. In that case + we don't care about its temporary roots. */ + if (lockFile(*fd, ltWrite, false)) { + printMsg(lvlError, format("removing stale temporary roots file `%1%'") % path); + unlink(path.c_str()); + writeFull(*fd, (const unsigned char *) "d", 1); + continue; + } + + /* Acquire a read lock. This will prevent the owning process + from upgrading to a write lock, therefore it will block in + addTempRoot(). */ + debug(format("waiting for read lock on `%1%'") % path); + lockFile(*fd, ltRead, true); + + /* Read the entire file. */ + string contents = readFile(*fd); + + /* Extract the roots. */ + string::size_type pos = 0, end; + + while ((end = contents.find((char) 0, pos)) != string::npos) { + Path root(contents, pos, end - pos); + debug(format("got temporary root `%1%'") % root); + assertStorePath(root); + tempRoots.insert(root); + pos = end + 1; + } + + fds.push_back(fd); /* keep open */ + } +} + + +static void foundRoot(StoreAPI & store, + const Path & path, const Path & target, Roots & roots) +{ + Path storePath = toStorePath(target); + if (store.isValidPath(storePath)) + roots[path] = storePath; + else + printMsg(lvlInfo, format("skipping invalid root from `%1%' to `%2%'") % path % storePath); +} + + +static void findRoots(StoreAPI & store, const Path & path, Roots & roots) +{ + try { + + struct stat st = lstat(path); + + if (S_ISDIR(st.st_mode)) { + Strings names = readDirectory(path); + foreach (Strings::iterator, i, names) + findRoots(store, path + "/" + *i, roots); + } + + else if (S_ISLNK(st.st_mode)) { + Path target = readLink(path); + if (isInStore(target)) + foundRoot(store, path, target, roots); + + /* Handle indirect roots. */ + else { + target = absPath(target, dirOf(path)); + if (!pathExists(target)) { + if (isInDir(path, settings.nixStateDir + "/" + gcRootsDir + "/auto")) { + printMsg(lvlInfo, format("removing stale link from `%1%' to `%2%'") % path % target); + unlink(path.c_str()); + } + } else { + struct stat st2 = lstat(target); + if (!S_ISLNK(st2.st_mode)) return; + Path target2 = readLink(target); + if (isInStore(target2)) foundRoot(store, target, target2, roots); + } + } + } + + } + + catch (SysError & e) { + /* We only ignore permanent failures. */ + if (e.errNo == EACCES || e.errNo == ENOENT || e.errNo == ENOTDIR) + printMsg(lvlInfo, format("cannot read potential root `%1%'") % path); + else + throw; + } +} + + +Roots LocalStore::findRoots() +{ + Roots roots; + + /* Process direct roots in {gcroots,manifests,profiles}. */ + nix::findRoots(*this, settings.nixStateDir + "/" + gcRootsDir, roots); + nix::findRoots(*this, settings.nixStateDir + "/manifests", roots); + nix::findRoots(*this, settings.nixStateDir + "/profiles", roots); + + return roots; +} + + +static void addAdditionalRoots(StoreAPI & store, PathSet & roots) +{ + Path rootFinder = getEnv("NIX_ROOT_FINDER", + settings.nixLibexecDir + "/nix/find-runtime-roots.pl"); + + if (rootFinder.empty()) return; + + debug(format("executing `%1%' to find additional roots") % rootFinder); + + string result = runProgram(rootFinder); + + StringSet paths = tokenizeString<StringSet>(result, "\n"); + + foreach (StringSet::iterator, i, paths) { + if (isInStore(*i)) { + Path path = toStorePath(*i); + if (roots.find(path) == roots.end() && store.isValidPath(path)) { + debug(format("got additional root `%1%'") % path); + roots.insert(path); + } + } + } +} + + +struct GCLimitReached { }; + + +struct LocalStore::GCState +{ + GCOptions options; + GCResults & results; + PathSet roots; + PathSet tempRoots; + PathSet dead; + PathSet alive; + bool gcKeepOutputs; + bool gcKeepDerivations; + unsigned long long bytesInvalidated; + Path trashDir; + bool shouldDelete; + GCState(GCResults & results_) : results(results_), bytesInvalidated(0) { } +}; + + +bool LocalStore::isActiveTempFile(const GCState & state, + const Path & path, const string & suffix) +{ + return hasSuffix(path, suffix) + && state.tempRoots.find(string(path, 0, path.size() - suffix.size())) != state.tempRoots.end(); +} + + +void LocalStore::deleteGarbage(GCState & state, const Path & path) +{ + unsigned long long bytesFreed; + deletePath(path, bytesFreed); + state.results.bytesFreed += bytesFreed; +} + + +void LocalStore::deletePathRecursive(GCState & state, const Path & path) +{ + checkInterrupt(); + + unsigned long long size = 0; + + if (isValidPath(path)) { + PathSet referrers; + queryReferrers(path, referrers); + foreach (PathSet::iterator, i, referrers) + if (*i != path) deletePathRecursive(state, *i); + size = queryPathInfo(path).narSize; + invalidatePathChecked(path); + } + + struct stat st; + if (lstat(path.c_str(), &st)) { + if (errno == ENOENT) return; + throw SysError(format("getting status of %1%") % path); + } + + printMsg(lvlInfo, format("deleting `%1%'") % path); + + state.results.paths.insert(path); + + /* If the path is not a regular file or symlink, move it to the + trash directory. The move is to ensure that later (when we're + not holding the global GC lock) we can delete the path without + being afraid that the path has become alive again. Otherwise + delete it right away. */ + if (S_ISDIR(st.st_mode)) { + // Estimate the amount freed using the narSize field. FIXME: + // if the path was not valid, need to determine the actual + // size. + state.bytesInvalidated += size; + // Mac OS X cannot rename directories if they are read-only. + if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1) + throw SysError(format("making `%1%' writable") % path); + Path tmp = state.trashDir + "/" + baseNameOf(path); + if (rename(path.c_str(), tmp.c_str())) + throw SysError(format("unable to rename `%1%' to `%2%'") % path % tmp); + } else + deleteGarbage(state, path); + + if (state.results.bytesFreed + state.bytesInvalidated > state.options.maxFreed) { + printMsg(lvlInfo, format("deleted or invalidated more than %1% bytes; stopping") % state.options.maxFreed); + throw GCLimitReached(); + } +} + + +bool LocalStore::canReachRoot(GCState & state, PathSet & visited, const Path & path) +{ + if (visited.find(path) != visited.end()) return false; + + if (state.alive.find(path) != state.alive.end()) { + return true; + } + + if (state.dead.find(path) != state.dead.end()) { + return false; + } + + if (state.roots.find(path) != state.roots.end()) { + printMsg(lvlDebug, format("cannot delete `%1%' because it's a root") % path); + state.alive.insert(path); + return true; + } + + visited.insert(path); + + if (!isValidPath(path)) return false; + + PathSet incoming; + + /* Don't delete this path if any of its referrers are alive. */ + queryReferrers(path, incoming); + + /* If gc-keep-derivations is set and this is a derivation, then + don't delete the derivation if any of the outputs are alive. */ + if (state.gcKeepDerivations && isDerivation(path)) { + PathSet outputs = queryDerivationOutputs(path); + foreach (PathSet::iterator, i, outputs) + if (isValidPath(*i) && queryDeriver(*i) == path) + incoming.insert(*i); + } + + /* If gc-keep-outputs is set, then don't delete this path if there + are derivers of this path that are not garbage. */ + if (state.gcKeepOutputs) { + PathSet derivers = queryValidDerivers(path); + foreach (PathSet::iterator, i, derivers) + incoming.insert(*i); + } + + foreach (PathSet::iterator, i, incoming) + if (*i != path) + if (canReachRoot(state, visited, *i)) { + state.alive.insert(path); + return true; + } + + return false; +} + + +void LocalStore::tryToDelete(GCState & state, const Path & path) +{ + checkInterrupt(); + + if (path == linksDir || path == state.trashDir) return; + + startNest(nest, lvlDebug, format("considering whether to delete `%1%'") % path); + + if (!isValidPath(path)) { + /* A lock file belonging to a path that we're building right + now isn't garbage. */ + if (isActiveTempFile(state, path, ".lock")) return; + + /* Don't delete .chroot directories for derivations that are + currently being built. */ + if (isActiveTempFile(state, path, ".chroot")) return; + } + + PathSet visited; + + if (canReachRoot(state, visited, path)) { + printMsg(lvlDebug, format("cannot delete `%1%' because it's still reachable") % path); + } else { + /* No path we visited was a root, so everything is garbage. + But we only delete ‘path’ and its referrers here so that + ‘nix-store --delete’ doesn't have the unexpected effect of + recursing into derivations and outputs. */ + state.dead.insert(visited.begin(), visited.end()); + if (state.shouldDelete) + deletePathRecursive(state, path); + } +} + + +/* Unlink all files in /nix/store/.links that have a link count of 1, + which indicates that there are no other links and so they can be + safely deleted. FIXME: race condition with optimisePath(): we + might see a link count of 1 just before optimisePath() increases + the link count. */ +void LocalStore::removeUnusedLinks(const GCState & state) +{ + AutoCloseDir dir = opendir(linksDir.c_str()); + if (!dir) throw SysError(format("opening directory `%1%'") % linksDir); + + long long actualSize = 0, unsharedSize = 0; + + struct dirent * dirent; + while (errno = 0, dirent = readdir(dir)) { + checkInterrupt(); + string name = dirent->d_name; + if (name == "." || name == "..") continue; + Path path = linksDir + "/" + name; + + struct stat st; + if (lstat(path.c_str(), &st) == -1) + throw SysError(format("statting `%1%'") % path); + + if (st.st_nlink != 1) { + unsigned long long size = st.st_blocks * 512ULL; + actualSize += size; + unsharedSize += (st.st_nlink - 1) * size; + continue; + } + + printMsg(lvlTalkative, format("deleting unused link `%1%'") % path); + + if (unlink(path.c_str()) == -1) + throw SysError(format("deleting `%1%'") % path); + + state.results.bytesFreed += st.st_blocks * 512; + } + + struct stat st; + if (stat(linksDir.c_str(), &st) == -1) + throw SysError(format("statting `%1%'") % linksDir); + long long overhead = st.st_blocks * 512ULL; + + printMsg(lvlInfo, format("note: currently hard linking saves %.2f MiB") + % ((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0))); +} + + +void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) +{ + GCState state(results); + state.options = options; + state.trashDir = settings.nixStore + "/trash"; + state.gcKeepOutputs = settings.gcKeepOutputs; + state.gcKeepDerivations = settings.gcKeepDerivations; + + /* Using `--ignore-liveness' with `--delete' can have unintended + consequences if `gc-keep-outputs' or `gc-keep-derivations' are + true (the garbage collector will recurse into deleting the + outputs or derivers, respectively). So disable them. */ + if (options.action == GCOptions::gcDeleteSpecific && options.ignoreLiveness) { + state.gcKeepOutputs = false; + state.gcKeepDerivations = false; + } + + state.shouldDelete = options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific; + + /* Acquire the global GC root. This prevents + a) New roots from being added. + b) Processes from creating new temporary root files. */ + AutoCloseFD fdGCLock = openGCLock(ltWrite); + + /* Find the roots. Since we've grabbed the GC lock, the set of + permanent roots cannot increase now. */ + printMsg(lvlError, format("finding garbage collector roots...")); + Roots rootMap = options.ignoreLiveness ? Roots() : findRoots(); + + foreach (Roots::iterator, i, rootMap) state.roots.insert(i->second); + + /* Add additional roots returned by the program specified by the + NIX_ROOT_FINDER environment variable. This is typically used + to add running programs to the set of roots (to prevent them + from being garbage collected). */ + if (!options.ignoreLiveness) + addAdditionalRoots(*this, state.roots); + + /* Read the temporary roots. This acquires read locks on all + per-process temporary root files. So after this point no paths + can be added to the set of temporary roots. */ + FDs fds; + readTempRoots(state.tempRoots, fds); + state.roots.insert(state.tempRoots.begin(), state.tempRoots.end()); + + /* After this point the set of roots or temporary roots cannot + increase, since we hold locks on everything. So everything + that is not reachable from `roots'. */ + + if (state.shouldDelete) { + if (pathExists(state.trashDir)) deleteGarbage(state, state.trashDir); + createDirs(state.trashDir); + } + + /* Now either delete all garbage paths, or just the specified + paths (for gcDeleteSpecific). */ + + if (options.action == GCOptions::gcDeleteSpecific) { + + foreach (PathSet::iterator, i, options.pathsToDelete) { + assertStorePath(*i); + tryToDelete(state, *i); + if (state.dead.find(*i) == state.dead.end()) + throw Error(format("cannot delete path `%1%' since it is still alive") % *i); + } + + } else if (options.maxFreed > 0) { + + if (state.shouldDelete) + printMsg(lvlError, format("deleting garbage...")); + else + printMsg(lvlError, format("determining live/dead paths...")); + + try { + + AutoCloseDir dir = opendir(settings.nixStore.c_str()); + if (!dir) throw SysError(format("opening directory `%1%'") % settings.nixStore); + + /* Read the store and immediately delete all paths that + aren't valid. When using --max-freed etc., deleting + invalid paths is preferred over deleting unreachable + paths, since unreachable paths could become reachable + again. We don't use readDirectory() here so that GCing + can start faster. */ + Paths entries; + struct dirent * dirent; + while (errno = 0, dirent = readdir(dir)) { + checkInterrupt(); + string name = dirent->d_name; + if (name == "." || name == "..") continue; + Path path = settings.nixStore + "/" + name; + if (isValidPath(path)) + entries.push_back(path); + else + tryToDelete(state, path); + } + + dir.close(); + + /* Now delete the unreachable valid paths. Randomise the + order in which we delete entries to make the collector + less biased towards deleting paths that come + alphabetically first (e.g. /nix/store/000...). This + matters when using --max-freed etc. */ + vector<Path> entries_(entries.begin(), entries.end()); + random_shuffle(entries_.begin(), entries_.end()); + + foreach (vector<Path>::iterator, i, entries_) + tryToDelete(state, *i); + + } catch (GCLimitReached & e) { + } + } + + if (state.options.action == GCOptions::gcReturnLive) { + state.results.paths = state.alive; + return; + } + + if (state.options.action == GCOptions::gcReturnDead) { + state.results.paths = state.dead; + return; + } + + /* Allow other processes to add to the store from here on. */ + fdGCLock.close(); + fds.clear(); + + /* Delete the trash directory. */ + printMsg(lvlInfo, format("deleting `%1%'") % state.trashDir); + deleteGarbage(state, state.trashDir); + + /* Clean up the links directory. */ + if (options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific) { + printMsg(lvlError, format("deleting unused links...")); + removeUnusedLinks(state); + } + + /* While we're at it, vacuum the database. */ + if (options.action == GCOptions::gcDeleteDead) vacuumDB(); +} + + +} diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc new file mode 100644 index 000000000000..68add1982f43 --- /dev/null +++ b/src/libstore/globals.cc @@ -0,0 +1,218 @@ +#include "config.h" + +#include "globals.hh" +#include "util.hh" + +#include <map> +#include <algorithm> + + +namespace nix { + + +/* The default location of the daemon socket, relative to nixStateDir. + The socket is in a directory to allow you to control access to the + Nix daemon by setting the mode/ownership of the directory + appropriately. (This wouldn't work on the socket itself since it + must be deleted and recreated on startup.) */ +#define DEFAULT_SOCKET_PATH "/daemon-socket/socket" + + +Settings settings; + + +Settings::Settings() +{ + keepFailed = false; + keepGoing = false; + tryFallback = false; + buildVerbosity = lvlError; + maxBuildJobs = 1; + buildCores = 1; + readOnlyMode = false; + thisSystem = SYSTEM; + maxSilentTime = 0; + buildTimeout = 0; + useBuildHook = true; + printBuildTrace = false; + reservedSize = 1024 * 1024; + fsyncMetadata = true; + useSQLiteWAL = true; + syncBeforeRegistering = false; + useSubstitutes = true; + useChroot = false; + dirsInChroot.insert("/dev"); + dirsInChroot.insert("/dev/pts"); + impersonateLinux26 = false; + keepLog = true; + compressLog = true; + maxLogSize = 0; + cacheFailure = false; + pollInterval = 5; + checkRootReachability = false; + gcKeepOutputs = false; + gcKeepDerivations = true; + autoOptimiseStore = false; + envKeepDerivations = false; + lockCPU = getEnv("NIX_AFFINITY_HACK", "1") == "1"; + showTrace = false; +} + + +void Settings::processEnvironment() +{ + nixStore = canonPath(getEnv("NIX_STORE_DIR", getEnv("NIX_STORE", NIX_STORE_DIR))); + nixDataDir = canonPath(getEnv("NIX_DATA_DIR", NIX_DATA_DIR)); + nixLogDir = canonPath(getEnv("NIX_LOG_DIR", NIX_LOG_DIR)); + nixStateDir = canonPath(getEnv("NIX_STATE_DIR", NIX_STATE_DIR)); + nixDBPath = getEnv("NIX_DB_DIR", nixStateDir + "/db"); + nixConfDir = canonPath(getEnv("NIX_CONF_DIR", NIX_CONF_DIR)); + nixLibexecDir = canonPath(getEnv("NIX_LIBEXEC_DIR", NIX_LIBEXEC_DIR)); + nixBinDir = canonPath(getEnv("NIX_BIN_DIR", NIX_BIN_DIR)); + nixDaemonSocketFile = canonPath(nixStateDir + DEFAULT_SOCKET_PATH); + + string subs = getEnv("NIX_SUBSTITUTERS", "default"); + if (subs == "default") { +#if 0 + if (getEnv("NIX_OTHER_STORES") != "") + substituters.push_back(nixLibexecDir + "/nix/substituters/copy-from-other-stores.pl"); +#endif + substituters.push_back(nixLibexecDir + "/nix/substituters/download-using-manifests.pl"); + substituters.push_back(nixLibexecDir + "/nix/substituters/download-from-binary-cache.pl"); + } else + substituters = tokenizeString<Strings>(subs, ":"); +} + + +void Settings::loadConfFile() +{ + Path settingsFile = (format("%1%/%2%") % nixConfDir % "nix.conf").str(); + if (!pathExists(settingsFile)) return; + string contents = readFile(settingsFile); + + unsigned int pos = 0; + + while (pos < contents.size()) { + string line; + while (pos < contents.size() && contents[pos] != '\n') + line += contents[pos++]; + pos++; + + string::size_type hash = line.find('#'); + if (hash != string::npos) + line = string(line, 0, hash); + + vector<string> tokens = tokenizeString<vector<string> >(line); + if (tokens.empty()) continue; + + if (tokens.size() < 2 || tokens[1] != "=") + throw Error(format("illegal configuration line `%1%' in `%2%'") % line % settingsFile); + + string name = tokens[0]; + + vector<string>::iterator i = tokens.begin(); + advance(i, 2); + settings[name] = concatStringsSep(" ", Strings(i, tokens.end())); // FIXME: slow + }; +} + + +void Settings::set(const string & name, const string & value) +{ + settings[name] = value; + overrides[name] = value; +} + + +void Settings::update() +{ + get(tryFallback, "build-fallback"); + get(maxBuildJobs, "build-max-jobs"); + get(buildCores, "build-cores"); + get(thisSystem, "system"); + get(maxSilentTime, "build-max-silent-time"); + get(buildTimeout, "build-timeout"); + get(reservedSize, "gc-reserved-space"); + get(fsyncMetadata, "fsync-metadata"); + get(useSQLiteWAL, "use-sqlite-wal"); + get(syncBeforeRegistering, "sync-before-registering"); + get(useSubstitutes, "build-use-substitutes"); + get(buildUsersGroup, "build-users-group"); + get(useChroot, "build-use-chroot"); + get(dirsInChroot, "build-chroot-dirs"); + get(impersonateLinux26, "build-impersonate-linux-26"); + get(keepLog, "build-keep-log"); + get(compressLog, "build-compress-log"); + get(maxLogSize, "build-max-log-size"); + get(cacheFailure, "build-cache-failure"); + get(pollInterval, "build-poll-interval"); + get(checkRootReachability, "gc-check-reachability"); + get(gcKeepOutputs, "gc-keep-outputs"); + get(gcKeepDerivations, "gc-keep-derivations"); + get(autoOptimiseStore, "auto-optimise-store"); + get(envKeepDerivations, "env-keep-derivations"); +} + + +void Settings::get(string & res, const string & name) +{ + SettingsMap::iterator i = settings.find(name); + if (i == settings.end()) return; + res = i->second; +} + + +void Settings::get(bool & res, const string & name) +{ + SettingsMap::iterator i = settings.find(name); + if (i == settings.end()) return; + if (i->second == "true") res = true; + else if (i->second == "false") res = false; + else throw Error(format("configuration option `%1%' should be either `true' or `false', not `%2%'") + % name % i->second); +} + + +void Settings::get(StringSet & res, const string & name) +{ + SettingsMap::iterator i = settings.find(name); + if (i == settings.end()) return; + res.clear(); + Strings ss = tokenizeString<Strings>(i->second); + res.insert(ss.begin(), ss.end()); +} + + +template<class N> void Settings::get(N & res, const string & name) +{ + SettingsMap::iterator i = settings.find(name); + if (i == settings.end()) return; + if (!string2Int(i->second, res)) + throw Error(format("configuration setting `%1%' should have an integer value") % name); +} + + +string Settings::pack() +{ + string s; + foreach (SettingsMap::iterator, i, settings) { + if (i->first.find('\n') != string::npos || + i->first.find('=') != string::npos || + i->second.find('\n') != string::npos) + throw Error("illegal option name/value"); + s += i->first; s += '='; s += i->second; s += '\n'; + } + return s; +} + + +Settings::SettingsMap Settings::getOverrides() +{ + return overrides; +} + + +const string nixVersion = PACKAGE_VERSION; + + +} diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh new file mode 100644 index 000000000000..9300edbe9615 --- /dev/null +++ b/src/libstore/globals.hh @@ -0,0 +1,209 @@ +#pragma once + +#include "types.hh" + +#include <map> +#include <sys/types.h> + + +namespace nix { + + +struct Settings { + + typedef std::map<string, string> SettingsMap; + + Settings(); + + void processEnvironment(); + + void loadConfFile(); + + void set(const string & name, const string & value); + + void update(); + + string pack(); + + SettingsMap getOverrides(); + + /* The directory where we store sources and derived files. */ + Path nixStore; + + Path nixDataDir; /* !!! fix */ + + /* The directory where we log various operations. */ + Path nixLogDir; + + /* The directory where state is stored. */ + Path nixStateDir; + + /* The directory where we keep the SQLite database. */ + Path nixDBPath; + + /* The directory where configuration files are stored. */ + Path nixConfDir; + + /* The directory where internal helper programs are stored. */ + Path nixLibexecDir; + + /* The directory where the main programs are stored. */ + Path nixBinDir; + + /* File name of the socket the daemon listens to. */ + Path nixDaemonSocketFile; + + /* Whether to keep temporary directories of failed builds. */ + bool keepFailed; + + /* Whether to keep building subgoals when a sibling (another + subgoal of the same goal) fails. */ + bool keepGoing; + + /* Whether, if we cannot realise the known closure corresponding + to a derivation, we should try to normalise the derivation + instead. */ + bool tryFallback; + + /* Verbosity level for build output. */ + Verbosity buildVerbosity; + + /* Maximum number of parallel build jobs. 0 means unlimited. */ + unsigned int maxBuildJobs; + + /* Number of CPU cores to utilize in parallel within a build, + i.e. by passing this number to Make via '-j'. 0 means that the + number of actual CPU cores on the local host ought to be + auto-detected. */ + unsigned int buildCores; + + /* Read-only mode. Don't copy stuff to the store, don't change + the database. */ + bool readOnlyMode; + + /* The canonical system name, as returned by config.guess. */ + string thisSystem; + + /* The maximum time in seconds that a builer can go without + producing any output on stdout/stderr before it is killed. 0 + means infinity. */ + time_t maxSilentTime; + + /* The maximum duration in seconds that a builder can run. 0 + means infinity. */ + time_t buildTimeout; + + /* The substituters. There are programs that can somehow realise + a store path without building, e.g., by downloading it or + copying it from a CD. */ + Paths substituters; + + /* Whether to use build hooks (for distributed builds). Sometimes + users want to disable this from the command-line. */ + bool useBuildHook; + + /* Whether buildDerivations() should print out lines on stderr in + a fixed format to allow its progress to be monitored. Each + line starts with a "@". The following are defined: + + @ build-started <drvpath> <outpath> <system> <logfile> + @ build-failed <drvpath> <outpath> <exitcode> <error text> + @ build-succeeded <drvpath> <outpath> + @ substituter-started <outpath> <substituter> + @ substituter-failed <outpath> <exitcode> <error text> + @ substituter-succeeded <outpath> + + Best combined with --no-build-output, otherwise stderr might + conceivably contain lines in this format printed by the + builders. */ + bool printBuildTrace; + + /* Amount of reserved space for the garbage collector + (/nix/var/nix/db/reserved). */ + off_t reservedSize; + + /* Whether SQLite should use fsync. */ + bool fsyncMetadata; + + /* Whether SQLite should use WAL mode. */ + bool useSQLiteWAL; + + /* Whether to call sync() before registering a path as valid. */ + bool syncBeforeRegistering; + + /* Whether to use substitutes. */ + bool useSubstitutes; + + /* The Unix group that contains the build users. */ + string buildUsersGroup; + + /* Whether to build in chroot. */ + bool useChroot; + + /* The directories from the host filesystem to be included in the + chroot. */ + StringSet dirsInChroot; + + /* Whether to impersonate a Linux 2.6 machine on newer kernels. */ + bool impersonateLinux26; + + /* Whether to store build logs. */ + bool keepLog; + + /* Whether to compress logs. */ + bool compressLog; + + /* Maximum number of bytes a builder can write to stdout/stderr + before being killed (0 means no limit). */ + unsigned long maxLogSize; + + /* Whether to cache build failures. */ + bool cacheFailure; + + /* How often (in seconds) to poll for locks. */ + unsigned int pollInterval; + + /* Whether to check if new GC roots can in fact be found by the + garbage collector. */ + bool checkRootReachability; + + /* Whether the garbage collector should keep outputs of live + derivations. */ + bool gcKeepOutputs; + + /* Whether the garbage collector should keep derivers of live + paths. */ + bool gcKeepDerivations; + + /* Whether to automatically replace files with identical contents + with hard links. */ + bool autoOptimiseStore; + + /* Whether to add derivations as a dependency of user environments + (to prevent them from being GCed). */ + bool envKeepDerivations; + + /* Whether to lock the Nix client and worker to the same CPU. */ + bool lockCPU; + + /* Whether to show a stack trace if Nix evaluation fails. */ + bool showTrace; + +private: + SettingsMap settings, overrides; + + void get(string & res, const string & name); + void get(bool & res, const string & name); + void get(StringSet & res, const string & name); + template<class N> void get(N & res, const string & name); +}; + + +// FIXME: don't use a global variable. +extern Settings settings; + + +extern const string nixVersion; + + +} diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc new file mode 100644 index 000000000000..4ac7d7f4e6f7 --- /dev/null +++ b/src/libstore/local-store.cc @@ -0,0 +1,2000 @@ +#include "config.h" +#include "local-store.hh" +#include "globals.hh" +#include "archive.hh" +#include "pathlocks.hh" +#include "worker-protocol.hh" +#include "derivations.hh" +#include "affinity.hh" + +#include <iostream> +#include <algorithm> +#include <cstring> + +#include <sys/types.h> +#include <sys/stat.h> +#include <sys/time.h> +#include <unistd.h> +#include <utime.h> +#include <fcntl.h> +#include <errno.h> +#include <stdio.h> +#include <time.h> + +#if HAVE_UNSHARE && HAVE_STATVFS && HAVE_SYS_MOUNT_H +#include <sched.h> +#include <sys/statvfs.h> +#include <sys/mount.h> +#endif + +#if HAVE_LINUX_FS_H +#include <linux/fs.h> +#include <sys/ioctl.h> +#include <errno.h> +#endif + +#include <sqlite3.h> + + +namespace nix { + + +MakeError(SQLiteError, Error); +MakeError(SQLiteBusy, SQLiteError); + + +static void throwSQLiteError(sqlite3 * db, const format & f) + __attribute__ ((noreturn)); + +static void throwSQLiteError(sqlite3 * db, const format & f) +{ + int err = sqlite3_errcode(db); + if (err == SQLITE_BUSY || err == SQLITE_PROTOCOL) { + if (err == SQLITE_PROTOCOL) + printMsg(lvlError, "warning: SQLite database is busy (SQLITE_PROTOCOL)"); + else { + static bool warned = false; + if (!warned) { + printMsg(lvlError, "warning: SQLite database is busy"); + warned = true; + } + } + /* Sleep for a while since retrying the transaction right away + is likely to fail again. */ +#if HAVE_NANOSLEEP + struct timespec t; + t.tv_sec = 0; + t.tv_nsec = (random() % 100) * 1000 * 1000; /* <= 0.1s */ + nanosleep(&t, 0); +#else + sleep(1); +#endif + throw SQLiteBusy(format("%1%: %2%") % f.str() % sqlite3_errmsg(db)); + } + else + throw SQLiteError(format("%1%: %2%") % f.str() % sqlite3_errmsg(db)); +} + + +/* Convenience macros for retrying a SQLite transaction. */ +#define retry_sqlite while (1) { try { +#define end_retry_sqlite break; } catch (SQLiteBusy & e) { } } + + +SQLite::~SQLite() +{ + try { + if (db && sqlite3_close(db) != SQLITE_OK) + throwSQLiteError(db, "closing database"); + } catch (...) { + ignoreException(); + } +} + + +void SQLiteStmt::create(sqlite3 * db, const string & s) +{ + checkInterrupt(); + assert(!stmt); + if (sqlite3_prepare_v2(db, s.c_str(), -1, &stmt, 0) != SQLITE_OK) + throwSQLiteError(db, "creating statement"); + this->db = db; +} + + +void SQLiteStmt::reset() +{ + assert(stmt); + /* Note: sqlite3_reset() returns the error code for the most + recent call to sqlite3_step(). So ignore it. */ + sqlite3_reset(stmt); + curArg = 1; +} + + +SQLiteStmt::~SQLiteStmt() +{ + try { + if (stmt && sqlite3_finalize(stmt) != SQLITE_OK) + throwSQLiteError(db, "finalizing statement"); + } catch (...) { + ignoreException(); + } +} + + +void SQLiteStmt::bind(const string & value) +{ + if (sqlite3_bind_text(stmt, curArg++, value.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) + throwSQLiteError(db, "binding argument"); +} + + +void SQLiteStmt::bind(int value) +{ + if (sqlite3_bind_int(stmt, curArg++, value) != SQLITE_OK) + throwSQLiteError(db, "binding argument"); +} + + +void SQLiteStmt::bind64(long long value) +{ + if (sqlite3_bind_int64(stmt, curArg++, value) != SQLITE_OK) + throwSQLiteError(db, "binding argument"); +} + + +void SQLiteStmt::bind() +{ + if (sqlite3_bind_null(stmt, curArg++) != SQLITE_OK) + throwSQLiteError(db, "binding argument"); +} + + +/* Helper class to ensure that prepared statements are reset when + leaving the scope that uses them. Unfinished prepared statements + prevent transactions from being aborted, and can cause locks to be + kept when they should be released. */ +struct SQLiteStmtUse +{ + SQLiteStmt & stmt; + SQLiteStmtUse(SQLiteStmt & stmt) : stmt(stmt) + { + stmt.reset(); + } + ~SQLiteStmtUse() + { + try { + stmt.reset(); + } catch (...) { + ignoreException(); + } + } +}; + + +struct SQLiteTxn +{ + bool active; + sqlite3 * db; + + SQLiteTxn(sqlite3 * db) : active(false) { + this->db = db; + if (sqlite3_exec(db, "begin;", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(db, "starting transaction"); + active = true; + } + + void commit() + { + if (sqlite3_exec(db, "commit;", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(db, "committing transaction"); + active = false; + } + + ~SQLiteTxn() + { + try { + if (active && sqlite3_exec(db, "rollback;", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(db, "aborting transaction"); + } catch (...) { + ignoreException(); + } + } +}; + + +void checkStoreNotSymlink() +{ + if (getEnv("NIX_IGNORE_SYMLINK_STORE") == "1") return; + Path path = settings.nixStore; + struct stat st; + while (path != "/") { + if (lstat(path.c_str(), &st)) + throw SysError(format("getting status of `%1%'") % path); + if (S_ISLNK(st.st_mode)) + throw Error(format( + "the path `%1%' is a symlink; " + "this is not allowed for the Nix store and its parent directories") + % path); + path = dirOf(path); + } +} + + +LocalStore::LocalStore(bool reserveSpace) + : didSetSubstituterEnv(false) +{ + schemaPath = settings.nixDBPath + "/schema"; + + if (settings.readOnlyMode) { + openDB(false); + return; + } + + /* Create missing state directories if they don't already exist. */ + createDirs(settings.nixStore); + makeStoreWritable(); + createDirs(linksDir = settings.nixStore + "/.links"); + Path profilesDir = settings.nixStateDir + "/profiles"; + createDirs(settings.nixStateDir + "/profiles"); + createDirs(settings.nixStateDir + "/temproots"); + createDirs(settings.nixDBPath); + Path gcRootsDir = settings.nixStateDir + "/gcroots"; + if (!pathExists(gcRootsDir)) { + createDirs(gcRootsDir); + if (symlink(profilesDir.c_str(), (gcRootsDir + "/profiles").c_str()) == -1) + throw SysError(format("creating symlink to `%1%'") % profilesDir); + } + + checkStoreNotSymlink(); + + /* We can't open a SQLite database if the disk is full. Since + this prevents the garbage collector from running when it's most + needed, we reserve some dummy space that we can free just + before doing a garbage collection. */ + try { + Path reservedPath = settings.nixDBPath + "/reserved"; + if (reserveSpace) { + struct stat st; + if (stat(reservedPath.c_str(), &st) == -1 || + st.st_size != settings.reservedSize) + writeFile(reservedPath, string(settings.reservedSize, 'X')); + } + else + deletePath(reservedPath); + } catch (SysError & e) { /* don't care about errors */ + } + + /* Acquire the big fat lock in shared mode to make sure that no + schema upgrade is in progress. */ + try { + Path globalLockPath = settings.nixDBPath + "/big-lock"; + globalLock = openLockFile(globalLockPath.c_str(), true); + } catch (SysError & e) { + if (e.errNo != EACCES) throw; + settings.readOnlyMode = true; + openDB(false); + return; + } + + if (!lockFile(globalLock, ltRead, false)) { + printMsg(lvlError, "waiting for the big Nix store lock..."); + lockFile(globalLock, ltRead, true); + } + + /* Check the current database schema and if necessary do an + upgrade. */ + int curSchema = getSchema(); + if (curSchema > nixSchemaVersion) + throw Error(format("current Nix store schema is version %1%, but I only support %2%") + % curSchema % nixSchemaVersion); + + else if (curSchema == 0) { /* new store */ + curSchema = nixSchemaVersion; + openDB(true); + writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str()); + } + + else if (curSchema < nixSchemaVersion) { + if (curSchema < 5) + throw Error( + "Your Nix store has a database in Berkeley DB format,\n" + "which is no longer supported. To convert to the new format,\n" + "please upgrade Nix to version 0.12 first."); + + if (!lockFile(globalLock, ltWrite, false)) { + printMsg(lvlError, "waiting for exclusive access to the Nix store..."); + lockFile(globalLock, ltWrite, true); + } + + /* Get the schema version again, because another process may + have performed the upgrade already. */ + curSchema = getSchema(); + + if (curSchema < 6) upgradeStore6(); + else if (curSchema < 7) { upgradeStore7(); openDB(true); } + + writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str()); + + lockFile(globalLock, ltRead, true); + } + + else openDB(false); +} + + +LocalStore::~LocalStore() +{ + try { + foreach (RunningSubstituters::iterator, i, runningSubstituters) { + if (i->second.disabled) continue; + i->second.to.close(); + i->second.from.close(); + i->second.error.close(); + i->second.pid.wait(true); + } + } catch (...) { + ignoreException(); + } +} + + +int LocalStore::getSchema() +{ + int curSchema = 0; + if (pathExists(schemaPath)) { + string s = readFile(schemaPath); + if (!string2Int(s, curSchema)) + throw Error(format("`%1%' is corrupt") % schemaPath); + } + return curSchema; +} + + +void LocalStore::openDB(bool create) +{ + if (access(settings.nixDBPath.c_str(), R_OK | W_OK)) + throw SysError(format("Nix database directory `%1%' is not writable") % settings.nixDBPath); + + /* Open the Nix database. */ + string dbPath = settings.nixDBPath + "/db.sqlite"; + if (sqlite3_open_v2(dbPath.c_str(), &db.db, + SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK) + throw Error(format("cannot open Nix database `%1%'") % dbPath); + + if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK) + throwSQLiteError(db, "setting timeout"); + + if (sqlite3_exec(db, "pragma foreign_keys = 1;", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(db, "enabling foreign keys"); + + /* !!! check whether sqlite has been built with foreign key + support */ + + /* Whether SQLite should fsync(). "Normal" synchronous mode + should be safe enough. If the user asks for it, don't sync at + all. This can cause database corruption if the system + crashes. */ + string syncMode = settings.fsyncMetadata ? "normal" : "off"; + if (sqlite3_exec(db, ("pragma synchronous = " + syncMode + ";").c_str(), 0, 0, 0) != SQLITE_OK) + throwSQLiteError(db, "setting synchronous mode"); + + /* Set the SQLite journal mode. WAL mode is fastest, so it's the + default. */ + string mode = settings.useSQLiteWAL ? "wal" : "truncate"; + string prevMode; + { + SQLiteStmt stmt; + stmt.create(db, "pragma main.journal_mode;"); + if (sqlite3_step(stmt) != SQLITE_ROW) + throwSQLiteError(db, "querying journal mode"); + prevMode = string((const char *) sqlite3_column_text(stmt, 0)); + } + if (prevMode != mode && + sqlite3_exec(db, ("pragma main.journal_mode = " + mode + ";").c_str(), 0, 0, 0) != SQLITE_OK) + throwSQLiteError(db, "setting journal mode"); + + /* Increase the auto-checkpoint interval to 40000 pages. This + seems enough to ensure that instantiating the NixOS system + derivation is done in a single fsync(). */ + if (mode == "wal" && sqlite3_exec(db, "pragma wal_autocheckpoint = 40000;", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(db, "setting autocheckpoint interval"); + + /* Initialise the database schema, if necessary. */ + if (create) { + const char * schema = +#include "schema.sql.hh" + ; + if (sqlite3_exec(db, (const char *) schema, 0, 0, 0) != SQLITE_OK) + throwSQLiteError(db, "initialising database schema"); + } + + /* Prepare SQL statements. */ + stmtRegisterValidPath.create(db, + "insert into ValidPaths (path, hash, registrationTime, deriver, narSize) values (?, ?, ?, ?, ?);"); + stmtUpdatePathInfo.create(db, + "update ValidPaths set narSize = ?, hash = ? where path = ?;"); + stmtAddReference.create(db, + "insert or replace into Refs (referrer, reference) values (?, ?);"); + stmtQueryPathInfo.create(db, + "select id, hash, registrationTime, deriver, narSize from ValidPaths where path = ?;"); + stmtQueryReferences.create(db, + "select path from Refs join ValidPaths on reference = id where referrer = ?;"); + stmtQueryReferrers.create(db, + "select path from Refs join ValidPaths on referrer = id where reference = (select id from ValidPaths where path = ?);"); + stmtInvalidatePath.create(db, + "delete from ValidPaths where path = ?;"); + stmtRegisterFailedPath.create(db, + "insert or ignore into FailedPaths (path, time) values (?, ?);"); + stmtHasPathFailed.create(db, + "select time from FailedPaths where path = ?;"); + stmtQueryFailedPaths.create(db, + "select path from FailedPaths;"); + // If the path is a derivation, then clear its outputs. + stmtClearFailedPath.create(db, + "delete from FailedPaths where ?1 = '*' or path = ?1 " + "or path in (select d.path from DerivationOutputs d join ValidPaths v on d.drv = v.id where v.path = ?1);"); + stmtAddDerivationOutput.create(db, + "insert or replace into DerivationOutputs (drv, id, path) values (?, ?, ?);"); + stmtQueryValidDerivers.create(db, + "select v.id, v.path from DerivationOutputs d join ValidPaths v on d.drv = v.id where d.path = ?;"); + stmtQueryDerivationOutputs.create(db, + "select id, path from DerivationOutputs where drv = ?;"); + // Use "path >= ?" with limit 1 rather than "path like '?%'" to + // ensure efficient lookup. + stmtQueryPathFromHashPart.create(db, + "select path from ValidPaths where path >= ? limit 1;"); +} + + +/* To improve purity, users may want to make the Nix store a read-only + bind mount. So make the Nix store writable for this process. */ +void LocalStore::makeStoreWritable() +{ +#if HAVE_UNSHARE && HAVE_STATVFS && HAVE_SYS_MOUNT_H && defined(MS_BIND) && defined(MS_REMOUNT) + if (getuid() != 0) return; + /* Check if /nix/store is on a read-only mount. */ + struct statvfs stat; + if (statvfs(settings.nixStore.c_str(), &stat) !=0) + throw SysError("Getting info of nix store mountpoint"); + + if (stat.f_flag & ST_RDONLY) { + if (unshare(CLONE_NEWNS) == -1) + throw SysError("setting up a private mount namespace"); + + if (mount(0, settings.nixStore.c_str(), 0, MS_REMOUNT | MS_BIND, 0) == -1) + throw SysError(format("remounting %1% writable") % settings.nixStore); + } +#endif +} + + +const time_t mtimeStore = 1; /* 1 second into the epoch */ + + +static void canonicaliseTimestampAndPermissions(const Path & path, const struct stat & st) +{ + if (!S_ISLNK(st.st_mode)) { + + /* Mask out all type related bits. */ + mode_t mode = st.st_mode & ~S_IFMT; + + if (mode != 0444 && mode != 0555) { + mode = (st.st_mode & S_IFMT) + | 0444 + | (st.st_mode & S_IXUSR ? 0111 : 0); + if (chmod(path.c_str(), mode) == -1) + throw SysError(format("changing mode of `%1%' to %2$o") % path % mode); + } + + } + + if (st.st_mtime != mtimeStore) { + struct timeval times[2]; + times[0].tv_sec = st.st_atime; + times[0].tv_usec = 0; + times[1].tv_sec = mtimeStore; + times[1].tv_usec = 0; +#if HAVE_LUTIMES + if (lutimes(path.c_str(), times) == -1) + if (errno != ENOSYS || + (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1)) +#else + if (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1) +#endif + throw SysError(format("changing modification time of `%1%'") % path); + } +} + + +void canonicaliseTimestampAndPermissions(const Path & path) +{ + struct stat st; + if (lstat(path.c_str(), &st)) + throw SysError(format("getting attributes of path `%1%'") % path); + canonicaliseTimestampAndPermissions(path, st); +} + + +static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSeen & inodesSeen) +{ + checkInterrupt(); + + struct stat st; + if (lstat(path.c_str(), &st)) + throw SysError(format("getting attributes of path `%1%'") % path); + + /* Really make sure that the path is of a supported type. This + has already been checked in dumpPath(). */ + assert(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode)); + + /* Fail if the file is not owned by the build user. This prevents + us from messing up the ownership/permissions of files + hard-linked into the output (e.g. "ln /etc/shadow $out/foo"). + However, ignore files that we chown'ed ourselves previously to + ensure that we don't fail on hard links within the same build + (i.e. "touch $out/foo; ln $out/foo $out/bar"). */ + if (fromUid != (uid_t) -1 && st.st_uid != fromUid) { + assert(!S_ISDIR(st.st_mode)); + if (inodesSeen.find(Inode(st.st_dev, st.st_ino)) == inodesSeen.end()) + throw BuildError(format("invalid ownership on file `%1%'") % path); + mode_t mode = st.st_mode & ~S_IFMT; + assert(S_ISLNK(st.st_mode) || (st.st_uid == geteuid() && (mode == 0444 || mode == 0555) && st.st_mtime == mtimeStore)); + return; + } + + inodesSeen.insert(Inode(st.st_dev, st.st_ino)); + + canonicaliseTimestampAndPermissions(path, st); + + /* Change ownership to the current uid. If it's a symlink, use + lchown if available, otherwise don't bother. Wrong ownership + of a symlink doesn't matter, since the owning user can't change + the symlink and can't delete it because the directory is not + writable. The only exception is top-level paths in the Nix + store (since that directory is group-writable for the Nix build + users group); we check for this case below. */ + if (st.st_uid != geteuid()) { +#if HAVE_LCHOWN + if (lchown(path.c_str(), geteuid(), (gid_t) -1) == -1) +#else + if (!S_ISLNK(st.st_mode) && + chown(path.c_str(), geteuid(), (gid_t) -1) == -1) +#endif + throw SysError(format("changing owner of `%1%' to %2%") + % path % geteuid()); + } + + if (S_ISDIR(st.st_mode)) { + Strings names = readDirectory(path); + foreach (Strings::iterator, i, names) + canonicalisePathMetaData_(path + "/" + *i, fromUid, inodesSeen); + } +} + + +void canonicalisePathMetaData(const Path & path, uid_t fromUid, InodesSeen & inodesSeen) +{ + canonicalisePathMetaData_(path, fromUid, inodesSeen); + + /* On platforms that don't have lchown(), the top-level path can't + be a symlink, since we can't change its ownership. */ + struct stat st; + if (lstat(path.c_str(), &st)) + throw SysError(format("getting attributes of path `%1%'") % path); + + if (st.st_uid != geteuid()) { + assert(S_ISLNK(st.st_mode)); + throw Error(format("wrong ownership of top-level store path `%1%'") % path); + } +} + + +void canonicalisePathMetaData(const Path & path, uid_t fromUid) +{ + InodesSeen inodesSeen; + canonicalisePathMetaData(path, fromUid, inodesSeen); +} + + +void LocalStore::checkDerivationOutputs(const Path & drvPath, const Derivation & drv) +{ + string drvName = storePathToName(drvPath); + assert(isDerivation(drvName)); + drvName = string(drvName, 0, drvName.size() - drvExtension.size()); + + if (isFixedOutputDrv(drv)) { + DerivationOutputs::const_iterator out = drv.outputs.find("out"); + if (out == drv.outputs.end()) + throw Error(format("derivation `%1%' does not have an output named `out'") % drvPath); + + bool recursive; HashType ht; Hash h; + out->second.parseHashInfo(recursive, ht, h); + Path outPath = makeFixedOutputPath(recursive, ht, h, drvName); + + StringPairs::const_iterator j = drv.env.find("out"); + if (out->second.path != outPath || j == drv.env.end() || j->second != outPath) + throw Error(format("derivation `%1%' has incorrect output `%2%', should be `%3%'") + % drvPath % out->second.path % outPath); + } + + else { + Derivation drvCopy(drv); + foreach (DerivationOutputs::iterator, i, drvCopy.outputs) { + i->second.path = ""; + drvCopy.env[i->first] = ""; + } + + Hash h = hashDerivationModulo(*this, drvCopy); + + foreach (DerivationOutputs::const_iterator, i, drv.outputs) { + Path outPath = makeOutputPath(i->first, h, drvName); + StringPairs::const_iterator j = drv.env.find(i->first); + if (i->second.path != outPath || j == drv.env.end() || j->second != outPath) + throw Error(format("derivation `%1%' has incorrect output `%2%', should be `%3%'") + % drvPath % i->second.path % outPath); + } + } +} + + +unsigned long long LocalStore::addValidPath(const ValidPathInfo & info, bool checkOutputs) +{ + SQLiteStmtUse use(stmtRegisterValidPath); + stmtRegisterValidPath.bind(info.path); + stmtRegisterValidPath.bind("sha256:" + printHash(info.hash)); + stmtRegisterValidPath.bind(info.registrationTime == 0 ? time(0) : info.registrationTime); + if (info.deriver != "") + stmtRegisterValidPath.bind(info.deriver); + else + stmtRegisterValidPath.bind(); // null + if (info.narSize != 0) + stmtRegisterValidPath.bind64(info.narSize); + else + stmtRegisterValidPath.bind(); // null + if (sqlite3_step(stmtRegisterValidPath) != SQLITE_DONE) + throwSQLiteError(db, format("registering valid path `%1%' in database") % info.path); + unsigned long long id = sqlite3_last_insert_rowid(db); + + /* If this is a derivation, then store the derivation outputs in + the database. This is useful for the garbage collector: it can + efficiently query whether a path is an output of some + derivation. */ + if (isDerivation(info.path)) { + Derivation drv = parseDerivation(readFile(info.path)); + + /* Verify that the output paths in the derivation are correct + (i.e., follow the scheme for computing output paths from + derivations). Note that if this throws an error, then the + DB transaction is rolled back, so the path validity + registration above is undone. */ + if (checkOutputs) checkDerivationOutputs(info.path, drv); + + foreach (DerivationOutputs::iterator, i, drv.outputs) { + SQLiteStmtUse use(stmtAddDerivationOutput); + stmtAddDerivationOutput.bind(id); + stmtAddDerivationOutput.bind(i->first); + stmtAddDerivationOutput.bind(i->second.path); + if (sqlite3_step(stmtAddDerivationOutput) != SQLITE_DONE) + throwSQLiteError(db, format("adding derivation output for `%1%' in database") % info.path); + } + } + + return id; +} + + +void LocalStore::addReference(unsigned long long referrer, unsigned long long reference) +{ + SQLiteStmtUse use(stmtAddReference); + stmtAddReference.bind(referrer); + stmtAddReference.bind(reference); + if (sqlite3_step(stmtAddReference) != SQLITE_DONE) + throwSQLiteError(db, "adding reference to database"); +} + + +void LocalStore::registerFailedPath(const Path & path) +{ + retry_sqlite { + SQLiteStmtUse use(stmtRegisterFailedPath); + stmtRegisterFailedPath.bind(path); + stmtRegisterFailedPath.bind(time(0)); + if (sqlite3_step(stmtRegisterFailedPath) != SQLITE_DONE) + throwSQLiteError(db, format("registering failed path `%1%'") % path); + } end_retry_sqlite; +} + + +bool LocalStore::hasPathFailed(const Path & path) +{ + retry_sqlite { + SQLiteStmtUse use(stmtHasPathFailed); + stmtHasPathFailed.bind(path); + int res = sqlite3_step(stmtHasPathFailed); + if (res != SQLITE_DONE && res != SQLITE_ROW) + throwSQLiteError(db, "querying whether path failed"); + return res == SQLITE_ROW; + } end_retry_sqlite; +} + + +PathSet LocalStore::queryFailedPaths() +{ + retry_sqlite { + SQLiteStmtUse use(stmtQueryFailedPaths); + + PathSet res; + int r; + while ((r = sqlite3_step(stmtQueryFailedPaths)) == SQLITE_ROW) { + const char * s = (const char *) sqlite3_column_text(stmtQueryFailedPaths, 0); + assert(s); + res.insert(s); + } + + if (r != SQLITE_DONE) + throwSQLiteError(db, "error querying failed paths"); + + return res; + } end_retry_sqlite; +} + + +void LocalStore::clearFailedPaths(const PathSet & paths) +{ + retry_sqlite { + SQLiteTxn txn(db); + + foreach (PathSet::const_iterator, i, paths) { + SQLiteStmtUse use(stmtClearFailedPath); + stmtClearFailedPath.bind(*i); + if (sqlite3_step(stmtClearFailedPath) != SQLITE_DONE) + throwSQLiteError(db, format("clearing failed path `%1%' in database") % *i); + } + + txn.commit(); + } end_retry_sqlite; +} + + +Hash parseHashField(const Path & path, const string & s) +{ + string::size_type colon = s.find(':'); + if (colon == string::npos) + throw Error(format("corrupt hash `%1%' in valid-path entry for `%2%'") + % s % path); + HashType ht = parseHashType(string(s, 0, colon)); + if (ht == htUnknown) + throw Error(format("unknown hash type `%1%' in valid-path entry for `%2%'") + % string(s, 0, colon) % path); + return parseHash(ht, string(s, colon + 1)); +} + + +ValidPathInfo LocalStore::queryPathInfo(const Path & path) +{ + ValidPathInfo info; + info.path = path; + + assertStorePath(path); + + retry_sqlite { + + /* Get the path info. */ + SQLiteStmtUse use1(stmtQueryPathInfo); + + stmtQueryPathInfo.bind(path); + + int r = sqlite3_step(stmtQueryPathInfo); + if (r == SQLITE_DONE) throw Error(format("path `%1%' is not valid") % path); + if (r != SQLITE_ROW) throwSQLiteError(db, "querying path in database"); + + info.id = sqlite3_column_int(stmtQueryPathInfo, 0); + + const char * s = (const char *) sqlite3_column_text(stmtQueryPathInfo, 1); + assert(s); + info.hash = parseHashField(path, s); + + info.registrationTime = sqlite3_column_int(stmtQueryPathInfo, 2); + + s = (const char *) sqlite3_column_text(stmtQueryPathInfo, 3); + if (s) info.deriver = s; + + /* Note that narSize = NULL yields 0. */ + info.narSize = sqlite3_column_int64(stmtQueryPathInfo, 4); + + /* Get the references. */ + SQLiteStmtUse use2(stmtQueryReferences); + + stmtQueryReferences.bind(info.id); + + while ((r = sqlite3_step(stmtQueryReferences)) == SQLITE_ROW) { + s = (const char *) sqlite3_column_text(stmtQueryReferences, 0); + assert(s); + info.references.insert(s); + } + + if (r != SQLITE_DONE) + throwSQLiteError(db, format("error getting references of `%1%'") % path); + + return info; + } end_retry_sqlite; +} + + +/* Update path info in the database. Currently only updates the + narSize field. */ +void LocalStore::updatePathInfo(const ValidPathInfo & info) +{ + SQLiteStmtUse use(stmtUpdatePathInfo); + if (info.narSize != 0) + stmtUpdatePathInfo.bind64(info.narSize); + else + stmtUpdatePathInfo.bind(); // null + stmtUpdatePathInfo.bind("sha256:" + printHash(info.hash)); + stmtUpdatePathInfo.bind(info.path); + if (sqlite3_step(stmtUpdatePathInfo) != SQLITE_DONE) + throwSQLiteError(db, format("updating info of path `%1%' in database") % info.path); +} + + +unsigned long long LocalStore::queryValidPathId(const Path & path) +{ + SQLiteStmtUse use(stmtQueryPathInfo); + stmtQueryPathInfo.bind(path); + int res = sqlite3_step(stmtQueryPathInfo); + if (res == SQLITE_ROW) return sqlite3_column_int(stmtQueryPathInfo, 0); + if (res == SQLITE_DONE) throw Error(format("path `%1%' is not valid") % path); + throwSQLiteError(db, "querying path in database"); +} + + +bool LocalStore::isValidPath_(const Path & path) +{ + SQLiteStmtUse use(stmtQueryPathInfo); + stmtQueryPathInfo.bind(path); + int res = sqlite3_step(stmtQueryPathInfo); + if (res != SQLITE_DONE && res != SQLITE_ROW) + throwSQLiteError(db, "querying path in database"); + return res == SQLITE_ROW; +} + + +bool LocalStore::isValidPath(const Path & path) +{ + retry_sqlite { + return isValidPath_(path); + } end_retry_sqlite; +} + + +PathSet LocalStore::queryValidPaths(const PathSet & paths) +{ + retry_sqlite { + PathSet res; + foreach (PathSet::const_iterator, i, paths) + if (isValidPath_(*i)) res.insert(*i); + return res; + } end_retry_sqlite; +} + + +PathSet LocalStore::queryAllValidPaths() +{ + retry_sqlite { + SQLiteStmt stmt; + stmt.create(db, "select path from ValidPaths"); + + PathSet res; + int r; + while ((r = sqlite3_step(stmt)) == SQLITE_ROW) { + const char * s = (const char *) sqlite3_column_text(stmt, 0); + assert(s); + res.insert(s); + } + + if (r != SQLITE_DONE) + throwSQLiteError(db, "error getting valid paths"); + + return res; + } end_retry_sqlite; +} + + +void LocalStore::queryReferences(const Path & path, + PathSet & references) +{ + ValidPathInfo info = queryPathInfo(path); + references.insert(info.references.begin(), info.references.end()); +} + + +void LocalStore::queryReferrers_(const Path & path, PathSet & referrers) +{ + SQLiteStmtUse use(stmtQueryReferrers); + + stmtQueryReferrers.bind(path); + + int r; + while ((r = sqlite3_step(stmtQueryReferrers)) == SQLITE_ROW) { + const char * s = (const char *) sqlite3_column_text(stmtQueryReferrers, 0); + assert(s); + referrers.insert(s); + } + + if (r != SQLITE_DONE) + throwSQLiteError(db, format("error getting references of `%1%'") % path); +} + + +void LocalStore::queryReferrers(const Path & path, PathSet & referrers) +{ + assertStorePath(path); + retry_sqlite { + queryReferrers_(path, referrers); + } end_retry_sqlite; +} + + +Path LocalStore::queryDeriver(const Path & path) +{ + return queryPathInfo(path).deriver; +} + + +PathSet LocalStore::queryValidDerivers(const Path & path) +{ + assertStorePath(path); + + retry_sqlite { + SQLiteStmtUse use(stmtQueryValidDerivers); + stmtQueryValidDerivers.bind(path); + + PathSet derivers; + int r; + while ((r = sqlite3_step(stmtQueryValidDerivers)) == SQLITE_ROW) { + const char * s = (const char *) sqlite3_column_text(stmtQueryValidDerivers, 1); + assert(s); + derivers.insert(s); + } + + if (r != SQLITE_DONE) + throwSQLiteError(db, format("error getting valid derivers of `%1%'") % path); + + return derivers; + } end_retry_sqlite; +} + + +PathSet LocalStore::queryDerivationOutputs(const Path & path) +{ + retry_sqlite { + SQLiteStmtUse use(stmtQueryDerivationOutputs); + stmtQueryDerivationOutputs.bind(queryValidPathId(path)); + + PathSet outputs; + int r; + while ((r = sqlite3_step(stmtQueryDerivationOutputs)) == SQLITE_ROW) { + const char * s = (const char *) sqlite3_column_text(stmtQueryDerivationOutputs, 1); + assert(s); + outputs.insert(s); + } + + if (r != SQLITE_DONE) + throwSQLiteError(db, format("error getting outputs of `%1%'") % path); + + return outputs; + } end_retry_sqlite; +} + + +StringSet LocalStore::queryDerivationOutputNames(const Path & path) +{ + retry_sqlite { + SQLiteStmtUse use(stmtQueryDerivationOutputs); + stmtQueryDerivationOutputs.bind(queryValidPathId(path)); + + StringSet outputNames; + int r; + while ((r = sqlite3_step(stmtQueryDerivationOutputs)) == SQLITE_ROW) { + const char * s = (const char *) sqlite3_column_text(stmtQueryDerivationOutputs, 0); + assert(s); + outputNames.insert(s); + } + + if (r != SQLITE_DONE) + throwSQLiteError(db, format("error getting output names of `%1%'") % path); + + return outputNames; + } end_retry_sqlite; +} + + +Path LocalStore::queryPathFromHashPart(const string & hashPart) +{ + if (hashPart.size() != 32) throw Error("invalid hash part"); + + Path prefix = settings.nixStore + "/" + hashPart; + + retry_sqlite { + SQLiteStmtUse use(stmtQueryPathFromHashPart); + stmtQueryPathFromHashPart.bind(prefix); + + int res = sqlite3_step(stmtQueryPathFromHashPart); + if (res == SQLITE_DONE) return ""; + if (res != SQLITE_ROW) throwSQLiteError(db, "finding path in database"); + + const char * s = (const char *) sqlite3_column_text(stmtQueryPathFromHashPart, 0); + return s && prefix.compare(0, prefix.size(), s, prefix.size()) == 0 ? s : ""; + } end_retry_sqlite; +} + + +void LocalStore::setSubstituterEnv() +{ + if (didSetSubstituterEnv) return; + + /* Pass configuration options (including those overridden with + --option) to substituters. */ + setenv("_NIX_OPTIONS", settings.pack().c_str(), 1); + + didSetSubstituterEnv = true; +} + + +void LocalStore::startSubstituter(const Path & substituter, RunningSubstituter & run) +{ + if (run.disabled || run.pid != -1) return; + + debug(format("starting substituter program `%1%'") % substituter); + + Pipe toPipe, fromPipe, errorPipe; + + toPipe.create(); + fromPipe.create(); + errorPipe.create(); + + setSubstituterEnv(); + + run.pid = maybeVfork(); + + switch (run.pid) { + + case -1: + throw SysError("unable to fork"); + + case 0: /* child */ + try { + restoreAffinity(); + if (dup2(toPipe.readSide, STDIN_FILENO) == -1) + throw SysError("dupping stdin"); + if (dup2(fromPipe.writeSide, STDOUT_FILENO) == -1) + throw SysError("dupping stdout"); + if (dup2(errorPipe.writeSide, STDERR_FILENO) == -1) + throw SysError("dupping stderr"); + execl(substituter.c_str(), substituter.c_str(), "--query", NULL); + throw SysError(format("executing `%1%'") % substituter); + } catch (std::exception & e) { + std::cerr << "error: " << e.what() << std::endl; + } + _exit(1); + } + + /* Parent. */ + + run.program = baseNameOf(substituter); + run.to = toPipe.writeSide.borrow(); + run.from = run.fromBuf.fd = fromPipe.readSide.borrow(); + run.error = errorPipe.readSide.borrow(); + + toPipe.readSide.close(); + fromPipe.writeSide.close(); + errorPipe.writeSide.close(); + + /* The substituter may exit right away if it's disabled in any way + (e.g. copy-from-other-stores.pl will exit if no other stores + are configured). */ + try { + getLineFromSubstituter(run); + } catch (EndOfFile & e) { + run.to.close(); + run.from.close(); + run.error.close(); + run.disabled = true; + if (run.pid.wait(true) != 0) throw; + } +} + + +/* Read a line from the substituter's stdout, while also processing + its stderr. */ +string LocalStore::getLineFromSubstituter(RunningSubstituter & run) +{ + string res, err; + + /* We might have stdout data left over from the last time. */ + if (run.fromBuf.hasData()) goto haveData; + + while (1) { + checkInterrupt(); + + fd_set fds; + FD_ZERO(&fds); + FD_SET(run.from, &fds); + FD_SET(run.error, &fds); + + /* Wait for data to appear on the substituter's stdout or + stderr. */ + if (select(run.from > run.error ? run.from + 1 : run.error + 1, &fds, 0, 0, 0) == -1) { + if (errno == EINTR) continue; + throw SysError("waiting for input from the substituter"); + } + + /* Completely drain stderr before dealing with stdout. */ + if (FD_ISSET(run.error, &fds)) { + char buf[4096]; + ssize_t n = read(run.error, (unsigned char *) buf, sizeof(buf)); + if (n == -1) { + if (errno == EINTR) continue; + throw SysError("reading from substituter's stderr"); + } + if (n == 0) throw EndOfFile(format("substituter `%1%' died unexpectedly") % run.program); + err.append(buf, n); + string::size_type p; + while ((p = err.find('\n')) != string::npos) { + printMsg(lvlError, run.program + ": " + string(err, 0, p)); + err = string(err, p + 1); + } + } + + /* Read from stdout until we get a newline or the buffer is empty. */ + else if (run.fromBuf.hasData() || FD_ISSET(run.from, &fds)) { + haveData: + do { + unsigned char c; + run.fromBuf(&c, 1); + if (c == '\n') { + if (!err.empty()) printMsg(lvlError, run.program + ": " + err); + return res; + } + res += c; + } while (run.fromBuf.hasData()); + } + } +} + + +template<class T> T LocalStore::getIntLineFromSubstituter(RunningSubstituter & run) +{ + string s = getLineFromSubstituter(run); + T res; + if (!string2Int(s, res)) throw Error("integer expected from stream"); + return res; +} + + +PathSet LocalStore::querySubstitutablePaths(const PathSet & paths) +{ + PathSet res; + foreach (Paths::iterator, i, settings.substituters) { + if (res.size() == paths.size()) break; + RunningSubstituter & run(runningSubstituters[*i]); + startSubstituter(*i, run); + if (run.disabled) continue; + string s = "have "; + foreach (PathSet::const_iterator, j, paths) + if (res.find(*j) == res.end()) { s += *j; s += " "; } + writeLine(run.to, s); + while (true) { + /* FIXME: we only read stderr when an error occurs, so + substituters should only write (short) messages to + stderr when they fail. I.e. they shouldn't write debug + output. */ + Path path = getLineFromSubstituter(run); + if (path == "") break; + res.insert(path); + } + } + return res; +} + + +void LocalStore::querySubstitutablePathInfos(const Path & substituter, + PathSet & paths, SubstitutablePathInfos & infos) +{ + RunningSubstituter & run(runningSubstituters[substituter]); + startSubstituter(substituter, run); + if (run.disabled) return; + + string s = "info "; + foreach (PathSet::const_iterator, i, paths) + if (infos.find(*i) == infos.end()) { s += *i; s += " "; } + writeLine(run.to, s); + + while (true) { + Path path = getLineFromSubstituter(run); + if (path == "") break; + if (paths.find(path) == paths.end()) + throw Error(format("got unexpected path `%1%' from substituter") % path); + paths.erase(path); + SubstitutablePathInfo & info(infos[path]); + info.deriver = getLineFromSubstituter(run); + if (info.deriver != "") assertStorePath(info.deriver); + int nrRefs = getIntLineFromSubstituter<int>(run); + while (nrRefs--) { + Path p = getLineFromSubstituter(run); + assertStorePath(p); + info.references.insert(p); + } + info.downloadSize = getIntLineFromSubstituter<long long>(run); + info.narSize = getIntLineFromSubstituter<long long>(run); + } +} + + +void LocalStore::querySubstitutablePathInfos(const PathSet & paths, + SubstitutablePathInfos & infos) +{ + PathSet todo = paths; + foreach (Paths::iterator, i, settings.substituters) { + if (todo.empty()) break; + querySubstitutablePathInfos(*i, todo, infos); + } +} + + +Hash LocalStore::queryPathHash(const Path & path) +{ + return queryPathInfo(path).hash; +} + + +void LocalStore::registerValidPath(const ValidPathInfo & info) +{ + ValidPathInfos infos; + infos.push_back(info); + registerValidPaths(infos); +} + + +void LocalStore::registerValidPaths(const ValidPathInfos & infos) +{ + /* SQLite will fsync by default, but the new valid paths may not be fsync-ed. + * So some may want to fsync them before registering the validity, at the + * expense of some speed of the path registering operation. */ + if (settings.syncBeforeRegistering) sync(); + + retry_sqlite { + SQLiteTxn txn(db); + PathSet paths; + + foreach (ValidPathInfos::const_iterator, i, infos) { + assert(i->hash.type == htSHA256); + if (isValidPath_(i->path)) + updatePathInfo(*i); + else + addValidPath(*i); + paths.insert(i->path); + } + + foreach (ValidPathInfos::const_iterator, i, infos) { + unsigned long long referrer = queryValidPathId(i->path); + foreach (PathSet::iterator, j, i->references) + addReference(referrer, queryValidPathId(*j)); + } + + /* Do a topological sort of the paths. This will throw an + error if a cycle is detected and roll back the + transaction. Cycles can only occur when a derivation + has multiple outputs. */ + topoSortPaths(*this, paths); + + txn.commit(); + } end_retry_sqlite; +} + + +/* Invalidate a path. The caller is responsible for checking that + there are no referrers. */ +void LocalStore::invalidatePath(const Path & path) +{ + debug(format("invalidating path `%1%'") % path); + + drvHashes.erase(path); + + SQLiteStmtUse use(stmtInvalidatePath); + + stmtInvalidatePath.bind(path); + + if (sqlite3_step(stmtInvalidatePath) != SQLITE_DONE) + throwSQLiteError(db, format("invalidating path `%1%' in database") % path); + + /* Note that the foreign key constraints on the Refs table take + care of deleting the references entries for `path'. */ +} + + +Path LocalStore::addToStoreFromDump(const string & dump, const string & name, + bool recursive, HashType hashAlgo, bool repair) +{ + Hash h = hashString(hashAlgo, dump); + + Path dstPath = makeFixedOutputPath(recursive, hashAlgo, h, name); + + addTempRoot(dstPath); + + if (repair || !isValidPath(dstPath)) { + + /* The first check above is an optimisation to prevent + unnecessary lock acquisition. */ + + PathLocks outputLock(singleton<PathSet, Path>(dstPath)); + + if (repair || !isValidPath(dstPath)) { + + if (pathExists(dstPath)) deletePath(dstPath); + + if (recursive) { + StringSource source(dump); + restorePath(dstPath, source); + } else + writeFile(dstPath, dump); + + canonicalisePathMetaData(dstPath, -1); + + /* Register the SHA-256 hash of the NAR serialisation of + the path in the database. We may just have computed it + above (if called with recursive == true and hashAlgo == + sha256); otherwise, compute it here. */ + HashResult hash; + if (recursive) { + hash.first = hashAlgo == htSHA256 ? h : hashString(htSHA256, dump); + hash.second = dump.size(); + } else + hash = hashPath(htSHA256, dstPath); + + optimisePath(dstPath); // FIXME: combine with hashPath() + + ValidPathInfo info; + info.path = dstPath; + info.hash = hash.first; + info.narSize = hash.second; + registerValidPath(info); + } + + outputLock.setDeletion(true); + } + + return dstPath; +} + + +Path LocalStore::addToStore(const Path & _srcPath, + bool recursive, HashType hashAlgo, PathFilter & filter, bool repair) +{ + Path srcPath(absPath(_srcPath)); + debug(format("adding `%1%' to the store") % srcPath); + + /* Read the whole path into memory. This is not a very scalable + method for very large paths, but `copyPath' is mainly used for + small files. */ + StringSink sink; + if (recursive) + dumpPath(srcPath, sink, filter); + else + sink.s = readFile(srcPath); + + return addToStoreFromDump(sink.s, baseNameOf(srcPath), recursive, hashAlgo, repair); +} + + +Path LocalStore::addTextToStore(const string & name, const string & s, + const PathSet & references, bool repair) +{ + Path dstPath = computeStorePathForText(name, s, references); + + addTempRoot(dstPath); + + if (repair || !isValidPath(dstPath)) { + + PathLocks outputLock(singleton<PathSet, Path>(dstPath)); + + if (repair || !isValidPath(dstPath)) { + + if (pathExists(dstPath)) deletePath(dstPath); + + writeFile(dstPath, s); + + canonicalisePathMetaData(dstPath, -1); + + HashResult hash = hashPath(htSHA256, dstPath); + + optimisePath(dstPath); + + ValidPathInfo info; + info.path = dstPath; + info.hash = hash.first; + info.narSize = hash.second; + info.references = references; + registerValidPath(info); + } + + outputLock.setDeletion(true); + } + + return dstPath; +} + + +struct HashAndWriteSink : Sink +{ + Sink & writeSink; + HashSink hashSink; + HashAndWriteSink(Sink & writeSink) : writeSink(writeSink), hashSink(htSHA256) + { + } + virtual void operator () (const unsigned char * data, size_t len) + { + writeSink(data, len); + hashSink(data, len); + } + Hash currentHash() + { + return hashSink.currentHash().first; + } +}; + + +#define EXPORT_MAGIC 0x4558494e + + +static void checkSecrecy(const Path & path) +{ + struct stat st; + if (stat(path.c_str(), &st)) + throw SysError(format("getting status of `%1%'") % path); + if ((st.st_mode & (S_IRWXG | S_IRWXO)) != 0) + throw Error(format("file `%1%' should be secret (inaccessible to everybody else)!") % path); +} + + +void LocalStore::exportPath(const Path & path, bool sign, + Sink & sink) +{ + assertStorePath(path); + + addTempRoot(path); + if (!isValidPath(path)) + throw Error(format("path `%1%' is not valid") % path); + + HashAndWriteSink hashAndWriteSink(sink); + + dumpPath(path, hashAndWriteSink); + + /* Refuse to export paths that have changed. This prevents + filesystem corruption from spreading to other machines. + Don't complain if the stored hash is zero (unknown). */ + Hash hash = hashAndWriteSink.currentHash(); + Hash storedHash = queryPathHash(path); + if (hash != storedHash && storedHash != Hash(storedHash.type)) + throw Error(format("hash of path `%1%' has changed from `%2%' to `%3%'!") % path + % printHash(storedHash) % printHash(hash)); + + writeInt(EXPORT_MAGIC, hashAndWriteSink); + + writeString(path, hashAndWriteSink); + + PathSet references; + queryReferences(path, references); + writeStrings(references, hashAndWriteSink); + + Path deriver = queryDeriver(path); + writeString(deriver, hashAndWriteSink); + + if (sign) { + Hash hash = hashAndWriteSink.currentHash(); + + writeInt(1, hashAndWriteSink); + + Path tmpDir = createTempDir(); + AutoDelete delTmp(tmpDir); + Path hashFile = tmpDir + "/hash"; + writeFile(hashFile, printHash(hash)); + + Path secretKey = settings.nixConfDir + "/signing-key.sec"; + checkSecrecy(secretKey); + + Strings args; + args.push_back("rsautl"); + args.push_back("-sign"); + args.push_back("-inkey"); + args.push_back(secretKey); + args.push_back("-in"); + args.push_back(hashFile); + string signature = runProgram(OPENSSL_PATH, true, args); + + writeString(signature, hashAndWriteSink); + + } else + writeInt(0, hashAndWriteSink); +} + + +struct HashAndReadSource : Source +{ + Source & readSource; + HashSink hashSink; + bool hashing; + HashAndReadSource(Source & readSource) : readSource(readSource), hashSink(htSHA256) + { + hashing = true; + } + size_t read(unsigned char * data, size_t len) + { + size_t n = readSource.read(data, len); + if (hashing) hashSink(data, n); + return n; + } +}; + + +/* Create a temporary directory in the store that won't be + garbage-collected. */ +Path LocalStore::createTempDirInStore() +{ + Path tmpDir; + do { + /* There is a slight possibility that `tmpDir' gets deleted by + the GC between createTempDir() and addTempRoot(), so repeat + until `tmpDir' exists. */ + tmpDir = createTempDir(settings.nixStore); + addTempRoot(tmpDir); + } while (!pathExists(tmpDir)); + return tmpDir; +} + + +Path LocalStore::importPath(bool requireSignature, Source & source) +{ + HashAndReadSource hashAndReadSource(source); + + /* We don't yet know what store path this archive contains (the + store path follows the archive data proper), and besides, we + don't know yet whether the signature is valid. */ + Path tmpDir = createTempDirInStore(); + AutoDelete delTmp(tmpDir); + Path unpacked = tmpDir + "/unpacked"; + + restorePath(unpacked, hashAndReadSource); + + unsigned int magic = readInt(hashAndReadSource); + if (magic != EXPORT_MAGIC) + throw Error("Nix archive cannot be imported; wrong format"); + + Path dstPath = readStorePath(hashAndReadSource); + + printMsg(lvlInfo, format("importing path `%1%'") % dstPath); + + PathSet references = readStorePaths<PathSet>(hashAndReadSource); + + Path deriver = readString(hashAndReadSource); + if (deriver != "") assertStorePath(deriver); + + Hash hash = hashAndReadSource.hashSink.finish().first; + hashAndReadSource.hashing = false; + + bool haveSignature = readInt(hashAndReadSource) == 1; + + if (requireSignature && !haveSignature) + throw Error(format("imported archive of `%1%' lacks a signature") % dstPath); + + if (haveSignature) { + string signature = readString(hashAndReadSource); + + if (requireSignature) { + Path sigFile = tmpDir + "/sig"; + writeFile(sigFile, signature); + + Strings args; + args.push_back("rsautl"); + args.push_back("-verify"); + args.push_back("-inkey"); + args.push_back(settings.nixConfDir + "/signing-key.pub"); + args.push_back("-pubin"); + args.push_back("-in"); + args.push_back(sigFile); + string hash2 = runProgram(OPENSSL_PATH, true, args); + + /* Note: runProgram() throws an exception if the signature + is invalid. */ + + if (printHash(hash) != hash2) + throw Error( + "signed hash doesn't match actual contents of imported " + "archive; archive could be corrupt, or someone is trying " + "to import a Trojan horse"); + } + } + + /* Do the actual import. */ + + /* !!! way too much code duplication with addTextToStore() etc. */ + addTempRoot(dstPath); + + if (!isValidPath(dstPath)) { + + PathLocks outputLock; + + /* Lock the output path. But don't lock if we're being called + from a build hook (whose parent process already acquired a + lock on this path). */ + Strings locksHeld = tokenizeString<Strings>(getEnv("NIX_HELD_LOCKS")); + if (find(locksHeld.begin(), locksHeld.end(), dstPath) == locksHeld.end()) + outputLock.lockPaths(singleton<PathSet, Path>(dstPath)); + + if (!isValidPath(dstPath)) { + + if (pathExists(dstPath)) deletePath(dstPath); + + if (rename(unpacked.c_str(), dstPath.c_str()) == -1) + throw SysError(format("cannot move `%1%' to `%2%'") + % unpacked % dstPath); + + canonicalisePathMetaData(dstPath, -1); + + /* !!! if we were clever, we could prevent the hashPath() + here. */ + HashResult hash = hashPath(htSHA256, dstPath); + + optimisePath(dstPath); // FIXME: combine with hashPath() + + ValidPathInfo info; + info.path = dstPath; + info.hash = hash.first; + info.narSize = hash.second; + info.references = references; + info.deriver = deriver != "" && isValidPath(deriver) ? deriver : ""; + registerValidPath(info); + } + + outputLock.setDeletion(true); + } + + return dstPath; +} + + +Paths LocalStore::importPaths(bool requireSignature, Source & source) +{ + Paths res; + while (true) { + unsigned long long n = readLongLong(source); + if (n == 0) break; + if (n != 1) throw Error("input doesn't look like something created by `nix-store --export'"); + res.push_back(importPath(requireSignature, source)); + } + return res; +} + + +void LocalStore::invalidatePathChecked(const Path & path) +{ + assertStorePath(path); + + retry_sqlite { + SQLiteTxn txn(db); + + if (isValidPath_(path)) { + PathSet referrers; queryReferrers_(path, referrers); + referrers.erase(path); /* ignore self-references */ + if (!referrers.empty()) + throw PathInUse(format("cannot delete path `%1%' because it is in use by %2%") + % path % showPaths(referrers)); + invalidatePath(path); + } + + txn.commit(); + } end_retry_sqlite; +} + + +bool LocalStore::verifyStore(bool checkContents, bool repair) +{ + printMsg(lvlError, format("reading the Nix store...")); + + bool errors = false; + + /* Acquire the global GC lock to prevent a garbage collection. */ + AutoCloseFD fdGCLock = openGCLock(ltWrite); + + Paths entries = readDirectory(settings.nixStore); + PathSet store(entries.begin(), entries.end()); + + /* Check whether all valid paths actually exist. */ + printMsg(lvlInfo, "checking path existence..."); + + PathSet validPaths2 = queryAllValidPaths(), validPaths, done; + + foreach (PathSet::iterator, i, validPaths2) + verifyPath(*i, store, done, validPaths, repair, errors); + + /* Release the GC lock so that checking content hashes (which can + take ages) doesn't block the GC or builds. */ + fdGCLock.close(); + + /* Optionally, check the content hashes (slow). */ + if (checkContents) { + printMsg(lvlInfo, "checking hashes..."); + + Hash nullHash(htSHA256); + + foreach (PathSet::iterator, i, validPaths) { + try { + ValidPathInfo info = queryPathInfo(*i); + + /* Check the content hash (optionally - slow). */ + printMsg(lvlTalkative, format("checking contents of `%1%'") % *i); + HashResult current = hashPath(info.hash.type, *i); + + if (info.hash != nullHash && info.hash != current.first) { + printMsg(lvlError, format("path `%1%' was modified! " + "expected hash `%2%', got `%3%'") + % *i % printHash(info.hash) % printHash(current.first)); + if (repair) repairPath(*i); else errors = true; + } else { + + bool update = false; + + /* Fill in missing hashes. */ + if (info.hash == nullHash) { + printMsg(lvlError, format("fixing missing hash on `%1%'") % *i); + info.hash = current.first; + update = true; + } + + /* Fill in missing narSize fields (from old stores). */ + if (info.narSize == 0) { + printMsg(lvlError, format("updating size field on `%1%' to %2%") % *i % current.second); + info.narSize = current.second; + update = true; + } + + if (update) updatePathInfo(info); + + } + + } catch (Error & e) { + /* It's possible that the path got GC'ed, so ignore + errors on invalid paths. */ + if (isValidPath(*i)) + printMsg(lvlError, format("error: %1%") % e.msg()); + else + printMsg(lvlError, format("warning: %1%") % e.msg()); + errors = true; + } + } + } + + return errors; +} + + +void LocalStore::verifyPath(const Path & path, const PathSet & store, + PathSet & done, PathSet & validPaths, bool repair, bool & errors) +{ + checkInterrupt(); + + if (done.find(path) != done.end()) return; + done.insert(path); + + if (!isStorePath(path)) { + printMsg(lvlError, format("path `%1%' is not in the Nix store") % path); + invalidatePath(path); + return; + } + + if (store.find(baseNameOf(path)) == store.end()) { + /* Check any referrers first. If we can invalidate them + first, then we can invalidate this path as well. */ + bool canInvalidate = true; + PathSet referrers; queryReferrers(path, referrers); + foreach (PathSet::iterator, i, referrers) + if (*i != path) { + verifyPath(*i, store, done, validPaths, repair, errors); + if (validPaths.find(*i) != validPaths.end()) + canInvalidate = false; + } + + if (canInvalidate) { + printMsg(lvlError, format("path `%1%' disappeared, removing from database...") % path); + invalidatePath(path); + } else { + printMsg(lvlError, format("path `%1%' disappeared, but it still has valid referrers!") % path); + if (repair) + try { + repairPath(path); + } catch (Error & e) { + printMsg(lvlError, format("warning: %1%") % e.msg()); + errors = true; + } + else errors = true; + } + + return; + } + + validPaths.insert(path); +} + + +bool LocalStore::pathContentsGood(const Path & path) +{ + std::map<Path, bool>::iterator i = pathContentsGoodCache.find(path); + if (i != pathContentsGoodCache.end()) return i->second; + printMsg(lvlInfo, format("checking path `%1%'...") % path); + ValidPathInfo info = queryPathInfo(path); + bool res; + if (!pathExists(path)) + res = false; + else { + HashResult current = hashPath(info.hash.type, path); + Hash nullHash(htSHA256); + res = info.hash == nullHash || info.hash == current.first; + } + pathContentsGoodCache[path] = res; + if (!res) printMsg(lvlError, format("path `%1%' is corrupted or missing!") % path); + return res; +} + + +void LocalStore::markContentsGood(const Path & path) +{ + pathContentsGoodCache[path] = true; +} + + +/* Functions for upgrading from the pre-SQLite database. */ + +PathSet LocalStore::queryValidPathsOld() +{ + PathSet paths; + Strings entries = readDirectory(settings.nixDBPath + "/info"); + foreach (Strings::iterator, i, entries) + if (i->at(0) != '.') paths.insert(settings.nixStore + "/" + *i); + return paths; +} + + +ValidPathInfo LocalStore::queryPathInfoOld(const Path & path) +{ + ValidPathInfo res; + res.path = path; + + /* Read the info file. */ + string baseName = baseNameOf(path); + Path infoFile = (format("%1%/info/%2%") % settings.nixDBPath % baseName).str(); + if (!pathExists(infoFile)) + throw Error(format("path `%1%' is not valid") % path); + string info = readFile(infoFile); + + /* Parse it. */ + Strings lines = tokenizeString<Strings>(info, "\n"); + + foreach (Strings::iterator, i, lines) { + string::size_type p = i->find(':'); + if (p == string::npos) + throw Error(format("corrupt line in `%1%': %2%") % infoFile % *i); + string name(*i, 0, p); + string value(*i, p + 2); + if (name == "References") { + Strings refs = tokenizeString<Strings>(value, " "); + res.references = PathSet(refs.begin(), refs.end()); + } else if (name == "Deriver") { + res.deriver = value; + } else if (name == "Hash") { + res.hash = parseHashField(path, value); + } else if (name == "Registered-At") { + int n = 0; + string2Int(value, n); + res.registrationTime = n; + } + } + + return res; +} + + +/* Upgrade from schema 5 (Nix 0.12) to schema 6 (Nix >= 0.15). */ +void LocalStore::upgradeStore6() +{ + printMsg(lvlError, "upgrading Nix store to new schema (this may take a while)..."); + + openDB(true); + + PathSet validPaths = queryValidPathsOld(); + + SQLiteTxn txn(db); + + foreach (PathSet::iterator, i, validPaths) { + addValidPath(queryPathInfoOld(*i), false); + std::cerr << "."; + } + + std::cerr << "|"; + + foreach (PathSet::iterator, i, validPaths) { + ValidPathInfo info = queryPathInfoOld(*i); + unsigned long long referrer = queryValidPathId(*i); + foreach (PathSet::iterator, j, info.references) + addReference(referrer, queryValidPathId(*j)); + std::cerr << "."; + } + + std::cerr << "\n"; + + txn.commit(); +} + + +#if defined(FS_IOC_SETFLAGS) && defined(FS_IOC_GETFLAGS) && defined(FS_IMMUTABLE_FL) + +static void makeMutable(const Path & path) +{ + checkInterrupt(); + + struct stat st = lstat(path); + + if (!S_ISDIR(st.st_mode) && !S_ISREG(st.st_mode)) return; + + if (S_ISDIR(st.st_mode)) { + Strings names = readDirectory(path); + foreach (Strings::iterator, i, names) + makeMutable(path + "/" + *i); + } + + /* The O_NOFOLLOW is important to prevent us from changing the + mutable bit on the target of a symlink (which would be a + security hole). */ + AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_NOFOLLOW); + if (fd == -1) { + if (errno == ELOOP) return; // it's a symlink + throw SysError(format("opening file `%1%'") % path); + } + + unsigned int flags = 0, old; + + /* Silently ignore errors getting/setting the immutable flag so + that we work correctly on filesystems that don't support it. */ + if (ioctl(fd, FS_IOC_GETFLAGS, &flags)) return; + old = flags; + flags &= ~FS_IMMUTABLE_FL; + if (old == flags) return; + if (ioctl(fd, FS_IOC_SETFLAGS, &flags)) return; +} + +/* Upgrade from schema 6 (Nix 0.15) to schema 7 (Nix >= 1.3). */ +void LocalStore::upgradeStore7() +{ + if (getuid() != 0) return; + printMsg(lvlError, "removing immutable bits from the Nix store (this may take a while)..."); + makeMutable(settings.nixStore); +} + +#else + +void LocalStore::upgradeStore7() +{ +} + +#endif + + +void LocalStore::vacuumDB() +{ + if (sqlite3_exec(db, "vacuum;", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(db, "vacuuming SQLite database"); +} + + +} diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh new file mode 100644 index 000000000000..1ace7cec996b --- /dev/null +++ b/src/libstore/local-store.hh @@ -0,0 +1,333 @@ +#pragma once + +#include <string> + +#include "store-api.hh" +#include "util.hh" +#include "pathlocks.hh" + + +class sqlite3; +class sqlite3_stmt; + + +namespace nix { + + +/* Nix store and database schema version. Version 1 (or 0) was Nix <= + 0.7. Version 2 was Nix 0.8 and 0.9. Version 3 is Nix 0.10. + Version 4 is Nix 0.11. Version 5 is Nix 0.12-0.16. Version 6 is + Nix 1.0. Version 7 is Nix 1.3. */ +const int nixSchemaVersion = 7; + + +extern string drvsLogDir; + + +struct Derivation; + + +struct OptimiseStats +{ + unsigned long totalFiles; + unsigned long sameContents; + unsigned long filesLinked; + unsigned long long bytesFreed; + unsigned long long blocksFreed; + OptimiseStats() + { + totalFiles = sameContents = filesLinked = 0; + bytesFreed = blocksFreed = 0; + } +}; + + +struct RunningSubstituter +{ + Path program; + Pid pid; + AutoCloseFD to, from, error; + FdSource fromBuf; + bool disabled; + RunningSubstituter() : disabled(false) { }; +}; + + +/* Wrapper object to close the SQLite database automatically. */ +struct SQLite +{ + sqlite3 * db; + SQLite() { db = 0; } + ~SQLite(); + operator sqlite3 * () { return db; } +}; + + +/* Wrapper object to create and destroy SQLite prepared statements. */ +struct SQLiteStmt +{ + sqlite3 * db; + sqlite3_stmt * stmt; + unsigned int curArg; + SQLiteStmt() { stmt = 0; } + void create(sqlite3 * db, const string & s); + void reset(); + ~SQLiteStmt(); + operator sqlite3_stmt * () { return stmt; } + void bind(const string & value); + void bind(int value); + void bind64(long long value); + void bind(); +}; + + +class LocalStore : public StoreAPI +{ +private: + typedef std::map<Path, RunningSubstituter> RunningSubstituters; + RunningSubstituters runningSubstituters; + + Path linksDir; + +public: + + /* Initialise the local store, upgrading the schema if + necessary. */ + LocalStore(bool reserveSpace = true); + + ~LocalStore(); + + /* Implementations of abstract store API methods. */ + + bool isValidPath(const Path & path); + + PathSet queryValidPaths(const PathSet & paths); + + PathSet queryAllValidPaths(); + + ValidPathInfo queryPathInfo(const Path & path); + + Hash queryPathHash(const Path & path); + + void queryReferences(const Path & path, PathSet & references); + + void queryReferrers(const Path & path, PathSet & referrers); + + Path queryDeriver(const Path & path); + + PathSet queryValidDerivers(const Path & path); + + PathSet queryDerivationOutputs(const Path & path); + + StringSet queryDerivationOutputNames(const Path & path); + + Path queryPathFromHashPart(const string & hashPart); + + PathSet querySubstitutablePaths(const PathSet & paths); + + void querySubstitutablePathInfos(const Path & substituter, + PathSet & paths, SubstitutablePathInfos & infos); + + void querySubstitutablePathInfos(const PathSet & paths, + SubstitutablePathInfos & infos); + + Path addToStore(const Path & srcPath, + bool recursive = true, HashType hashAlgo = htSHA256, + PathFilter & filter = defaultPathFilter, bool repair = false); + + /* Like addToStore(), but the contents of the path are contained + in `dump', which is either a NAR serialisation (if recursive == + true) or simply the contents of a regular file (if recursive == + false). */ + Path addToStoreFromDump(const string & dump, const string & name, + bool recursive = true, HashType hashAlgo = htSHA256, bool repair = false); + + Path addTextToStore(const string & name, const string & s, + const PathSet & references, bool repair = false); + + void exportPath(const Path & path, bool sign, + Sink & sink); + + Paths importPaths(bool requireSignature, Source & source); + + void buildPaths(const PathSet & paths, bool repair = false); + + void ensurePath(const Path & path); + + void addTempRoot(const Path & path); + + void addIndirectRoot(const Path & path); + + void syncWithGC(); + + Roots findRoots(); + + void collectGarbage(const GCOptions & options, GCResults & results); + + /* Optimise the disk space usage of the Nix store by hard-linking + files with the same contents. */ + void optimiseStore(OptimiseStats & stats); + + /* Optimise a single store path. */ + void optimisePath(const Path & path); + + /* Check the integrity of the Nix store. Returns true if errors + remain. */ + bool verifyStore(bool checkContents, bool repair); + + /* Register the validity of a path, i.e., that `path' exists, that + the paths referenced by it exists, and in the case of an output + path of a derivation, that it has been produced by a successful + execution of the derivation (or something equivalent). Also + register the hash of the file system contents of the path. The + hash must be a SHA-256 hash. */ + void registerValidPath(const ValidPathInfo & info); + + void registerValidPaths(const ValidPathInfos & infos); + + /* Register that the build of a derivation with output `path' has + failed. */ + void registerFailedPath(const Path & path); + + /* Query whether `path' previously failed to build. */ + bool hasPathFailed(const Path & path); + + PathSet queryFailedPaths(); + + void clearFailedPaths(const PathSet & paths); + + void vacuumDB(); + + /* Repair the contents of the given path by redownloading it using + a substituter (if available). */ + void repairPath(const Path & path); + + /* Check whether the given valid path exists and has the right + contents. */ + bool pathContentsGood(const Path & path); + + void markContentsGood(const Path & path); + + void setSubstituterEnv(); + +private: + + Path schemaPath; + + /* Lock file used for upgrading. */ + AutoCloseFD globalLock; + + /* The SQLite database object. */ + SQLite db; + + /* Some precompiled SQLite statements. */ + SQLiteStmt stmtRegisterValidPath; + SQLiteStmt stmtUpdatePathInfo; + SQLiteStmt stmtAddReference; + SQLiteStmt stmtQueryPathInfo; + SQLiteStmt stmtQueryReferences; + SQLiteStmt stmtQueryReferrers; + SQLiteStmt stmtInvalidatePath; + SQLiteStmt stmtRegisterFailedPath; + SQLiteStmt stmtHasPathFailed; + SQLiteStmt stmtQueryFailedPaths; + SQLiteStmt stmtClearFailedPath; + SQLiteStmt stmtAddDerivationOutput; + SQLiteStmt stmtQueryValidDerivers; + SQLiteStmt stmtQueryDerivationOutputs; + SQLiteStmt stmtQueryPathFromHashPart; + + /* Cache for pathContentsGood(). */ + std::map<Path, bool> pathContentsGoodCache; + + bool didSetSubstituterEnv; + + int getSchema(); + + void openDB(bool create); + + void makeStoreWritable(); + + unsigned long long queryValidPathId(const Path & path); + + unsigned long long addValidPath(const ValidPathInfo & info, bool checkOutputs = true); + + void addReference(unsigned long long referrer, unsigned long long reference); + + void appendReferrer(const Path & from, const Path & to, bool lock); + + void rewriteReferrers(const Path & path, bool purge, PathSet referrers); + + void invalidatePath(const Path & path); + + /* Delete a path from the Nix store. */ + void invalidatePathChecked(const Path & path); + + void verifyPath(const Path & path, const PathSet & store, + PathSet & done, PathSet & validPaths, bool repair, bool & errors); + + void updatePathInfo(const ValidPathInfo & info); + + void upgradeStore6(); + void upgradeStore7(); + PathSet queryValidPathsOld(); + ValidPathInfo queryPathInfoOld(const Path & path); + + struct GCState; + + void deleteGarbage(GCState & state, const Path & path); + + void tryToDelete(GCState & state, const Path & path); + + bool canReachRoot(GCState & state, PathSet & visited, const Path & path); + + void deletePathRecursive(GCState & state, const Path & path); + + bool isActiveTempFile(const GCState & state, + const Path & path, const string & suffix); + + int openGCLock(LockType lockType); + + void removeUnusedLinks(const GCState & state); + + void startSubstituter(const Path & substituter, + RunningSubstituter & runningSubstituter); + + string getLineFromSubstituter(RunningSubstituter & run); + + template<class T> T getIntLineFromSubstituter(RunningSubstituter & run); + + Path createTempDirInStore(); + + Path importPath(bool requireSignature, Source & source); + + void checkDerivationOutputs(const Path & drvPath, const Derivation & drv); + + void optimisePath_(OptimiseStats & stats, const Path & path); + + // Internal versions that are not wrapped in retry_sqlite. + bool isValidPath_(const Path & path); + void queryReferrers_(const Path & path, PathSet & referrers); +}; + + +typedef std::pair<dev_t, ino_t> Inode; +typedef set<Inode> InodesSeen; + + +/* "Fix", or canonicalise, the meta-data of the files in a store path + after it has been built. In particular: + - the last modification date on each file is set to 1 (i.e., + 00:00:01 1/1/1970 UTC) + - the permissions are set of 444 or 555 (i.e., read-only with or + without execute permission; setuid bits etc. are cleared) + - the owner and group are set to the Nix user and group, if we're + running as root. */ +void canonicalisePathMetaData(const Path & path, uid_t fromUid, InodesSeen & inodesSeen); +void canonicalisePathMetaData(const Path & path, uid_t fromUid); + +void canonicaliseTimestampAndPermissions(const Path & path); + +MakeError(PathInUse, Error); + +} diff --git a/src/libstore/local.mk b/src/libstore/local.mk new file mode 100644 index 000000000000..2dddce740dcc --- /dev/null +++ b/src/libstore/local.mk @@ -0,0 +1,28 @@ +libraries += libstore + +libstore_NAME = libnixstore + +libstore_DIR := $(d) + +libstore_SOURCES := $(wildcard $(d)/*.cc) + +libstore_LIBS = libutil libformat + +libstore_LDFLAGS = -lsqlite3 -lbz2 + +libstore_CXXFLAGS = \ + -DNIX_STORE_DIR=\"$(storedir)\" \ + -DNIX_DATA_DIR=\"$(datadir)\" \ + -DNIX_STATE_DIR=\"$(localstatedir)/nix\" \ + -DNIX_LOG_DIR=\"$(localstatedir)/log/nix\" \ + -DNIX_CONF_DIR=\"$(sysconfdir)/nix\" \ + -DNIX_LIBEXEC_DIR=\"$(libexecdir)\" \ + -DNIX_BIN_DIR=\"$(bindir)\" \ + -DPACKAGE_VERSION=\"$(PACKAGE_VERSION)\" + +$(d)/local-store.cc: $(d)/schema.sql.hh + +%.sql.hh: %.sql + $(trace-gen) sed -e 's/"/\\"/g' -e 's/\(.*\)/"\1\\n"/' < $< > $@ || (rm $@ && exit 1) + +clean-files += $(d)/schema.sql.hh diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc new file mode 100644 index 000000000000..1bf3f9378245 --- /dev/null +++ b/src/libstore/misc.cc @@ -0,0 +1,220 @@ +#include "misc.hh" +#include "store-api.hh" +#include "local-store.hh" +#include "globals.hh" + + +namespace nix { + + +Derivation derivationFromPath(StoreAPI & store, const Path & drvPath) +{ + assertStorePath(drvPath); + store.ensurePath(drvPath); + return parseDerivation(readFile(drvPath)); +} + + +void computeFSClosure(StoreAPI & store, const Path & path, + PathSet & paths, bool flipDirection, bool includeOutputs, bool includeDerivers) +{ + if (paths.find(path) != paths.end()) return; + paths.insert(path); + + PathSet edges; + + if (flipDirection) { + store.queryReferrers(path, edges); + + if (includeOutputs) { + PathSet derivers = store.queryValidDerivers(path); + foreach (PathSet::iterator, i, derivers) + edges.insert(*i); + } + + if (includeDerivers && isDerivation(path)) { + PathSet outputs = store.queryDerivationOutputs(path); + foreach (PathSet::iterator, i, outputs) + if (store.isValidPath(*i) && store.queryDeriver(*i) == path) + edges.insert(*i); + } + + } else { + store.queryReferences(path, edges); + + if (includeOutputs && isDerivation(path)) { + PathSet outputs = store.queryDerivationOutputs(path); + foreach (PathSet::iterator, i, outputs) + if (store.isValidPath(*i)) edges.insert(*i); + } + + if (includeDerivers) { + Path deriver = store.queryDeriver(path); + if (store.isValidPath(deriver)) edges.insert(deriver); + } + } + + foreach (PathSet::iterator, i, edges) + computeFSClosure(store, *i, paths, flipDirection, includeOutputs, includeDerivers); +} + + +Path findOutput(const Derivation & drv, string id) +{ + foreach (DerivationOutputs::const_iterator, i, drv.outputs) + if (i->first == id) return i->second.path; + throw Error(format("derivation has no output `%1%'") % id); +} + + +void queryMissing(StoreAPI & store, const PathSet & targets, + PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown, + unsigned long long & downloadSize, unsigned long long & narSize) +{ + downloadSize = narSize = 0; + + PathSet todo(targets.begin(), targets.end()), done; + + /* Getting substitute info has high latency when using the binary + cache substituter. Thus it's essential to do substitute + queries in parallel as much as possible. To accomplish this + we do the following: + + - For all paths still to be processed (‘todo’), we add all + paths for which we need info to the set ‘query’. For an + unbuilt derivation this is the output paths; otherwise, it's + the path itself. + + - We get info about all paths in ‘query’ in parallel. + + - We process the results and add new items to ‘todo’ if + necessary. E.g. if a path is substitutable, then we need to + get info on its references. + + - Repeat until ‘todo’ is empty. + */ + + while (!todo.empty()) { + + PathSet query, todoDrv, todoNonDrv; + + foreach (PathSet::iterator, i, todo) { + if (done.find(*i) != done.end()) continue; + done.insert(*i); + + DrvPathWithOutputs i2 = parseDrvPathWithOutputs(*i); + + if (isDerivation(i2.first)) { + if (!store.isValidPath(i2.first)) { + // FIXME: we could try to substitute p. + unknown.insert(*i); + continue; + } + Derivation drv = derivationFromPath(store, i2.first); + + PathSet invalid; + foreach (DerivationOutputs::iterator, j, drv.outputs) + if (wantOutput(j->first, i2.second) + && !store.isValidPath(j->second.path)) + invalid.insert(j->second.path); + if (invalid.empty()) continue; + + todoDrv.insert(*i); + if (settings.useSubstitutes && !willBuildLocally(drv)) + query.insert(invalid.begin(), invalid.end()); + } + + else { + if (store.isValidPath(*i)) continue; + query.insert(*i); + todoNonDrv.insert(*i); + } + } + + todo.clear(); + + SubstitutablePathInfos infos; + store.querySubstitutablePathInfos(query, infos); + + foreach (PathSet::iterator, i, todoDrv) { + DrvPathWithOutputs i2 = parseDrvPathWithOutputs(*i); + + // FIXME: cache this + Derivation drv = derivationFromPath(store, i2.first); + + PathSet outputs; + bool mustBuild = false; + if (settings.useSubstitutes && !willBuildLocally(drv)) { + foreach (DerivationOutputs::iterator, j, drv.outputs) { + if (!wantOutput(j->first, i2.second)) continue; + if (!store.isValidPath(j->second.path)) { + if (infos.find(j->second.path) == infos.end()) + mustBuild = true; + else + outputs.insert(j->second.path); + } + } + } else + mustBuild = true; + + if (mustBuild) { + willBuild.insert(i2.first); + todo.insert(drv.inputSrcs.begin(), drv.inputSrcs.end()); + foreach (DerivationInputs::iterator, j, drv.inputDrvs) + todo.insert(makeDrvPathWithOutputs(j->first, j->second)); + } else + todoNonDrv.insert(outputs.begin(), outputs.end()); + } + + foreach (PathSet::iterator, i, todoNonDrv) { + done.insert(*i); + SubstitutablePathInfos::iterator info = infos.find(*i); + if (info != infos.end()) { + willSubstitute.insert(*i); + downloadSize += info->second.downloadSize; + narSize += info->second.narSize; + todo.insert(info->second.references.begin(), info->second.references.end()); + } else + unknown.insert(*i); + } + } +} + + +static void dfsVisit(StoreAPI & store, const PathSet & paths, + const Path & path, PathSet & visited, Paths & sorted, + PathSet & parents) +{ + if (parents.find(path) != parents.end()) + throw BuildError(format("cycle detected in the references of `%1%'") % path); + + if (visited.find(path) != visited.end()) return; + visited.insert(path); + parents.insert(path); + + PathSet references; + if (store.isValidPath(path)) + store.queryReferences(path, references); + + foreach (PathSet::iterator, i, references) + /* Don't traverse into paths that don't exist. That can + happen due to substitutes for non-existent paths. */ + if (*i != path && paths.find(*i) != paths.end()) + dfsVisit(store, paths, *i, visited, sorted, parents); + + sorted.push_front(path); + parents.erase(path); +} + + +Paths topoSortPaths(StoreAPI & store, const PathSet & paths) +{ + Paths sorted; + PathSet visited, parents; + foreach (PathSet::const_iterator, i, paths) + dfsVisit(store, paths, *i, visited, sorted, parents); + return sorted; +} + + +} diff --git a/src/libstore/misc.hh b/src/libstore/misc.hh new file mode 100644 index 000000000000..144cb7f457cd --- /dev/null +++ b/src/libstore/misc.hh @@ -0,0 +1,38 @@ +#pragma once + +#include "derivations.hh" + + +namespace nix { + + +/* Read a derivation, after ensuring its existence through + ensurePath(). */ +Derivation derivationFromPath(StoreAPI & store, const Path & drvPath); + +/* Place in `paths' the set of all store paths in the file system + closure of `storePath'; that is, all paths than can be directly or + indirectly reached from it. `paths' is not cleared. If + `flipDirection' is true, the set of paths that can reach + `storePath' is returned; that is, the closures under the + `referrers' relation instead of the `references' relation is + returned. */ +void computeFSClosure(StoreAPI & store, const Path & path, + PathSet & paths, bool flipDirection = false, + bool includeOutputs = false, bool includeDerivers = false); + +/* Return the path corresponding to the output identifier `id' in the + given derivation. */ +Path findOutput(const Derivation & drv, string id); + +/* Given a set of paths that are to be built, return the set of + derivations that will be built, and the set of output paths that + will be substituted. */ +void queryMissing(StoreAPI & store, const PathSet & targets, + PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown, + unsigned long long & downloadSize, unsigned long long & narSize); + +bool willBuildLocally(const Derivation & drv); + + +} diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc new file mode 100644 index 000000000000..d833f3aa05b5 --- /dev/null +++ b/src/libstore/optimise-store.cc @@ -0,0 +1,180 @@ +#include "config.h" + +#include "util.hh" +#include "local-store.hh" +#include "globals.hh" + +#include <sys/types.h> +#include <sys/stat.h> +#include <unistd.h> +#include <errno.h> +#include <stdio.h> + + +namespace nix { + + +static void makeWritable(const Path & path) +{ + struct stat st; + if (lstat(path.c_str(), &st)) + throw SysError(format("getting attributes of path `%1%'") % path); + if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1) + throw SysError(format("changing writability of `%1%'") % path); +} + + +struct MakeReadOnly +{ + Path path; + MakeReadOnly(const Path & path) : path(path) { } + ~MakeReadOnly() + { + try { + /* This will make the path read-only. */ + if (path != "") canonicaliseTimestampAndPermissions(path); + } catch (...) { + ignoreException(); + } + } +}; + + +void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path) +{ + checkInterrupt(); + + struct stat st; + if (lstat(path.c_str(), &st)) + throw SysError(format("getting attributes of path `%1%'") % path); + + if (S_ISDIR(st.st_mode)) { + Strings names = readDirectory(path); + foreach (Strings::iterator, i, names) + optimisePath_(stats, path + "/" + *i); + return; + } + + /* We can hard link regular files and maybe symlinks. */ + if (!S_ISREG(st.st_mode) +#if CAN_LINK_SYMLINK + && !S_ISLNK(st.st_mode) +#endif + ) return; + + /* Sometimes SNAFUs can cause files in the Nix store to be + modified, in particular when running programs as root under + NixOS (example: $fontconfig/var/cache being modified). Skip + those files. FIXME: check the modification time. */ + if (S_ISREG(st.st_mode) && (st.st_mode & S_IWUSR)) { + printMsg(lvlError, format("skipping suspicious writable file `%1%'") % path); + return; + } + + /* Hash the file. Note that hashPath() returns the hash over the + NAR serialisation, which includes the execute bit on the file. + Thus, executable and non-executable files with the same + contents *won't* be linked (which is good because otherwise the + permissions would be screwed up). + + Also note that if `path' is a symlink, then we're hashing the + contents of the symlink (i.e. the result of readlink()), not + the contents of the target (which may not even exist). */ + Hash hash = hashPath(htSHA256, path).first; + stats.totalFiles++; + printMsg(lvlDebug, format("`%1%' has hash `%2%'") % path % printHash(hash)); + + /* Check if this is a known hash. */ + Path linkPath = linksDir + "/" + printHash32(hash); + + if (!pathExists(linkPath)) { + /* Nope, create a hard link in the links directory. */ + if (link(path.c_str(), linkPath.c_str()) == 0) return; + if (errno != EEXIST) + throw SysError(format("cannot link `%1%' to `%2%'") % linkPath % path); + /* Fall through if another process created ‘linkPath’ before + we did. */ + } + + /* Yes! We've seen a file with the same contents. Replace the + current file with a hard link to that file. */ + struct stat stLink; + if (lstat(linkPath.c_str(), &stLink)) + throw SysError(format("getting attributes of path `%1%'") % linkPath); + + stats.sameContents++; + if (st.st_ino == stLink.st_ino) { + printMsg(lvlDebug, format("`%1%' is already linked to `%2%'") % path % linkPath); + return; + } + + printMsg(lvlTalkative, format("linking `%1%' to `%2%'") % path % linkPath); + + /* Make the containing directory writable, but only if it's not + the store itself (we don't want or need to mess with its + permissions). */ + bool mustToggle = !isStorePath(path); + if (mustToggle) makeWritable(dirOf(path)); + + /* When we're done, make the directory read-only again and reset + its timestamp back to 0. */ + MakeReadOnly makeReadOnly(mustToggle ? dirOf(path) : ""); + + Path tempLink = (format("%1%/.tmp-link-%2%-%3%") + % settings.nixStore % getpid() % rand()).str(); + + if (link(linkPath.c_str(), tempLink.c_str()) == -1) { + if (errno == EMLINK) { + /* Too many links to the same file (>= 32000 on most file + systems). This is likely to happen with empty files. + Just shrug and ignore. */ + if (st.st_size) + printMsg(lvlInfo, format("`%1%' has maximum number of links") % linkPath); + return; + } + throw SysError(format("cannot link `%1%' to `%2%'") % tempLink % linkPath); + } + + /* Atomically replace the old file with the new hard link. */ + if (rename(tempLink.c_str(), path.c_str()) == -1) { + if (unlink(tempLink.c_str()) == -1) + printMsg(lvlError, format("unable to unlink `%1%'") % tempLink); + if (errno == EMLINK) { + /* Some filesystems generate too many links on the rename, + rather than on the original link. (Probably it + temporarily increases the st_nlink field before + decreasing it again.) */ + if (st.st_size) + printMsg(lvlInfo, format("`%1%' has maximum number of links") % linkPath); + return; + } + throw SysError(format("cannot rename `%1%' to `%2%'") % tempLink % path); + } + + stats.filesLinked++; + stats.bytesFreed += st.st_size; + stats.blocksFreed += st.st_blocks; +} + + +void LocalStore::optimiseStore(OptimiseStats & stats) +{ + PathSet paths = queryAllValidPaths(); + + foreach (PathSet::iterator, i, paths) { + addTempRoot(*i); + if (!isValidPath(*i)) continue; /* path was GC'ed, probably */ + startNest(nest, lvlChatty, format("hashing files in `%1%'") % *i); + optimisePath_(stats, *i); + } +} + + +void LocalStore::optimisePath(const Path & path) +{ + OptimiseStats stats; + if (settings.autoOptimiseStore) optimisePath_(stats, path); +} + + +} diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc new file mode 100644 index 000000000000..b858ed238de0 --- /dev/null +++ b/src/libstore/pathlocks.cc @@ -0,0 +1,199 @@ +#include "pathlocks.hh" +#include "util.hh" + +#include <cerrno> +#include <cstdlib> + +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> + + +namespace nix { + + +int openLockFile(const Path & path, bool create) +{ + AutoCloseFD fd; + + fd = open(path.c_str(), O_RDWR | (create ? O_CREAT : 0), 0600); + if (fd == -1 && (create || errno != ENOENT)) + throw SysError(format("opening lock file `%1%'") % path); + + closeOnExec(fd); + + return fd.borrow(); +} + + +void deleteLockFile(const Path & path, int fd) +{ + /* Get rid of the lock file. Have to be careful not to introduce + races. Write a (meaningless) token to the file to indicate to + other processes waiting on this lock that the lock is stale + (deleted). */ + unlink(path.c_str()); + writeFull(fd, (const unsigned char *) "d", 1); + /* Note that the result of unlink() is ignored; removing the lock + file is an optimisation, not a necessity. */ +} + + +bool lockFile(int fd, LockType lockType, bool wait) +{ + struct flock lock; + if (lockType == ltRead) lock.l_type = F_RDLCK; + else if (lockType == ltWrite) lock.l_type = F_WRLCK; + else if (lockType == ltNone) lock.l_type = F_UNLCK; + else abort(); + lock.l_whence = SEEK_SET; + lock.l_start = 0; + lock.l_len = 0; /* entire file */ + + if (wait) { + while (fcntl(fd, F_SETLKW, &lock) != 0) { + checkInterrupt(); + if (errno != EINTR) + throw SysError(format("acquiring/releasing lock")); + } + } else { + while (fcntl(fd, F_SETLK, &lock) != 0) { + checkInterrupt(); + if (errno == EACCES || errno == EAGAIN) return false; + if (errno != EINTR) + throw SysError(format("acquiring/releasing lock")); + } + } + + return true; +} + + +/* This enables us to check whether are not already holding a lock on + a file ourselves. POSIX locks (fcntl) suck in this respect: if we + close a descriptor, the previous lock will be closed as well. And + there is no way to query whether we already have a lock (F_GETLK + only works on locks held by other processes). */ +static StringSet lockedPaths; /* !!! not thread-safe */ + + +PathLocks::PathLocks() + : deletePaths(false) +{ +} + + +PathLocks::PathLocks(const PathSet & paths, const string & waitMsg) + : deletePaths(false) +{ + lockPaths(paths, waitMsg); +} + + +bool PathLocks::lockPaths(const PathSet & _paths, + const string & waitMsg, bool wait) +{ + assert(fds.empty()); + + /* Note that `fds' is built incrementally so that the destructor + will only release those locks that we have already acquired. */ + + /* Sort the paths. This assures that locks are always acquired in + the same order, thus preventing deadlocks. */ + Paths paths(_paths.begin(), _paths.end()); + paths.sort(); + + /* Acquire the lock for each path. */ + foreach (Paths::iterator, i, paths) { + checkInterrupt(); + Path path = *i; + Path lockPath = path + ".lock"; + + debug(format("locking path `%1%'") % path); + + if (lockedPaths.find(lockPath) != lockedPaths.end()) + throw Error("deadlock: trying to re-acquire self-held lock"); + + AutoCloseFD fd; + + while (1) { + + /* Open/create the lock file. */ + fd = openLockFile(lockPath, true); + + /* Acquire an exclusive lock. */ + if (!lockFile(fd, ltWrite, false)) { + if (wait) { + if (waitMsg != "") printMsg(lvlError, waitMsg); + lockFile(fd, ltWrite, true); + } else { + /* Failed to lock this path; release all other + locks. */ + unlock(); + return false; + } + } + + debug(format("lock acquired on `%1%'") % lockPath); + + /* Check that the lock file hasn't become stale (i.e., + hasn't been unlinked). */ + struct stat st; + if (fstat(fd, &st) == -1) + throw SysError(format("statting lock file `%1%'") % lockPath); + if (st.st_size != 0) + /* This lock file has been unlinked, so we're holding + a lock on a deleted file. This means that other + processes may create and acquire a lock on + `lockPath', and proceed. So we must retry. */ + debug(format("open lock file `%1%' has become stale") % lockPath); + else + break; + } + + /* Use borrow so that the descriptor isn't closed. */ + fds.push_back(FDPair(fd.borrow(), lockPath)); + lockedPaths.insert(lockPath); + } + + return true; +} + + +PathLocks::~PathLocks() +{ + unlock(); +} + + +void PathLocks::unlock() +{ + foreach (list<FDPair>::iterator, i, fds) { + if (deletePaths) deleteLockFile(i->second, i->first); + + lockedPaths.erase(i->second); + if (close(i->first) == -1) + printMsg(lvlError, + format("error (ignored): cannot close lock file on `%1%'") % i->second); + + debug(format("lock released on `%1%'") % i->second); + } + + fds.clear(); +} + + +void PathLocks::setDeletion(bool deletePaths) +{ + this->deletePaths = deletePaths; +} + + +bool pathIsLockedByMe(const Path & path) +{ + Path lockPath = path + ".lock"; + return lockedPaths.find(lockPath) != lockedPaths.end(); +} + + +} diff --git a/src/libstore/pathlocks.hh b/src/libstore/pathlocks.hh new file mode 100644 index 000000000000..8a6b1450da2a --- /dev/null +++ b/src/libstore/pathlocks.hh @@ -0,0 +1,45 @@ +#pragma once + +#include "types.hh" + + +namespace nix { + + +/* Open (possibly create) a lock file and return the file descriptor. + -1 is returned if create is false and the lock could not be opened + because it doesn't exist. Any other error throws an exception. */ +int openLockFile(const Path & path, bool create); + +/* Delete an open lock file. */ +void deleteLockFile(const Path & path, int fd); + +enum LockType { ltRead, ltWrite, ltNone }; + +bool lockFile(int fd, LockType lockType, bool wait); + + +class PathLocks +{ +private: + typedef std::pair<int, Path> FDPair; + list<FDPair> fds; + bool deletePaths; + +public: + PathLocks(); + PathLocks(const PathSet & paths, + const string & waitMsg = ""); + bool lockPaths(const PathSet & _paths, + const string & waitMsg = "", + bool wait = true); + ~PathLocks(); + void unlock(); + void setDeletion(bool deletePaths); +}; + + +bool pathIsLockedByMe(const Path & path); + + +} diff --git a/src/libstore/references.cc b/src/libstore/references.cc new file mode 100644 index 000000000000..282b848938b3 --- /dev/null +++ b/src/libstore/references.cc @@ -0,0 +1,122 @@ +#include "references.hh" +#include "hash.hh" +#include "util.hh" +#include "archive.hh" + +#include <map> +#include <cstdlib> + + +namespace nix { + + +static unsigned int refLength = 32; /* characters */ + + +static void search(const unsigned char * s, unsigned int len, + StringSet & hashes, StringSet & seen) +{ + static bool initialised = false; + static bool isBase32[256]; + if (!initialised) { + for (unsigned int i = 0; i < 256; ++i) isBase32[i] = false; + for (unsigned int i = 0; i < base32Chars.size(); ++i) + isBase32[(unsigned char) base32Chars[i]] = true; + initialised = true; + } + + for (unsigned int i = 0; i + refLength <= len; ) { + int j; + bool match = true; + for (j = refLength - 1; j >= 0; --j) + if (!isBase32[(unsigned char) s[i + j]]) { + i += j + 1; + match = false; + break; + } + if (!match) continue; + string ref((const char *) s + i, refLength); + if (hashes.find(ref) != hashes.end()) { + debug(format("found reference to `%1%' at offset `%2%'") + % ref % i); + seen.insert(ref); + hashes.erase(ref); + } + ++i; + } +} + + +struct RefScanSink : Sink +{ + HashSink hashSink; + StringSet hashes; + StringSet seen; + + string tail; + + RefScanSink() : hashSink(htSHA256) { } + + void operator () (const unsigned char * data, size_t len); +}; + + +void RefScanSink::operator () (const unsigned char * data, size_t len) +{ + hashSink(data, len); + + /* It's possible that a reference spans the previous and current + fragment, so search in the concatenation of the tail of the + previous fragment and the start of the current fragment. */ + string s = tail + string((const char *) data, len > refLength ? refLength : len); + search((const unsigned char *) s.data(), s.size(), hashes, seen); + + search(data, len, hashes, seen); + + unsigned int tailLen = len <= refLength ? len : refLength; + tail = + string(tail, tail.size() < refLength - tailLen ? 0 : tail.size() - (refLength - tailLen)) + + string((const char *) data + len - tailLen, tailLen); +} + + +PathSet scanForReferences(const string & path, + const PathSet & refs, HashResult & hash) +{ + RefScanSink sink; + std::map<string, Path> backMap; + + /* For efficiency (and a higher hit rate), just search for the + hash part of the file name. (This assumes that all references + have the form `HASH-bla'). */ + foreach (PathSet::const_iterator, i, refs) { + string baseName = baseNameOf(*i); + string::size_type pos = baseName.find('-'); + if (pos == string::npos) + throw Error(format("bad reference `%1%'") % *i); + string s = string(baseName, 0, pos); + assert(s.size() == refLength); + assert(backMap.find(s) == backMap.end()); + // parseHash(htSHA256, s); + sink.hashes.insert(s); + backMap[s] = *i; + } + + /* Look for the hashes in the NAR dump of the path. */ + dumpPath(path, sink); + + /* Map the hashes found back to their store paths. */ + PathSet found; + foreach (StringSet::iterator, i, sink.seen) { + std::map<string, Path>::iterator j; + if ((j = backMap.find(*i)) == backMap.end()) abort(); + found.insert(j->second); + } + + hash = sink.hashSink.finish(); + + return found; +} + + +} diff --git a/src/libstore/references.hh b/src/libstore/references.hh new file mode 100644 index 000000000000..013809d122f3 --- /dev/null +++ b/src/libstore/references.hh @@ -0,0 +1,11 @@ +#pragma once + +#include "types.hh" +#include "hash.hh" + +namespace nix { + +PathSet scanForReferences(const Path & path, const PathSet & refs, + HashResult & hash); + +} diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc new file mode 100644 index 000000000000..3017254baf3b --- /dev/null +++ b/src/libstore/remote-store.cc @@ -0,0 +1,602 @@ +#include "serialise.hh" +#include "util.hh" +#include "remote-store.hh" +#include "worker-protocol.hh" +#include "archive.hh" +#include "affinity.hh" +#include "globals.hh" + +#include <sys/types.h> +#include <sys/stat.h> +#include <sys/socket.h> +#include <sys/un.h> +#include <fcntl.h> + +#include <iostream> +#include <unistd.h> +#include <cstring> + +namespace nix { + + +Path readStorePath(Source & from) +{ + Path path = readString(from); + assertStorePath(path); + return path; +} + + +template<class T> T readStorePaths(Source & from) +{ + T paths = readStrings<T>(from); + foreach (typename T::iterator, i, paths) assertStorePath(*i); + return paths; +} + +template PathSet readStorePaths(Source & from); + + +RemoteStore::RemoteStore() +{ + initialised = false; +} + + +void RemoteStore::openConnection(bool reserveSpace) +{ + if (initialised) return; + initialised = true; + + string remoteMode = getEnv("NIX_REMOTE"); + + if (remoteMode == "daemon") + /* Connect to a daemon that does the privileged work for + us. */ + connectToDaemon(); + else + throw Error(format("invalid setting for NIX_REMOTE, `%1%'") % remoteMode); + + from.fd = fdSocket; + to.fd = fdSocket; + + /* Send the magic greeting, check for the reply. */ + try { + writeInt(WORKER_MAGIC_1, to); + to.flush(); + unsigned int magic = readInt(from); + if (magic != WORKER_MAGIC_2) throw Error("protocol mismatch"); + + daemonVersion = readInt(from); + if (GET_PROTOCOL_MAJOR(daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION)) + throw Error("Nix daemon protocol version not supported"); + writeInt(PROTOCOL_VERSION, to); + + if (GET_PROTOCOL_MINOR(daemonVersion) >= 14) { + int cpu = settings.lockCPU ? lockToCurrentCPU() : -1; + if (cpu != -1) { + writeInt(1, to); + writeInt(cpu, to); + } else + writeInt(0, to); + } + + if (GET_PROTOCOL_MINOR(daemonVersion) >= 11) + writeInt(reserveSpace, to); + + processStderr(); + } + catch (Error & e) { + throw Error(format("cannot start worker (%1%)") + % e.msg()); + } + + setOptions(); +} + + +void RemoteStore::connectToDaemon() +{ + fdSocket = socket(PF_UNIX, SOCK_STREAM, 0); + if (fdSocket == -1) + throw SysError("cannot create Unix domain socket"); + closeOnExec(fdSocket); + + string socketPath = settings.nixDaemonSocketFile; + + /* Urgh, sockaddr_un allows path names of only 108 characters. So + chdir to the socket directory so that we can pass a relative + path name. !!! this is probably a bad idea in multi-threaded + applications... */ + AutoCloseFD fdPrevDir = open(".", O_RDONLY); + if (fdPrevDir == -1) throw SysError("couldn't open current directory"); + chdir(dirOf(socketPath).c_str()); + Path socketPathRel = "./" + baseNameOf(socketPath); + + struct sockaddr_un addr; + addr.sun_family = AF_UNIX; + if (socketPathRel.size() >= sizeof(addr.sun_path)) + throw Error(format("socket path `%1%' is too long") % socketPathRel); + using namespace std; + strcpy(addr.sun_path, socketPathRel.c_str()); + + if (connect(fdSocket, (struct sockaddr *) &addr, sizeof(addr)) == -1) + throw SysError(format("cannot connect to daemon at `%1%'") % socketPath); + + if (fchdir(fdPrevDir) == -1) + throw SysError("couldn't change back to previous directory"); +} + + +RemoteStore::~RemoteStore() +{ + try { + to.flush(); + fdSocket.close(); + if (child != -1) + child.wait(true); + } catch (...) { + ignoreException(); + } +} + + +void RemoteStore::setOptions() +{ + writeInt(wopSetOptions, to); + + writeInt(settings.keepFailed, to); + writeInt(settings.keepGoing, to); + writeInt(settings.tryFallback, to); + writeInt(verbosity, to); + writeInt(settings.maxBuildJobs, to); + writeInt(settings.maxSilentTime, to); + if (GET_PROTOCOL_MINOR(daemonVersion) >= 2) + writeInt(settings.useBuildHook, to); + if (GET_PROTOCOL_MINOR(daemonVersion) >= 4) { + writeInt(settings.buildVerbosity, to); + writeInt(logType, to); + writeInt(settings.printBuildTrace, to); + } + if (GET_PROTOCOL_MINOR(daemonVersion) >= 6) + writeInt(settings.buildCores, to); + if (GET_PROTOCOL_MINOR(daemonVersion) >= 10) + writeInt(settings.useSubstitutes, to); + + if (GET_PROTOCOL_MINOR(daemonVersion) >= 12) { + Settings::SettingsMap overrides = settings.getOverrides(); + writeInt(overrides.size(), to); + foreach (Settings::SettingsMap::iterator, i, overrides) { + writeString(i->first, to); + writeString(i->second, to); + } + } + + processStderr(); +} + + +bool RemoteStore::isValidPath(const Path & path) +{ + openConnection(); + writeInt(wopIsValidPath, to); + writeString(path, to); + processStderr(); + unsigned int reply = readInt(from); + return reply != 0; +} + + +PathSet RemoteStore::queryValidPaths(const PathSet & paths) +{ + openConnection(); + if (GET_PROTOCOL_MINOR(daemonVersion) < 12) { + PathSet res; + foreach (PathSet::const_iterator, i, paths) + if (isValidPath(*i)) res.insert(*i); + return res; + } else { + writeInt(wopQueryValidPaths, to); + writeStrings(paths, to); + processStderr(); + return readStorePaths<PathSet>(from); + } +} + + +PathSet RemoteStore::queryAllValidPaths() +{ + openConnection(); + writeInt(wopQueryAllValidPaths, to); + processStderr(); + return readStorePaths<PathSet>(from); +} + + +PathSet RemoteStore::querySubstitutablePaths(const PathSet & paths) +{ + openConnection(); + if (GET_PROTOCOL_MINOR(daemonVersion) < 12) { + PathSet res; + foreach (PathSet::const_iterator, i, paths) { + writeInt(wopHasSubstitutes, to); + writeString(*i, to); + processStderr(); + if (readInt(from)) res.insert(*i); + } + return res; + } else { + writeInt(wopQuerySubstitutablePaths, to); + writeStrings(paths, to); + processStderr(); + return readStorePaths<PathSet>(from); + } +} + + +void RemoteStore::querySubstitutablePathInfos(const PathSet & paths, + SubstitutablePathInfos & infos) +{ + if (paths.empty()) return; + + openConnection(); + + if (GET_PROTOCOL_MINOR(daemonVersion) < 3) return; + + if (GET_PROTOCOL_MINOR(daemonVersion) < 12) { + + foreach (PathSet::const_iterator, i, paths) { + SubstitutablePathInfo info; + writeInt(wopQuerySubstitutablePathInfo, to); + writeString(*i, to); + processStderr(); + unsigned int reply = readInt(from); + if (reply == 0) continue; + info.deriver = readString(from); + if (info.deriver != "") assertStorePath(info.deriver); + info.references = readStorePaths<PathSet>(from); + info.downloadSize = readLongLong(from); + info.narSize = GET_PROTOCOL_MINOR(daemonVersion) >= 7 ? readLongLong(from) : 0; + infos[*i] = info; + } + + } else { + + writeInt(wopQuerySubstitutablePathInfos, to); + writeStrings(paths, to); + processStderr(); + unsigned int count = readInt(from); + for (unsigned int n = 0; n < count; n++) { + Path path = readStorePath(from); + SubstitutablePathInfo & info(infos[path]); + info.deriver = readString(from); + if (info.deriver != "") assertStorePath(info.deriver); + info.references = readStorePaths<PathSet>(from); + info.downloadSize = readLongLong(from); + info.narSize = readLongLong(from); + } + + } +} + + +ValidPathInfo RemoteStore::queryPathInfo(const Path & path) +{ + openConnection(); + writeInt(wopQueryPathInfo, to); + writeString(path, to); + processStderr(); + ValidPathInfo info; + info.path = path; + info.deriver = readString(from); + if (info.deriver != "") assertStorePath(info.deriver); + info.hash = parseHash(htSHA256, readString(from)); + info.references = readStorePaths<PathSet>(from); + info.registrationTime = readInt(from); + info.narSize = readLongLong(from); + return info; +} + + +Hash RemoteStore::queryPathHash(const Path & path) +{ + openConnection(); + writeInt(wopQueryPathHash, to); + writeString(path, to); + processStderr(); + string hash = readString(from); + return parseHash(htSHA256, hash); +} + + +void RemoteStore::queryReferences(const Path & path, + PathSet & references) +{ + openConnection(); + writeInt(wopQueryReferences, to); + writeString(path, to); + processStderr(); + PathSet references2 = readStorePaths<PathSet>(from); + references.insert(references2.begin(), references2.end()); +} + + +void RemoteStore::queryReferrers(const Path & path, + PathSet & referrers) +{ + openConnection(); + writeInt(wopQueryReferrers, to); + writeString(path, to); + processStderr(); + PathSet referrers2 = readStorePaths<PathSet>(from); + referrers.insert(referrers2.begin(), referrers2.end()); +} + + +Path RemoteStore::queryDeriver(const Path & path) +{ + openConnection(); + writeInt(wopQueryDeriver, to); + writeString(path, to); + processStderr(); + Path drvPath = readString(from); + if (drvPath != "") assertStorePath(drvPath); + return drvPath; +} + + +PathSet RemoteStore::queryValidDerivers(const Path & path) +{ + openConnection(); + writeInt(wopQueryValidDerivers, to); + writeString(path, to); + processStderr(); + return readStorePaths<PathSet>(from); +} + + +PathSet RemoteStore::queryDerivationOutputs(const Path & path) +{ + openConnection(); + writeInt(wopQueryDerivationOutputs, to); + writeString(path, to); + processStderr(); + return readStorePaths<PathSet>(from); +} + + +PathSet RemoteStore::queryDerivationOutputNames(const Path & path) +{ + openConnection(); + writeInt(wopQueryDerivationOutputNames, to); + writeString(path, to); + processStderr(); + return readStrings<PathSet>(from); +} + + +Path RemoteStore::queryPathFromHashPart(const string & hashPart) +{ + openConnection(); + writeInt(wopQueryPathFromHashPart, to); + writeString(hashPart, to); + processStderr(); + Path path = readString(from); + if (!path.empty()) assertStorePath(path); + return path; +} + + +Path RemoteStore::addToStore(const Path & _srcPath, + bool recursive, HashType hashAlgo, PathFilter & filter, bool repair) +{ + if (repair) throw Error("repairing is not supported when building through the Nix daemon"); + + openConnection(); + + Path srcPath(absPath(_srcPath)); + + writeInt(wopAddToStore, to); + writeString(baseNameOf(srcPath), to); + /* backwards compatibility hack */ + writeInt((hashAlgo == htSHA256 && recursive) ? 0 : 1, to); + writeInt(recursive ? 1 : 0, to); + writeString(printHashType(hashAlgo), to); + dumpPath(srcPath, to, filter); + processStderr(); + return readStorePath(from); +} + + +Path RemoteStore::addTextToStore(const string & name, const string & s, + const PathSet & references, bool repair) +{ + if (repair) throw Error("repairing is not supported when building through the Nix daemon"); + + openConnection(); + writeInt(wopAddTextToStore, to); + writeString(name, to); + writeString(s, to); + writeStrings(references, to); + + processStderr(); + return readStorePath(from); +} + + +void RemoteStore::exportPath(const Path & path, bool sign, + Sink & sink) +{ + openConnection(); + writeInt(wopExportPath, to); + writeString(path, to); + writeInt(sign ? 1 : 0, to); + processStderr(&sink); /* sink receives the actual data */ + readInt(from); +} + + +Paths RemoteStore::importPaths(bool requireSignature, Source & source) +{ + openConnection(); + writeInt(wopImportPaths, to); + /* We ignore requireSignature, since the worker forces it to true + anyway. */ + processStderr(0, &source); + return readStorePaths<Paths>(from); +} + + +void RemoteStore::buildPaths(const PathSet & drvPaths, bool repair) +{ + if (repair) throw Error("repairing is not supported when building through the Nix daemon"); + openConnection(); + writeInt(wopBuildPaths, to); + if (GET_PROTOCOL_MINOR(daemonVersion) >= 13) + writeStrings(drvPaths, to); + else { + /* For backwards compatibility with old daemons, strip output + identifiers. */ + PathSet drvPaths2; + foreach (PathSet::const_iterator, i, drvPaths) + drvPaths2.insert(string(*i, 0, i->find('!'))); + writeStrings(drvPaths2, to); + } + processStderr(); + readInt(from); +} + + +void RemoteStore::ensurePath(const Path & path) +{ + openConnection(); + writeInt(wopEnsurePath, to); + writeString(path, to); + processStderr(); + readInt(from); +} + + +void RemoteStore::addTempRoot(const Path & path) +{ + openConnection(); + writeInt(wopAddTempRoot, to); + writeString(path, to); + processStderr(); + readInt(from); +} + + +void RemoteStore::addIndirectRoot(const Path & path) +{ + openConnection(); + writeInt(wopAddIndirectRoot, to); + writeString(path, to); + processStderr(); + readInt(from); +} + + +void RemoteStore::syncWithGC() +{ + openConnection(); + writeInt(wopSyncWithGC, to); + processStderr(); + readInt(from); +} + + +Roots RemoteStore::findRoots() +{ + openConnection(); + writeInt(wopFindRoots, to); + processStderr(); + unsigned int count = readInt(from); + Roots result; + while (count--) { + Path link = readString(from); + Path target = readStorePath(from); + result[link] = target; + } + return result; +} + + +void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) +{ + openConnection(false); + + writeInt(wopCollectGarbage, to); + writeInt(options.action, to); + writeStrings(options.pathsToDelete, to); + writeInt(options.ignoreLiveness, to); + writeLongLong(options.maxFreed, to); + writeInt(0, to); + if (GET_PROTOCOL_MINOR(daemonVersion) >= 5) { + /* removed options */ + writeInt(0, to); + writeInt(0, to); + } + + processStderr(); + + results.paths = readStrings<PathSet>(from); + results.bytesFreed = readLongLong(from); + readLongLong(from); // obsolete +} + + +PathSet RemoteStore::queryFailedPaths() +{ + openConnection(); + writeInt(wopQueryFailedPaths, to); + processStderr(); + return readStorePaths<PathSet>(from); +} + + +void RemoteStore::clearFailedPaths(const PathSet & paths) +{ + openConnection(); + writeInt(wopClearFailedPaths, to); + writeStrings(paths, to); + processStderr(); + readInt(from); +} + + +void RemoteStore::processStderr(Sink * sink, Source * source) +{ + to.flush(); + unsigned int msg; + while ((msg = readInt(from)) == STDERR_NEXT + || msg == STDERR_READ || msg == STDERR_WRITE) { + if (msg == STDERR_WRITE) { + string s = readString(from); + if (!sink) throw Error("no sink"); + (*sink)((const unsigned char *) s.data(), s.size()); + } + else if (msg == STDERR_READ) { + if (!source) throw Error("no source"); + size_t len = readInt(from); + unsigned char * buf = new unsigned char[len]; + AutoDeleteArray<unsigned char> d(buf); + writeString(buf, source->read(buf, len), to); + to.flush(); + } + else { + string s = readString(from); + writeToStderr(s); + } + } + if (msg == STDERR_ERROR) { + string error = readString(from); + unsigned int status = GET_PROTOCOL_MINOR(daemonVersion) >= 8 ? readInt(from) : 1; + throw Error(format("%1%") % error, status); + } + else if (msg != STDERR_LAST) + throw Error("protocol error processing standard error"); +} + + +} diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh new file mode 100644 index 000000000000..14253b92cd0e --- /dev/null +++ b/src/libstore/remote-store.hh @@ -0,0 +1,104 @@ +#pragma once + +#include <string> + +#include "store-api.hh" + + +namespace nix { + + +class Pipe; +class Pid; +struct FdSink; +struct FdSource; + + +class RemoteStore : public StoreAPI +{ +public: + + RemoteStore(); + + ~RemoteStore(); + + /* Implementations of abstract store API methods. */ + + bool isValidPath(const Path & path); + + PathSet queryValidPaths(const PathSet & paths); + + PathSet queryAllValidPaths(); + + ValidPathInfo queryPathInfo(const Path & path); + + Hash queryPathHash(const Path & path); + + void queryReferences(const Path & path, PathSet & references); + + void queryReferrers(const Path & path, PathSet & referrers); + + Path queryDeriver(const Path & path); + + PathSet queryValidDerivers(const Path & path); + + PathSet queryDerivationOutputs(const Path & path); + + StringSet queryDerivationOutputNames(const Path & path); + + Path queryPathFromHashPart(const string & hashPart); + + PathSet querySubstitutablePaths(const PathSet & paths); + + void querySubstitutablePathInfos(const PathSet & paths, + SubstitutablePathInfos & infos); + + Path addToStore(const Path & srcPath, + bool recursive = true, HashType hashAlgo = htSHA256, + PathFilter & filter = defaultPathFilter, bool repair = false); + + Path addTextToStore(const string & name, const string & s, + const PathSet & references, bool repair = false); + + void exportPath(const Path & path, bool sign, + Sink & sink); + + Paths importPaths(bool requireSignature, Source & source); + + void buildPaths(const PathSet & paths, bool repair = false); + + void ensurePath(const Path & path); + + void addTempRoot(const Path & path); + + void addIndirectRoot(const Path & path); + + void syncWithGC(); + + Roots findRoots(); + + void collectGarbage(const GCOptions & options, GCResults & results); + + PathSet queryFailedPaths(); + + void clearFailedPaths(const PathSet & paths); + +private: + AutoCloseFD fdSocket; + FdSink to; + FdSource from; + Pid child; + unsigned int daemonVersion; + bool initialised; + + void openConnection(bool reserveSpace = true); + + void processStderr(Sink * sink = 0, Source * source = 0); + + void connectToDaemon(); + + void setOptions(); +}; + + +} diff --git a/src/libstore/schema.sql b/src/libstore/schema.sql new file mode 100644 index 000000000000..c1b4a689afcb --- /dev/null +++ b/src/libstore/schema.sql @@ -0,0 +1,44 @@ +create table if not exists ValidPaths ( + id integer primary key autoincrement not null, + path text unique not null, + hash text not null, + registrationTime integer not null, + deriver text, + narSize integer +); + +create table if not exists Refs ( + referrer integer not null, + reference integer not null, + primary key (referrer, reference), + foreign key (referrer) references ValidPaths(id) on delete cascade, + foreign key (reference) references ValidPaths(id) on delete restrict +); + +create index if not exists IndexReferrer on Refs(referrer); +create index if not exists IndexReference on Refs(reference); + +-- Paths can refer to themselves, causing a tuple (N, N) in the Refs +-- table. This causes a deletion of the corresponding row in +-- ValidPaths to cause a foreign key constraint violation (due to `on +-- delete restrict' on the `reference' column). Therefore, explicitly +-- get rid of self-references. +create trigger if not exists DeleteSelfRefs before delete on ValidPaths + begin + delete from Refs where referrer = old.id and reference = old.id; + end; + +create table if not exists DerivationOutputs ( + drv integer not null, + id text not null, -- symbolic output id, usually "out" + path text not null, + primary key (drv, id), + foreign key (drv) references ValidPaths(id) on delete cascade +); + +create index if not exists IndexDerivationOutputs on DerivationOutputs(path); + +create table if not exists FailedPaths ( + path text primary key not null, + time integer not null +); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc new file mode 100644 index 000000000000..0f250a3c7c04 --- /dev/null +++ b/src/libstore/store-api.cc @@ -0,0 +1,331 @@ +#include "store-api.hh" +#include "globals.hh" +#include "util.hh" + +#include <climits> + + +namespace nix { + + +GCOptions::GCOptions() +{ + action = gcDeleteDead; + ignoreLiveness = false; + maxFreed = ULLONG_MAX; +} + + +bool isInStore(const Path & path) +{ + return isInDir(path, settings.nixStore); +} + + +bool isStorePath(const Path & path) +{ + return isInStore(path) + && path.find('/', settings.nixStore.size() + 1) == Path::npos; +} + + +void assertStorePath(const Path & path) +{ + if (!isStorePath(path)) + throw Error(format("path `%1%' is not in the Nix store") % path); +} + + +Path toStorePath(const Path & path) +{ + if (!isInStore(path)) + throw Error(format("path `%1%' is not in the Nix store") % path); + Path::size_type slash = path.find('/', settings.nixStore.size() + 1); + if (slash == Path::npos) + return path; + else + return Path(path, 0, slash); +} + + +Path followLinksToStore(const Path & _path) +{ + Path path = absPath(_path); + while (!isInStore(path)) { + if (!isLink(path)) break; + string target = readLink(path); + path = absPath(target, dirOf(path)); + } + if (!isInStore(path)) + throw Error(format("path `%1%' is not in the Nix store") % path); + return path; +} + + +Path followLinksToStorePath(const Path & path) +{ + return toStorePath(followLinksToStore(path)); +} + + +string storePathToName(const Path & path) +{ + assertStorePath(path); + return string(path, settings.nixStore.size() + 34); +} + + +void checkStoreName(const string & name) +{ + string validChars = "+-._?="; + /* Disallow names starting with a dot for possible security + reasons (e.g., "." and ".."). */ + if (string(name, 0, 1) == ".") + throw Error(format("illegal name: `%1%'") % name); + foreach (string::const_iterator, i, name) + if (!((*i >= 'A' && *i <= 'Z') || + (*i >= 'a' && *i <= 'z') || + (*i >= '0' && *i <= '9') || + validChars.find(*i) != string::npos)) + { + throw Error(format("invalid character `%1%' in name `%2%'") + % *i % name); + } +} + + +/* Store paths have the following form: + + <store>/<h>-<name> + + where + + <store> = the location of the Nix store, usually /nix/store + + <name> = a human readable name for the path, typically obtained + from the name attribute of the derivation, or the name of the + source file from which the store path is created. For derivation + outputs other than the default "out" output, the string "-<id>" + is suffixed to <name>. + + <h> = base-32 representation of the first 160 bits of a SHA-256 + hash of <s>; the hash part of the store name + + <s> = the string "<type>:sha256:<h2>:<store>:<name>"; + note that it includes the location of the store as well as the + name to make sure that changes to either of those are reflected + in the hash (e.g. you won't get /nix/store/<h>-name1 and + /nix/store/<h>-name2 with equal hash parts). + + <type> = one of: + "text:<r1>:<r2>:...<rN>" + for plain text files written to the store using + addTextToStore(); <r1> ... <rN> are the references of the + path. + "source" + for paths copied to the store using addToStore() when recursive + = true and hashAlgo = "sha256" + "output:<id>" + for either the outputs created by derivations, OR paths copied + to the store using addToStore() with recursive != true or + hashAlgo != "sha256" (in that case "source" is used; it's + silly, but it's done that way for compatibility). <id> is the + name of the output (usually, "out"). + + <h2> = base-16 representation of a SHA-256 hash of: + if <type> = "text:...": + the string written to the resulting store path + if <type> = "source": + the serialisation of the path from which this store path is + copied, as returned by hashPath() + if <type> = "output:out": + for non-fixed derivation outputs: + the derivation (see hashDerivationModulo() in + primops.cc) + for paths copied by addToStore() or produced by fixed-output + derivations: + the string "fixed:out:<rec><algo>:<hash>:", where + <rec> = "r:" for recursive (path) hashes, or "" or flat + (file) hashes + <algo> = "md5", "sha1" or "sha256" + <hash> = base-16 representation of the path or flat hash of + the contents of the path (or expected contents of the + path for fixed-output derivations) + + It would have been nicer to handle fixed-output derivations under + "source", e.g. have something like "source:<rec><algo>", but we're + stuck with this for now... + + The main reason for this way of computing names is to prevent name + collisions (for security). For instance, it shouldn't be feasible + to come up with a derivation whose output path collides with the + path for a copied source. The former would have a <s> starting with + "output:out:", while the latter would have a <2> starting with + "source:". +*/ + + +Path makeStorePath(const string & type, + const Hash & hash, const string & name) +{ + /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */ + string s = type + ":sha256:" + printHash(hash) + ":" + + settings.nixStore + ":" + name; + + checkStoreName(name); + + return settings.nixStore + "/" + + printHash32(compressHash(hashString(htSHA256, s), 20)) + + "-" + name; +} + + +Path makeOutputPath(const string & id, + const Hash & hash, const string & name) +{ + return makeStorePath("output:" + id, hash, + name + (id == "out" ? "" : "-" + id)); +} + + +Path makeFixedOutputPath(bool recursive, + HashType hashAlgo, Hash hash, string name) +{ + return hashAlgo == htSHA256 && recursive + ? makeStorePath("source", hash, name) + : makeStorePath("output:out", hashString(htSHA256, + "fixed:out:" + (recursive ? (string) "r:" : "") + + printHashType(hashAlgo) + ":" + printHash(hash) + ":"), + name); +} + + +std::pair<Path, Hash> computeStorePathForPath(const Path & srcPath, + bool recursive, HashType hashAlgo, PathFilter & filter) +{ + HashType ht(hashAlgo); + Hash h = recursive ? hashPath(ht, srcPath, filter).first : hashFile(ht, srcPath); + string name = baseNameOf(srcPath); + Path dstPath = makeFixedOutputPath(recursive, hashAlgo, h, name); + return std::pair<Path, Hash>(dstPath, h); +} + + +Path computeStorePathForText(const string & name, const string & s, + const PathSet & references) +{ + Hash hash = hashString(htSHA256, s); + /* Stuff the references (if any) into the type. This is a bit + hacky, but we can't put them in `s' since that would be + ambiguous. */ + string type = "text"; + foreach (PathSet::const_iterator, i, references) { + type += ":"; + type += *i; + } + return makeStorePath(type, hash, name); +} + + +/* Return a string accepted by decodeValidPathInfo() that + registers the specified paths as valid. Note: it's the + responsibility of the caller to provide a closure. */ +string StoreAPI::makeValidityRegistration(const PathSet & paths, + bool showDerivers, bool showHash) +{ + string s = ""; + + foreach (PathSet::iterator, i, paths) { + s += *i + "\n"; + + ValidPathInfo info = queryPathInfo(*i); + + if (showHash) { + s += printHash(info.hash) + "\n"; + s += (format("%1%\n") % info.narSize).str(); + } + + Path deriver = showDerivers ? info.deriver : ""; + s += deriver + "\n"; + + s += (format("%1%\n") % info.references.size()).str(); + + foreach (PathSet::iterator, j, info.references) + s += *j + "\n"; + } + + return s; +} + + +ValidPathInfo decodeValidPathInfo(std::istream & str, bool hashGiven) +{ + ValidPathInfo info; + getline(str, info.path); + if (str.eof()) { info.path = ""; return info; } + if (hashGiven) { + string s; + getline(str, s); + info.hash = parseHash(htSHA256, s); + getline(str, s); + if (!string2Int(s, info.narSize)) throw Error("number expected"); + } + getline(str, info.deriver); + string s; int n; + getline(str, s); + if (!string2Int(s, n)) throw Error("number expected"); + while (n--) { + getline(str, s); + info.references.insert(s); + } + if (!str || str.eof()) throw Error("missing input"); + return info; +} + + +string showPaths(const PathSet & paths) +{ + string s; + foreach (PathSet::const_iterator, i, paths) { + if (s.size() != 0) s += ", "; + s += "`" + *i + "'"; + } + return s; +} + + +void exportPaths(StoreAPI & store, const Paths & paths, + bool sign, Sink & sink) +{ + foreach (Paths::const_iterator, i, paths) { + writeInt(1, sink); + store.exportPath(*i, sign, sink); + } + writeInt(0, sink); +} + + +} + + +#include "local-store.hh" +#include "serialise.hh" +#include "remote-store.hh" + + +namespace nix { + + +boost::shared_ptr<StoreAPI> store; + + +boost::shared_ptr<StoreAPI> openStore(bool reserveSpace) +{ + if (getEnv("NIX_REMOTE") == "") + return boost::shared_ptr<StoreAPI>(new LocalStore(reserveSpace)); + else + return boost::shared_ptr<StoreAPI>(new RemoteStore()); +} + + +} diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh new file mode 100644 index 000000000000..a82fe3221639 --- /dev/null +++ b/src/libstore/store-api.hh @@ -0,0 +1,364 @@ +#pragma once + +#include "hash.hh" +#include "serialise.hh" + +#include <string> +#include <map> + +#include <boost/shared_ptr.hpp> + + +namespace nix { + + +typedef std::map<Path, Path> Roots; + + +struct GCOptions +{ + /* Garbage collector operation: + + - `gcReturnLive': return the set of paths reachable from + (i.e. in the closure of) the roots. + + - `gcReturnDead': return the set of paths not reachable from + the roots. + + - `gcDeleteDead': actually delete the latter set. + + - `gcDeleteSpecific': delete the paths listed in + `pathsToDelete', insofar as they are not reachable. + */ + typedef enum { + gcReturnLive, + gcReturnDead, + gcDeleteDead, + gcDeleteSpecific, + } GCAction; + + GCAction action; + + /* If `ignoreLiveness' is set, then reachability from the roots is + ignored (dangerous!). However, the paths must still be + unreferenced *within* the store (i.e., there can be no other + store paths that depend on them). */ + bool ignoreLiveness; + + /* For `gcDeleteSpecific', the paths to delete. */ + PathSet pathsToDelete; + + /* Stop after at least `maxFreed' bytes have been freed. */ + unsigned long long maxFreed; + + GCOptions(); +}; + + +struct GCResults +{ + /* Depending on the action, the GC roots, or the paths that would + be or have been deleted. */ + PathSet paths; + + /* For `gcReturnDead', `gcDeleteDead' and `gcDeleteSpecific', the + number of bytes that would be or was freed. */ + unsigned long long bytesFreed; + + GCResults() + { + bytesFreed = 0; + } +}; + + +struct SubstitutablePathInfo +{ + Path deriver; + PathSet references; + unsigned long long downloadSize; /* 0 = unknown or inapplicable */ + unsigned long long narSize; /* 0 = unknown */ +}; + +typedef std::map<Path, SubstitutablePathInfo> SubstitutablePathInfos; + + +struct ValidPathInfo +{ + Path path; + Path deriver; + Hash hash; + PathSet references; + time_t registrationTime; + unsigned long long narSize; // 0 = unknown + unsigned long long id; // internal use only + ValidPathInfo() : registrationTime(0), narSize(0) { } +}; + +typedef list<ValidPathInfo> ValidPathInfos; + + +class StoreAPI +{ +public: + + virtual ~StoreAPI() { } + + /* Check whether a path is valid. */ + virtual bool isValidPath(const Path & path) = 0; + + /* Query which of the given paths is valid. */ + virtual PathSet queryValidPaths(const PathSet & paths) = 0; + + /* Query the set of all valid paths. */ + virtual PathSet queryAllValidPaths() = 0; + + /* Query information about a valid path. */ + virtual ValidPathInfo queryPathInfo(const Path & path) = 0; + + /* Query the hash of a valid path. */ + virtual Hash queryPathHash(const Path & path) = 0; + + /* Query the set of outgoing FS references for a store path. The + result is not cleared. */ + virtual void queryReferences(const Path & path, + PathSet & references) = 0; + + /* Queries the set of incoming FS references for a store path. + The result is not cleared. */ + virtual void queryReferrers(const Path & path, + PathSet & referrers) = 0; + + /* Query the deriver of a store path. Return the empty string if + no deriver has been set. */ + virtual Path queryDeriver(const Path & path) = 0; + + /* Return all currently valid derivations that have `path' as an + output. (Note that the result of `queryDeriver()' is the + derivation that was actually used to produce `path', which may + not exist anymore.) */ + virtual PathSet queryValidDerivers(const Path & path) = 0; + + /* Query the outputs of the derivation denoted by `path'. */ + virtual PathSet queryDerivationOutputs(const Path & path) = 0; + + /* Query the output names of the derivation denoted by `path'. */ + virtual StringSet queryDerivationOutputNames(const Path & path) = 0; + + /* Query the full store path given the hash part of a valid store + path, or "" if the path doesn't exist. */ + virtual Path queryPathFromHashPart(const string & hashPart) = 0; + + /* Query which of the given paths have substitutes. */ + virtual PathSet querySubstitutablePaths(const PathSet & paths) = 0; + + /* Query substitute info (i.e. references, derivers and download + sizes) of a set of paths. If a path does not have substitute + info, it's omitted from the resulting ‘infos’ map. */ + virtual void querySubstitutablePathInfos(const PathSet & paths, + SubstitutablePathInfos & infos) = 0; + + /* Copy the contents of a path to the store and register the + validity the resulting path. The resulting path is returned. + The function object `filter' can be used to exclude files (see + libutil/archive.hh). */ + virtual Path addToStore(const Path & srcPath, + bool recursive = true, HashType hashAlgo = htSHA256, + PathFilter & filter = defaultPathFilter, bool repair = false) = 0; + + /* Like addToStore, but the contents written to the output path is + a regular file containing the given string. */ + virtual Path addTextToStore(const string & name, const string & s, + const PathSet & references, bool repair = false) = 0; + + /* Export a store path, that is, create a NAR dump of the store + path and append its references and its deriver. Optionally, a + cryptographic signature (created by OpenSSL) of the preceding + data is attached. */ + virtual void exportPath(const Path & path, bool sign, + Sink & sink) = 0; + + /* Import a sequence of NAR dumps created by exportPaths() into + the Nix store. */ + virtual Paths importPaths(bool requireSignature, Source & source) = 0; + + /* For each path, if it's a derivation, build it. Building a + derivation means ensuring that the output paths are valid. If + they are already valid, this is a no-op. Otherwise, validity + can be reached in two ways. First, if the output paths is + substitutable, then build the path that way. Second, the + output paths can be created by running the builder, after + recursively building any sub-derivations. For inputs that are + not derivations, substitute them. */ + virtual void buildPaths(const PathSet & paths, bool repair = false) = 0; + + /* Ensure that a path is valid. If it is not currently valid, it + may be made valid by running a substitute (if defined for the + path). */ + virtual void ensurePath(const Path & path) = 0; + + /* Add a store path as a temporary root of the garbage collector. + The root disappears as soon as we exit. */ + virtual void addTempRoot(const Path & path) = 0; + + /* Add an indirect root, which is merely a symlink to `path' from + /nix/var/nix/gcroots/auto/<hash of `path'>. `path' is supposed + to be a symlink to a store path. The garbage collector will + automatically remove the indirect root when it finds that + `path' has disappeared. */ + virtual void addIndirectRoot(const Path & path) = 0; + + /* Acquire the global GC lock, then immediately release it. This + function must be called after registering a new permanent root, + but before exiting. Otherwise, it is possible that a running + garbage collector doesn't see the new root and deletes the + stuff we've just built. By acquiring the lock briefly, we + ensure that either: + + - The collector is already running, and so we block until the + collector is finished. The collector will know about our + *temporary* locks, which should include whatever it is we + want to register as a permanent lock. + + - The collector isn't running, or it's just started but hasn't + acquired the GC lock yet. In that case we get and release + the lock right away, then exit. The collector scans the + permanent root and sees our's. + + In either case the permanent root is seen by the collector. */ + virtual void syncWithGC() = 0; + + /* Find the roots of the garbage collector. Each root is a pair + (link, storepath) where `link' is the path of the symlink + outside of the Nix store that point to `storePath'. */ + virtual Roots findRoots() = 0; + + /* Perform a garbage collection. */ + virtual void collectGarbage(const GCOptions & options, GCResults & results) = 0; + + /* Return the set of paths that have failed to build.*/ + virtual PathSet queryFailedPaths() = 0; + + /* Clear the "failed" status of the given paths. The special + value `*' causes all failed paths to be cleared. */ + virtual void clearFailedPaths(const PathSet & paths) = 0; + + /* Return a string representing information about the path that + can be loaded into the database using `nix-store --load-db' or + `nix-store --register-validity'. */ + string makeValidityRegistration(const PathSet & paths, + bool showDerivers, bool showHash); +}; + + +/* !!! These should be part of the store API, I guess. */ + +/* Throw an exception if `path' is not directly in the Nix store. */ +void assertStorePath(const Path & path); + +bool isInStore(const Path & path); +bool isStorePath(const Path & path); + +/* Extract the name part of the given store path. */ +string storePathToName(const Path & path); + +void checkStoreName(const string & name); + + +/* Chop off the parts after the top-level store name, e.g., + /nix/store/abcd-foo/bar => /nix/store/abcd-foo. */ +Path toStorePath(const Path & path); + + +/* Follow symlinks until we end up with a path in the Nix store. */ +Path followLinksToStore(const Path & path); + + +/* Same as followLinksToStore(), but apply toStorePath() to the + result. */ +Path followLinksToStorePath(const Path & path); + + +/* Constructs a unique store path name. */ +Path makeStorePath(const string & type, + const Hash & hash, const string & name); + +Path makeOutputPath(const string & id, + const Hash & hash, const string & name); + +Path makeFixedOutputPath(bool recursive, + HashType hashAlgo, Hash hash, string name); + + +/* This is the preparatory part of addToStore() and addToStoreFixed(); + it computes the store path to which srcPath is to be copied. + Returns the store path and the cryptographic hash of the + contents of srcPath. */ +std::pair<Path, Hash> computeStorePathForPath(const Path & srcPath, + bool recursive = true, HashType hashAlgo = htSHA256, + PathFilter & filter = defaultPathFilter); + +/* Preparatory part of addTextToStore(). + + !!! Computation of the path should take the references given to + addTextToStore() into account, otherwise we have a (relatively + minor) security hole: a caller can register a source file with + bogus references. If there are too many references, the path may + not be garbage collected when it has to be (not really a problem, + the caller could create a root anyway), or it may be garbage + collected when it shouldn't be (more serious). + + Hashing the references would solve this (bogus references would + simply yield a different store path, so other users wouldn't be + affected), but it has some backwards compatibility issues (the + hashing scheme changes), so I'm not doing that for now. */ +Path computeStorePathForText(const string & name, const string & s, + const PathSet & references); + + +/* Remove the temporary roots file for this process. Any temporary + root becomes garbage after this point unless it has been registered + as a (permanent) root. */ +void removeTempRoots(); + + +/* Register a permanent GC root. */ +Path addPermRoot(StoreAPI & store, const Path & storePath, + const Path & gcRoot, bool indirect, bool allowOutsideRootsDir = false); + + +/* Sort a set of paths topologically under the references relation. + If p refers to q, then p preceeds q in this list. */ +Paths topoSortPaths(StoreAPI & store, const PathSet & paths); + + +/* For now, there is a single global store API object, but we'll + purify that in the future. */ +extern boost::shared_ptr<StoreAPI> store; + + +/* Factory method: open the Nix database, either through the local or + remote implementation. */ +boost::shared_ptr<StoreAPI> openStore(bool reserveSpace = true); + + +/* Display a set of paths in human-readable form (i.e., between quotes + and separated by commas). */ +string showPaths(const PathSet & paths); + + +ValidPathInfo decodeValidPathInfo(std::istream & str, + bool hashGiven = false); + + +/* Export multiple paths in the format expected by ‘nix-store + --import’. */ +void exportPaths(StoreAPI & store, const Paths & paths, + bool sign, Sink & sink); + + +MakeError(SubstError, Error) +MakeError(BuildError, Error) /* denotes a permanent build failure */ + + +} diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh new file mode 100644 index 000000000000..9317f89c3763 --- /dev/null +++ b/src/libstore/worker-protocol.hh @@ -0,0 +1,60 @@ +#pragma once + +namespace nix { + + +#define WORKER_MAGIC_1 0x6e697863 +#define WORKER_MAGIC_2 0x6478696f + +#define PROTOCOL_VERSION 0x10e +#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) +#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) + + +typedef enum { + wopQuit = 0, + wopIsValidPath = 1, + wopHasSubstitutes = 3, + wopQueryPathHash = 4, + wopQueryReferences = 5, + wopQueryReferrers = 6, + wopAddToStore = 7, + wopAddTextToStore = 8, + wopBuildPaths = 9, + wopEnsurePath = 10, + wopAddTempRoot = 11, + wopAddIndirectRoot = 12, + wopSyncWithGC = 13, + wopFindRoots = 14, + wopExportPath = 16, + wopQueryDeriver = 18, + wopSetOptions = 19, + wopCollectGarbage = 20, + wopQuerySubstitutablePathInfo = 21, + wopQueryDerivationOutputs = 22, + wopQueryAllValidPaths = 23, + wopQueryFailedPaths = 24, + wopClearFailedPaths = 25, + wopQueryPathInfo = 26, + wopImportPaths = 27, + wopQueryDerivationOutputNames = 28, + wopQueryPathFromHashPart = 29, + wopQuerySubstitutablePathInfos = 30, + wopQueryValidPaths = 31, + wopQuerySubstitutablePaths = 32, + wopQueryValidDerivers = 33, +} WorkerOp; + + +#define STDERR_NEXT 0x6f6c6d67 +#define STDERR_READ 0x64617461 // data needed from source +#define STDERR_WRITE 0x64617416 // data for sink +#define STDERR_LAST 0x616c7473 +#define STDERR_ERROR 0x63787470 + + +Path readStorePath(Source & from); +template<class T> T readStorePaths(Source & from); + + +} diff --git a/src/libutil/affinity.cc b/src/libutil/affinity.cc new file mode 100644 index 000000000000..3e21f43a2e9d --- /dev/null +++ b/src/libutil/affinity.cc @@ -0,0 +1,55 @@ +#include "types.hh" +#include "util.hh" +#include "affinity.hh" + +#if HAVE_SCHED_H +#include <sched.h> +#endif + +namespace nix { + + +#if HAVE_SCHED_SETAFFINITY +static bool didSaveAffinity = false; +static cpu_set_t savedAffinity; +#endif + + +void setAffinityTo(int cpu) +{ +#if HAVE_SCHED_SETAFFINITY + if (sched_getaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1) return; + didSaveAffinity = true; + printMsg(lvlDebug, format("locking this thread to CPU %1%") % cpu); + cpu_set_t newAffinity; + CPU_ZERO(&newAffinity); + CPU_SET(cpu, &newAffinity); + if (sched_setaffinity(0, sizeof(cpu_set_t), &newAffinity) == -1) + printMsg(lvlError, format("failed to lock thread to CPU %1%") % cpu); +#endif +} + + +int lockToCurrentCPU() +{ +#if HAVE_SCHED_SETAFFINITY + int cpu = sched_getcpu(); + if (cpu != -1) setAffinityTo(cpu); + return cpu; +#else + return -1; +#endif +} + + +void restoreAffinity() +{ +#if HAVE_SCHED_SETAFFINITY + if (!didSaveAffinity) return; + if (sched_setaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1) + printMsg(lvlError, "failed to restore affinity %1%"); +#endif +} + + +} diff --git a/src/libutil/affinity.hh b/src/libutil/affinity.hh new file mode 100644 index 000000000000..c1bd28e1367a --- /dev/null +++ b/src/libutil/affinity.hh @@ -0,0 +1,9 @@ +#pragma once + +namespace nix { + +void setAffinityTo(int cpu); +int lockToCurrentCPU(); +void restoreAffinity(); + +} diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc new file mode 100644 index 000000000000..b19ff4bf99a1 --- /dev/null +++ b/src/libutil/archive.cc @@ -0,0 +1,336 @@ +#include "config.h" + +#include <cerrno> +#include <algorithm> +#include <vector> + +#define _XOPEN_SOURCE 600 +#include <sys/types.h> +#include <sys/stat.h> +#include <unistd.h> +#include <dirent.h> +#include <fcntl.h> + +#include "archive.hh" +#include "util.hh" + + +namespace nix { + + +static string archiveVersion1 = "nix-archive-1"; + + +PathFilter defaultPathFilter; + + +static void dump(const string & path, Sink & sink, PathFilter & filter); + + +static void dumpEntries(const Path & path, Sink & sink, PathFilter & filter) +{ + Strings names = readDirectory(path); + vector<string> names2(names.begin(), names.end()); + sort(names2.begin(), names2.end()); + + for (vector<string>::iterator i = names2.begin(); + i != names2.end(); ++i) + { + Path entry = path + "/" + *i; + if (filter(entry)) { + writeString("entry", sink); + writeString("(", sink); + writeString("name", sink); + writeString(*i, sink); + writeString("node", sink); + dump(entry, sink, filter); + writeString(")", sink); + } + } +} + + +static void dumpContents(const Path & path, size_t size, + Sink & sink) +{ + writeString("contents", sink); + writeLongLong(size, sink); + + AutoCloseFD fd = open(path.c_str(), O_RDONLY); + if (fd == -1) throw SysError(format("opening file `%1%'") % path); + + unsigned char buf[65536]; + size_t left = size; + + while (left > 0) { + size_t n = left > sizeof(buf) ? sizeof(buf) : left; + readFull(fd, buf, n); + left -= n; + sink(buf, n); + } + + writePadding(size, sink); +} + + +static void dump(const Path & path, Sink & sink, PathFilter & filter) +{ + struct stat st; + if (lstat(path.c_str(), &st)) + throw SysError(format("getting attributes of path `%1%'") % path); + + writeString("(", sink); + + if (S_ISREG(st.st_mode)) { + writeString("type", sink); + writeString("regular", sink); + if (st.st_mode & S_IXUSR) { + writeString("executable", sink); + writeString("", sink); + } + dumpContents(path, (size_t) st.st_size, sink); + } + + else if (S_ISDIR(st.st_mode)) { + writeString("type", sink); + writeString("directory", sink); + dumpEntries(path, sink, filter); + } + + else if (S_ISLNK(st.st_mode)) { + writeString("type", sink); + writeString("symlink", sink); + writeString("target", sink); + writeString(readLink(path), sink); + } + + else throw Error(format("file `%1%' has an unknown type") % path); + + writeString(")", sink); +} + + +void dumpPath(const Path & path, Sink & sink, PathFilter & filter) +{ + writeString(archiveVersion1, sink); + dump(path, sink, filter); +} + + +static SerialisationError badArchive(string s) +{ + return SerialisationError("bad archive: " + s); +} + + +static void skipGeneric(Source & source) +{ + if (readString(source) == "(") { + while (readString(source) != ")") + skipGeneric(source); + } +} + + +static void parse(ParseSink & sink, Source & source, const Path & path); + + + +static void parseEntry(ParseSink & sink, Source & source, const Path & path) +{ + string s, name; + + s = readString(source); + if (s != "(") throw badArchive("expected open tag"); + + while (1) { + checkInterrupt(); + + s = readString(source); + + if (s == ")") { + break; + } else if (s == "name") { + name = readString(source); + } else if (s == "node") { + if (s == "") throw badArchive("entry name missing"); + parse(sink, source, path + "/" + name); + } else { + throw badArchive("unknown field " + s); + skipGeneric(source); + } + } +} + + +static void parseContents(ParseSink & sink, Source & source, const Path & path) +{ + unsigned long long size = readLongLong(source); + + sink.preallocateContents(size); + + unsigned long long left = size; + unsigned char buf[65536]; + + while (left) { + checkInterrupt(); + unsigned int n = sizeof(buf); + if ((unsigned long long) n > left) n = left; + source(buf, n); + sink.receiveContents(buf, n); + left -= n; + } + + readPadding(size, source); +} + + +static void parse(ParseSink & sink, Source & source, const Path & path) +{ + string s; + + s = readString(source); + if (s != "(") throw badArchive("expected open tag"); + + enum { tpUnknown, tpRegular, tpDirectory, tpSymlink } type = tpUnknown; + + while (1) { + checkInterrupt(); + + s = readString(source); + + if (s == ")") { + break; + } + + else if (s == "type") { + if (type != tpUnknown) + throw badArchive("multiple type fields"); + string t = readString(source); + + if (t == "regular") { + type = tpRegular; + sink.createRegularFile(path); + } + + else if (t == "directory") { + sink.createDirectory(path); + type = tpDirectory; + } + + else if (t == "symlink") { + type = tpSymlink; + } + + else throw badArchive("unknown file type " + t); + + } + + else if (s == "contents" && type == tpRegular) { + parseContents(sink, source, path); + } + + else if (s == "executable" && type == tpRegular) { + readString(source); + sink.isExecutable(); + } + + else if (s == "entry" && type == tpDirectory) { + parseEntry(sink, source, path); + } + + else if (s == "target" && type == tpSymlink) { + string target = readString(source); + sink.createSymlink(path, target); + } + + else { + throw badArchive("unknown field " + s); + skipGeneric(source); + } + } +} + + +void parseDump(ParseSink & sink, Source & source) +{ + string version; + try { + version = readString(source); + } catch (SerialisationError & e) { + /* This generally means the integer at the start couldn't be + decoded. Ignore and throw the exception below. */ + } + if (version != archiveVersion1) + throw badArchive("input doesn't look like a Nix archive"); + parse(sink, source, ""); +} + + +struct RestoreSink : ParseSink +{ + Path dstPath; + AutoCloseFD fd; + + void createDirectory(const Path & path) + { + Path p = dstPath + path; + if (mkdir(p.c_str(), 0777) == -1) + throw SysError(format("creating directory `%1%'") % p); + }; + + void createRegularFile(const Path & path) + { + Path p = dstPath + path; + fd.close(); + fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY, 0666); + if (fd == -1) throw SysError(format("creating file `%1%'") % p); + } + + void isExecutable() + { + struct stat st; + if (fstat(fd, &st) == -1) + throw SysError("fstat"); + if (fchmod(fd, st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1) + throw SysError("fchmod"); + } + + void preallocateContents(unsigned long long len) + { +#if HAVE_POSIX_FALLOCATE + if (len) { + errno = posix_fallocate(fd, 0, len); + /* Note that EINVAL may indicate that the underlying + filesystem doesn't support preallocation (e.g. on + OpenSolaris). Since preallocation is just an + optimisation, ignore it. */ + if (errno && errno != EINVAL) + throw SysError(format("preallocating file of %1% bytes") % len); + } +#endif + } + + void receiveContents(unsigned char * data, unsigned int len) + { + writeFull(fd, data, len); + } + + void createSymlink(const Path & path, const string & target) + { + Path p = dstPath + path; + if (symlink(target.c_str(), p.c_str()) == -1) + throw SysError(format("creating symlink `%1%'") % p); + } +}; + + +void restorePath(const Path & path, Source & source) +{ + RestoreSink sink; + sink.dstPath = path; + parseDump(sink, source); +} + + +} diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh new file mode 100644 index 000000000000..ccac92074d54 --- /dev/null +++ b/src/libutil/archive.hh @@ -0,0 +1,75 @@ +#pragma once + +#include "types.hh" +#include "serialise.hh" + + +namespace nix { + + +/* dumpPath creates a Nix archive of the specified path. The format + is as follows: + + IF path points to a REGULAR FILE: + dump(path) = attrs( + [ ("type", "regular") + , ("contents", contents(path)) + ]) + + IF path points to a DIRECTORY: + dump(path) = attrs( + [ ("type", "directory") + , ("entries", concat(map(f, sort(entries(path))))) + ]) + where f(fn) = attrs( + [ ("name", fn) + , ("file", dump(path + "/" + fn)) + ]) + + where: + + attrs(as) = concat(map(attr, as)) + encN(0) + attrs((a, b)) = encS(a) + encS(b) + + encS(s) = encN(len(s)) + s + (padding until next 64-bit boundary) + + encN(n) = 64-bit little-endian encoding of n. + + contents(path) = the contents of a regular file. + + sort(strings) = lexicographic sort by 8-bit value (strcmp). + + entries(path) = the entries of a directory, without `.' and + `..'. + + `+' denotes string concatenation. */ + +struct PathFilter +{ + virtual ~PathFilter() { } + virtual bool operator () (const Path & path) { return true; } +}; + +extern PathFilter defaultPathFilter; + +void dumpPath(const Path & path, Sink & sink, + PathFilter & filter = defaultPathFilter); + +struct ParseSink +{ + virtual void createDirectory(const Path & path) { }; + + virtual void createRegularFile(const Path & path) { }; + virtual void isExecutable() { }; + virtual void preallocateContents(unsigned long long size) { }; + virtual void receiveContents(unsigned char * data, unsigned int len) { }; + + virtual void createSymlink(const Path & path, const string & target) { }; +}; + +void parseDump(ParseSink & sink, Source & source); + +void restorePath(const Path & path, Source & source); + + +} diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc new file mode 100644 index 000000000000..de2c1ebd724f --- /dev/null +++ b/src/libutil/hash.cc @@ -0,0 +1,382 @@ +#include "config.h" + +#include <iostream> +#include <cstring> + +#ifdef HAVE_OPENSSL +#include <openssl/md5.h> +#include <openssl/sha.h> +#else +extern "C" { +#include "md5.h" +#include "sha1.h" +#include "sha256.h" +} +#endif + +#include "hash.hh" +#include "archive.hh" +#include "util.hh" + +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> + + +namespace nix { + + +Hash::Hash() +{ + type = htUnknown; + hashSize = 0; + memset(hash, 0, maxHashSize); +} + + +Hash::Hash(HashType type) +{ + this->type = type; + if (type == htMD5) hashSize = md5HashSize; + else if (type == htSHA1) hashSize = sha1HashSize; + else if (type == htSHA256) hashSize = sha256HashSize; + else throw Error("unknown hash type"); + assert(hashSize <= maxHashSize); + memset(hash, 0, maxHashSize); +} + + +bool Hash::operator == (const Hash & h2) const +{ + if (hashSize != h2.hashSize) return false; + for (unsigned int i = 0; i < hashSize; i++) + if (hash[i] != h2.hash[i]) return false; + return true; +} + + +bool Hash::operator != (const Hash & h2) const +{ + return !(*this == h2); +} + + +bool Hash::operator < (const Hash & h) const +{ + for (unsigned int i = 0; i < hashSize; i++) { + if (hash[i] < h.hash[i]) return true; + if (hash[i] > h.hash[i]) return false; + } + return false; +} + + +const string base16Chars = "0123456789abcdef"; + + +string printHash(const Hash & hash) +{ + char buf[hash.hashSize * 2]; + for (unsigned int i = 0; i < hash.hashSize; i++) { + buf[i * 2] = base16Chars[hash.hash[i] >> 4]; + buf[i * 2 + 1] = base16Chars[hash.hash[i] & 0x0f]; + } + return string(buf, hash.hashSize * 2); +} + + +Hash parseHash(HashType ht, const string & s) +{ + Hash hash(ht); + if (s.length() != hash.hashSize * 2) + throw Error(format("invalid hash `%1%'") % s); + for (unsigned int i = 0; i < hash.hashSize; i++) { + string s2(s, i * 2, 2); + if (!isxdigit(s2[0]) || !isxdigit(s2[1])) + throw Error(format("invalid hash `%1%'") % s); + std::istringstream str(s2); + int n; + str >> std::hex >> n; + hash.hash[i] = n; + } + return hash; +} + + +static unsigned char divMod(unsigned char * bytes, unsigned char y) +{ + unsigned int borrow = 0; + + int pos = Hash::maxHashSize - 1; + while (pos >= 0 && !bytes[pos]) --pos; + + for ( ; pos >= 0; --pos) { + unsigned int s = bytes[pos] + (borrow << 8); + unsigned int d = s / y; + borrow = s % y; + bytes[pos] = d; + } + + return borrow; +} + + +unsigned int hashLength32(const Hash & hash) +{ + return (hash.hashSize * 8 - 1) / 5 + 1; +} + + +// omitted: E O U T +const string base32Chars = "0123456789abcdfghijklmnpqrsvwxyz"; + + +string printHash32(const Hash & hash) +{ + Hash hash2(hash); + unsigned int len = hashLength32(hash); + + const char * chars = base32Chars.data(); + + string s(len, '0'); + + int pos = len - 1; + while (pos >= 0) { + unsigned char digit = divMod(hash2.hash, 32); + s[pos--] = chars[digit]; + } + + for (unsigned int i = 0; i < hash2.maxHashSize; ++i) + assert(hash2.hash[i] == 0); + + return s; +} + + +string printHash16or32(const Hash & hash) +{ + return hash.type == htMD5 ? printHash(hash) : printHash32(hash); +} + + +static bool mul(unsigned char * bytes, unsigned char y, int maxSize) +{ + unsigned char carry = 0; + + for (int pos = 0; pos < maxSize; ++pos) { + unsigned int m = bytes[pos] * y + carry; + bytes[pos] = m & 0xff; + carry = m >> 8; + } + + return carry; +} + + +static bool add(unsigned char * bytes, unsigned char y, int maxSize) +{ + unsigned char carry = y; + + for (int pos = 0; pos < maxSize; ++pos) { + unsigned int m = bytes[pos] + carry; + bytes[pos] = m & 0xff; + carry = m >> 8; + if (carry == 0) break; + } + + return carry; +} + + +Hash parseHash32(HashType ht, const string & s) +{ + Hash hash(ht); + + const char * chars = base32Chars.data(); + + for (unsigned int i = 0; i < s.length(); ++i) { + char c = s[i]; + unsigned char digit; + for (digit = 0; digit < base32Chars.size(); ++digit) /* !!! slow */ + if (chars[digit] == c) break; + if (digit >= 32) + throw Error(format("invalid base-32 hash `%1%'") % s); + if (mul(hash.hash, 32, hash.hashSize) || + add(hash.hash, digit, hash.hashSize)) + throw Error(format("base-32 hash `%1%' is too large") % s); + } + + return hash; +} + + +Hash parseHash16or32(HashType ht, const string & s) +{ + Hash hash(ht); + if (s.size() == hash.hashSize * 2) + /* hexadecimal representation */ + hash = parseHash(ht, s); + else if (s.size() == hashLength32(hash)) + /* base-32 representation */ + hash = parseHash32(ht, s); + else + throw Error(format("hash `%1%' has wrong length for hash type `%2%'") + % s % printHashType(ht)); + return hash; +} + + +bool isHash(const string & s) +{ + if (s.length() != 32) return false; + for (int i = 0; i < 32; i++) { + char c = s[i]; + if (!((c >= '0' && c <= '9') || + (c >= 'a' && c <= 'f'))) + return false; + } + return true; +} + + +union Ctx +{ + MD5_CTX md5; + SHA_CTX sha1; + SHA256_CTX sha256; +}; + + +static void start(HashType ht, Ctx & ctx) +{ + if (ht == htMD5) MD5_Init(&ctx.md5); + else if (ht == htSHA1) SHA1_Init(&ctx.sha1); + else if (ht == htSHA256) SHA256_Init(&ctx.sha256); +} + + +static void update(HashType ht, Ctx & ctx, + const unsigned char * bytes, unsigned int len) +{ + if (ht == htMD5) MD5_Update(&ctx.md5, bytes, len); + else if (ht == htSHA1) SHA1_Update(&ctx.sha1, bytes, len); + else if (ht == htSHA256) SHA256_Update(&ctx.sha256, bytes, len); +} + + +static void finish(HashType ht, Ctx & ctx, unsigned char * hash) +{ + if (ht == htMD5) MD5_Final(hash, &ctx.md5); + else if (ht == htSHA1) SHA1_Final(hash, &ctx.sha1); + else if (ht == htSHA256) SHA256_Final(hash, &ctx.sha256); +} + + +Hash hashString(HashType ht, const string & s) +{ + Ctx ctx; + Hash hash(ht); + start(ht, ctx); + update(ht, ctx, (const unsigned char *) s.data(), s.length()); + finish(ht, ctx, hash.hash); + return hash; +} + + +Hash hashFile(HashType ht, const Path & path) +{ + Ctx ctx; + Hash hash(ht); + start(ht, ctx); + + AutoCloseFD fd = open(path.c_str(), O_RDONLY); + if (fd == -1) throw SysError(format("opening file `%1%'") % path); + + unsigned char buf[8192]; + ssize_t n; + while ((n = read(fd, buf, sizeof(buf)))) { + checkInterrupt(); + if (n == -1) throw SysError(format("reading file `%1%'") % path); + update(ht, ctx, buf, n); + } + + finish(ht, ctx, hash.hash); + return hash; +} + + +HashSink::HashSink(HashType ht) : ht(ht) +{ + ctx = new Ctx; + bytes = 0; + start(ht, *ctx); +} + +HashSink::~HashSink() +{ + bufPos = 0; + delete ctx; +} + +void HashSink::write(const unsigned char * data, size_t len) +{ + bytes += len; + update(ht, *ctx, data, len); +} + +HashResult HashSink::finish() +{ + flush(); + Hash hash(ht); + nix::finish(ht, *ctx, hash.hash); + return HashResult(hash, bytes); +} + +HashResult HashSink::currentHash() +{ + flush(); + Ctx ctx2 = *ctx; + Hash hash(ht); + nix::finish(ht, ctx2, hash.hash); + return HashResult(hash, bytes); +} + + +HashResult hashPath( + HashType ht, const Path & path, PathFilter & filter) +{ + HashSink sink(ht); + dumpPath(path, sink, filter); + return sink.finish(); +} + + +Hash compressHash(const Hash & hash, unsigned int newSize) +{ + Hash h; + h.hashSize = newSize; + for (unsigned int i = 0; i < hash.hashSize; ++i) + h.hash[i % newSize] ^= hash.hash[i]; + return h; +} + + +HashType parseHashType(const string & s) +{ + if (s == "md5") return htMD5; + else if (s == "sha1") return htSHA1; + else if (s == "sha256") return htSHA256; + else return htUnknown; +} + + +string printHashType(HashType ht) +{ + if (ht == htMD5) return "md5"; + else if (ht == htSHA1) return "sha1"; + else if (ht == htSHA256) return "sha256"; + else throw Error("cannot print unknown hash type"); +} + + +} diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh new file mode 100644 index 000000000000..2c6f176ec74c --- /dev/null +++ b/src/libutil/hash.hh @@ -0,0 +1,113 @@ +#pragma once + +#include "types.hh" +#include "serialise.hh" + + +namespace nix { + + +typedef enum { htUnknown, htMD5, htSHA1, htSHA256 } HashType; + + +const int md5HashSize = 16; +const int sha1HashSize = 20; +const int sha256HashSize = 32; + +extern const string base32Chars; + + +struct Hash +{ + static const unsigned int maxHashSize = 32; + unsigned int hashSize; + unsigned char hash[maxHashSize]; + + HashType type; + + /* Create an unusable hash object. */ + Hash(); + + /* Create a zero-filled hash object. */ + Hash(HashType type); + + /* Check whether two hash are equal. */ + bool operator == (const Hash & h2) const; + + /* Check whether two hash are not equal. */ + bool operator != (const Hash & h2) const; + + /* For sorting. */ + bool operator < (const Hash & h) const; +}; + + +/* Convert a hash to a hexadecimal representation. */ +string printHash(const Hash & hash); + +/* Parse a hexadecimal representation of a hash code. */ +Hash parseHash(HashType ht, const string & s); + +/* Returns the length of a base-32 hash representation. */ +unsigned int hashLength32(const Hash & hash); + +/* Convert a hash to a base-32 representation. */ +string printHash32(const Hash & hash); + +/* Print a hash in base-16 if it's MD5, or base-32 otherwise. */ +string printHash16or32(const Hash & hash); + +/* Parse a base-32 representation of a hash code. */ +Hash parseHash32(HashType ht, const string & s); + +/* Parse a base-16 or base-32 representation of a hash code. */ +Hash parseHash16or32(HashType ht, const string & s); + +/* Verify that the given string is a valid hash code. */ +bool isHash(const string & s); + +/* Compute the hash of the given string. */ +Hash hashString(HashType ht, const string & s); + +/* Compute the hash of the given file. */ +Hash hashFile(HashType ht, const Path & path); + +/* Compute the hash of the given path. The hash is defined as + (essentially) hashString(ht, dumpPath(path)). */ +struct PathFilter; +extern PathFilter defaultPathFilter; +typedef std::pair<Hash, unsigned long long> HashResult; +HashResult hashPath(HashType ht, const Path & path, + PathFilter & filter = defaultPathFilter); + +/* Compress a hash to the specified number of bytes by cyclically + XORing bytes together. */ +Hash compressHash(const Hash & hash, unsigned int newSize); + +/* Parse a string representing a hash type. */ +HashType parseHashType(const string & s); + +/* And the reverse. */ +string printHashType(HashType ht); + + +union Ctx; + +class HashSink : public BufferedSink +{ +private: + HashType ht; + Ctx * ctx; + unsigned long long bytes; + +public: + HashSink(HashType ht); + HashSink(const HashSink & h); + ~HashSink(); + void write(const unsigned char * data, size_t len); + HashResult finish(); + HashResult currentHash(); +}; + + +} diff --git a/src/libutil/local.mk b/src/libutil/local.mk new file mode 100644 index 000000000000..8af2e78d9ce4 --- /dev/null +++ b/src/libutil/local.mk @@ -0,0 +1,15 @@ +libraries += libutil + +libutil_NAME = libnixutil + +libutil_DIR := $(d) + +libutil_SOURCES := $(wildcard $(d)/*.cc) + +ifeq ($(HAVE_OPENSSL), 1) + libutil_LDFLAGS = $(OPENSSL_LIBS) +else + libutil_SOURCES += $(d)/md5.c $(d)/sha1.c $(d)/sha256.c +endif + +libutil_LIBS = libformat diff --git a/src/libutil/md32_common.h b/src/libutil/md32_common.h new file mode 100644 index 000000000000..0cbcfaf8a20b --- /dev/null +++ b/src/libutil/md32_common.h @@ -0,0 +1,620 @@ +/* crypto/md32_common.h */ +/* ==================================================================== + * Copyright (c) 1999-2002 The OpenSSL Project. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * 3. All advertising materials mentioning features or use of this + * software must display the following acknowledgment: + * "This product includes software developed by the OpenSSL Project + * for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)" + * + * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to + * endorse or promote products derived from this software without + * prior written permission. For written permission, please contact + * licensing@OpenSSL.org. + * + * 5. Products derived from this software may not be called "OpenSSL" + * nor may "OpenSSL" appear in their names without prior written + * permission of the OpenSSL Project. + * + * 6. Redistributions of any form whatsoever must retain the following + * acknowledgment: + * "This product includes software developed by the OpenSSL Project + * for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)" + * + * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY + * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR + * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + * ==================================================================== + * + * This product includes cryptographic software written by Eric Young + * (eay@cryptsoft.com). This product includes software written by Tim + * Hudson (tjh@cryptsoft.com). + * + */ + +/* + * This is a generic 32 bit "collector" for message digest algorithms. + * Whenever needed it collects input character stream into chunks of + * 32 bit values and invokes a block function that performs actual hash + * calculations. + * + * Porting guide. + * + * Obligatory macros: + * + * DATA_ORDER_IS_BIG_ENDIAN or DATA_ORDER_IS_LITTLE_ENDIAN + * this macro defines byte order of input stream. + * HASH_CBLOCK + * size of a unit chunk HASH_BLOCK operates on. + * HASH_LONG + * has to be at lest 32 bit wide, if it's wider, then + * HASH_LONG_LOG2 *has to* be defined along + * HASH_CTX + * context structure that at least contains following + * members: + * typedef struct { + * ... + * HASH_LONG Nl,Nh; + * HASH_LONG data[HASH_LBLOCK]; + * unsigned int num; + * ... + * } HASH_CTX; + * HASH_UPDATE + * name of "Update" function, implemented here. + * HASH_TRANSFORM + * name of "Transform" function, implemented here. + * HASH_FINAL + * name of "Final" function, implemented here. + * HASH_BLOCK_HOST_ORDER + * name of "block" function treating *aligned* input message + * in host byte order, implemented externally. + * HASH_BLOCK_DATA_ORDER + * name of "block" function treating *unaligned* input message + * in original (data) byte order, implemented externally (it + * actually is optional if data and host are of the same + * "endianess"). + * HASH_MAKE_STRING + * macro convering context variables to an ASCII hash string. + * + * Optional macros: + * + * B_ENDIAN or L_ENDIAN + * defines host byte-order. + * HASH_LONG_LOG2 + * defaults to 2 if not states otherwise. + * HASH_LBLOCK + * assumed to be HASH_CBLOCK/4 if not stated otherwise. + * HASH_BLOCK_DATA_ORDER_ALIGNED + * alternative "block" function capable of treating + * aligned input message in original (data) order, + * implemented externally. + * + * MD5 example: + * + * #define DATA_ORDER_IS_LITTLE_ENDIAN + * + * #define HASH_LONG MD5_LONG + * #define HASH_LONG_LOG2 MD5_LONG_LOG2 + * #define HASH_CTX MD5_CTX + * #define HASH_CBLOCK MD5_CBLOCK + * #define HASH_LBLOCK MD5_LBLOCK + * #define HASH_UPDATE MD5_Update + * #define HASH_TRANSFORM MD5_Transform + * #define HASH_FINAL MD5_Final + * #define HASH_BLOCK_HOST_ORDER md5_block_host_order + * #define HASH_BLOCK_DATA_ORDER md5_block_data_order + * + * <appro@fy.chalmers.se> + */ + +#if !defined(DATA_ORDER_IS_BIG_ENDIAN) && !defined(DATA_ORDER_IS_LITTLE_ENDIAN) +#error "DATA_ORDER must be defined!" +#endif + +#ifndef HASH_CBLOCK +#error "HASH_CBLOCK must be defined!" +#endif +#ifndef HASH_LONG +#error "HASH_LONG must be defined!" +#endif +#ifndef HASH_CTX +#error "HASH_CTX must be defined!" +#endif + +#ifndef HASH_UPDATE +#error "HASH_UPDATE must be defined!" +#endif +#ifndef HASH_TRANSFORM +#error "HASH_TRANSFORM must be defined!" +#endif +#ifndef HASH_FINAL +#error "HASH_FINAL must be defined!" +#endif + +#ifndef HASH_BLOCK_HOST_ORDER +#error "HASH_BLOCK_HOST_ORDER must be defined!" +#endif + +#if 0 +/* + * Moved below as it's required only if HASH_BLOCK_DATA_ORDER_ALIGNED + * isn't defined. + */ +#ifndef HASH_BLOCK_DATA_ORDER +#error "HASH_BLOCK_DATA_ORDER must be defined!" +#endif +#endif + +#ifndef HASH_LBLOCK +#define HASH_LBLOCK (HASH_CBLOCK/4) +#endif + +#ifndef HASH_LONG_LOG2 +#define HASH_LONG_LOG2 2 +#endif + +/* + * Engage compiler specific rotate intrinsic function if available. + */ +#undef ROTATE +#ifndef PEDANTIC +# if defined(_MSC_VER) || defined(__ICC) +# define ROTATE(a,n) _lrotl(a,n) +# elif defined(__MWERKS__) +# if defined(__POWERPC__) +# define ROTATE(a,n) __rlwinm(a,n,0,31) +# elif defined(__MC68K__) + /* Motorola specific tweak. <appro@fy.chalmers.se> */ +# define ROTATE(a,n) ( n<24 ? __rol(a,n) : __ror(a,32-n) ) +# else +# define ROTATE(a,n) __rol(a,n) +# endif +# elif defined(__GNUC__) && __GNUC__>=2 && !defined(OPENSSL_NO_ASM) && !defined(OPENSSL_NO_INLINE_ASM) + /* + * Some GNU C inline assembler templates. Note that these are + * rotates by *constant* number of bits! But that's exactly + * what we need here... + * <appro@fy.chalmers.se> + */ +# if defined(__i386) || defined(__i386__) || defined(__x86_64) || defined(__x86_64__) +# define ROTATE(a,n) ({ register unsigned int ret; \ + asm ( \ + "roll %1,%0" \ + : "=r"(ret) \ + : "I"(n), "0"(a) \ + : "cc"); \ + ret; \ + }) +# elif defined(__powerpc) || defined(__ppc__) || defined(__powerpc64__) +# define ROTATE(a,n) ({ register unsigned int ret; \ + asm ( \ + "rlwinm %0,%1,%2,0,31" \ + : "=r"(ret) \ + : "r"(a), "I"(n)); \ + ret; \ + }) +# endif +# endif +#endif /* PEDANTIC */ + +#if HASH_LONG_LOG2==2 /* Engage only if sizeof(HASH_LONG)== 4 */ +/* A nice byte order reversal from Wei Dai <weidai@eskimo.com> */ +#ifdef ROTATE +/* 5 instructions with rotate instruction, else 9 */ +#define REVERSE_FETCH32(a,l) ( \ + l=*(const HASH_LONG *)(a), \ + ((ROTATE(l,8)&0x00FF00FF)|(ROTATE((l&0x00FF00FF),24))) \ + ) +#else +/* 6 instructions with rotate instruction, else 8 */ +#define REVERSE_FETCH32(a,l) ( \ + l=*(const HASH_LONG *)(a), \ + l=(((l>>8)&0x00FF00FF)|((l&0x00FF00FF)<<8)), \ + ROTATE(l,16) \ + ) +/* + * Originally the middle line started with l=(((l&0xFF00FF00)>>8)|... + * It's rewritten as above for two reasons: + * - RISCs aren't good at long constants and have to explicitely + * compose 'em with several (well, usually 2) instructions in a + * register before performing the actual operation and (as you + * already realized:-) having same constant should inspire the + * compiler to permanently allocate the only register for it; + * - most modern CPUs have two ALUs, but usually only one has + * circuitry for shifts:-( this minor tweak inspires compiler + * to schedule shift instructions in a better way... + * + * <appro@fy.chalmers.se> + */ +#endif +#endif + +#ifndef ROTATE +#define ROTATE(a,n) (((a)<<(n))|(((a)&0xffffffff)>>(32-(n)))) +#endif + +/* + * Make some obvious choices. E.g., HASH_BLOCK_DATA_ORDER_ALIGNED + * and HASH_BLOCK_HOST_ORDER ought to be the same if input data + * and host are of the same "endianess". It's possible to mask + * this with blank #define HASH_BLOCK_DATA_ORDER though... + * + * <appro@fy.chalmers.se> + */ +#if defined(B_ENDIAN) +# if defined(DATA_ORDER_IS_BIG_ENDIAN) +# if !defined(HASH_BLOCK_DATA_ORDER_ALIGNED) && HASH_LONG_LOG2==2 +# define HASH_BLOCK_DATA_ORDER_ALIGNED HASH_BLOCK_HOST_ORDER +# endif +# endif +#elif defined(L_ENDIAN) +# if defined(DATA_ORDER_IS_LITTLE_ENDIAN) +# if !defined(HASH_BLOCK_DATA_ORDER_ALIGNED) && HASH_LONG_LOG2==2 +# define HASH_BLOCK_DATA_ORDER_ALIGNED HASH_BLOCK_HOST_ORDER +# endif +# endif +#endif + +#if !defined(HASH_BLOCK_DATA_ORDER_ALIGNED) +#ifndef HASH_BLOCK_DATA_ORDER +#error "HASH_BLOCK_DATA_ORDER must be defined!" +#endif +#endif + +#if defined(DATA_ORDER_IS_BIG_ENDIAN) + +#ifndef PEDANTIC +# if defined(__GNUC__) && __GNUC__>=2 && !defined(OPENSSL_NO_ASM) && !defined(OPENSSL_NO_INLINE_ASM) +# if defined(__i386) || defined(__i386__) || defined(__x86_64) || defined(__x86_64__) + /* + * This gives ~30-40% performance improvement in SHA-256 compiled + * with gcc [on P4]. Well, first macro to be frank. We can pull + * this trick on x86* platforms only, because these CPUs can fetch + * unaligned data without raising an exception. + */ +# define HOST_c2l(c,l) ({ unsigned int r=*((const unsigned int *)(c)); \ + asm ("bswapl %0":"=r"(r):"0"(r)); \ + (c)+=4; (l)=r; }) +# define HOST_l2c(l,c) ({ unsigned int r=(l); \ + asm ("bswapl %0":"=r"(r):"0"(r)); \ + *((unsigned int *)(c))=r; (c)+=4; r; }) +# endif +# endif +#endif + +#ifndef HOST_c2l +#define HOST_c2l(c,l) (l =(((unsigned long)(*((c)++)))<<24), \ + l|=(((unsigned long)(*((c)++)))<<16), \ + l|=(((unsigned long)(*((c)++)))<< 8), \ + l|=(((unsigned long)(*((c)++))) ), \ + l) +#endif +#define HOST_p_c2l(c,l,n) { \ + switch (n) { \ + case 0: l =((unsigned long)(*((c)++)))<<24; \ + case 1: l|=((unsigned long)(*((c)++)))<<16; \ + case 2: l|=((unsigned long)(*((c)++)))<< 8; \ + case 3: l|=((unsigned long)(*((c)++))); \ + } } +#define HOST_p_c2l_p(c,l,sc,len) { \ + switch (sc) { \ + case 0: l =((unsigned long)(*((c)++)))<<24; \ + if (--len == 0) break; \ + case 1: l|=((unsigned long)(*((c)++)))<<16; \ + if (--len == 0) break; \ + case 2: l|=((unsigned long)(*((c)++)))<< 8; \ + } } +/* NOTE the pointer is not incremented at the end of this */ +#define HOST_c2l_p(c,l,n) { \ + l=0; (c)+=n; \ + switch (n) { \ + case 3: l =((unsigned long)(*(--(c))))<< 8; \ + case 2: l|=((unsigned long)(*(--(c))))<<16; \ + case 1: l|=((unsigned long)(*(--(c))))<<24; \ + } } +#ifndef HOST_l2c +#define HOST_l2c(l,c) (*((c)++)=(unsigned char)(((l)>>24)&0xff), \ + *((c)++)=(unsigned char)(((l)>>16)&0xff), \ + *((c)++)=(unsigned char)(((l)>> 8)&0xff), \ + *((c)++)=(unsigned char)(((l) )&0xff), \ + l) +#endif + +#elif defined(DATA_ORDER_IS_LITTLE_ENDIAN) + +#if defined(__i386) || defined(__i386__) || defined(__x86_64) || defined(__x86_64__) + /* See comment in DATA_ORDER_IS_BIG_ENDIAN section. */ +# define HOST_c2l(c,l) ((l)=*((const unsigned int *)(c)), (c)+=4, l) +# define HOST_l2c(l,c) (*((unsigned int *)(c))=(l), (c)+=4, l) +#endif + +#ifndef HOST_c2l +#define HOST_c2l(c,l) (l =(((unsigned long)(*((c)++))) ), \ + l|=(((unsigned long)(*((c)++)))<< 8), \ + l|=(((unsigned long)(*((c)++)))<<16), \ + l|=(((unsigned long)(*((c)++)))<<24), \ + l) +#endif +#define HOST_p_c2l(c,l,n) { \ + switch (n) { \ + case 0: l =((unsigned long)(*((c)++))); \ + case 1: l|=((unsigned long)(*((c)++)))<< 8; \ + case 2: l|=((unsigned long)(*((c)++)))<<16; \ + case 3: l|=((unsigned long)(*((c)++)))<<24; \ + } } +#define HOST_p_c2l_p(c,l,sc,len) { \ + switch (sc) { \ + case 0: l =((unsigned long)(*((c)++))); \ + if (--len == 0) break; \ + case 1: l|=((unsigned long)(*((c)++)))<< 8; \ + if (--len == 0) break; \ + case 2: l|=((unsigned long)(*((c)++)))<<16; \ + } } +/* NOTE the pointer is not incremented at the end of this */ +#define HOST_c2l_p(c,l,n) { \ + l=0; (c)+=n; \ + switch (n) { \ + case 3: l =((unsigned long)(*(--(c))))<<16; \ + case 2: l|=((unsigned long)(*(--(c))))<< 8; \ + case 1: l|=((unsigned long)(*(--(c)))); \ + } } +#ifndef HOST_l2c +#define HOST_l2c(l,c) (*((c)++)=(unsigned char)(((l) )&0xff), \ + *((c)++)=(unsigned char)(((l)>> 8)&0xff), \ + *((c)++)=(unsigned char)(((l)>>16)&0xff), \ + *((c)++)=(unsigned char)(((l)>>24)&0xff), \ + l) +#endif + +#endif + +/* + * Time for some action:-) + */ + +int HASH_UPDATE (HASH_CTX *c, const void *data_, size_t len) + { + const unsigned char *data=data_; + register HASH_LONG * p; + register HASH_LONG l; + size_t sw,sc,ew,ec; + + if (len==0) return 1; + + l=(c->Nl+(((HASH_LONG)len)<<3))&0xffffffffUL; + /* 95-05-24 eay Fixed a bug with the overflow handling, thanks to + * Wei Dai <weidai@eskimo.com> for pointing it out. */ + if (l < c->Nl) /* overflow */ + c->Nh++; + c->Nh+=(len>>29); /* might cause compiler warning on 16-bit */ + c->Nl=l; + + if (c->num != 0) + { + p=c->data; + sw=c->num>>2; + sc=c->num&0x03; + + if ((c->num+len) >= HASH_CBLOCK) + { + l=p[sw]; HOST_p_c2l(data,l,sc); p[sw++]=l; + for (; sw<HASH_LBLOCK; sw++) + { + HOST_c2l(data,l); p[sw]=l; + } + HASH_BLOCK_HOST_ORDER (c,p,1); + len-=(HASH_CBLOCK-c->num); + c->num=0; + /* drop through and do the rest */ + } + else + { + c->num+=(unsigned int)len; + if ((sc+len) < 4) /* ugly, add char's to a word */ + { + l=p[sw]; HOST_p_c2l_p(data,l,sc,len); p[sw]=l; + } + else + { + ew=(c->num>>2); + ec=(c->num&0x03); + if (sc) + l=p[sw]; + HOST_p_c2l(data,l,sc); + p[sw++]=l; + for (; sw < ew; sw++) + { + HOST_c2l(data,l); p[sw]=l; + } + if (ec) + { + HOST_c2l_p(data,l,ec); p[sw]=l; + } + } + return 1; + } + } + + sw=len/HASH_CBLOCK; + if (sw > 0) + { +#if defined(HASH_BLOCK_DATA_ORDER_ALIGNED) + /* + * Note that HASH_BLOCK_DATA_ORDER_ALIGNED gets defined + * only if sizeof(HASH_LONG)==4. + */ + if ((((size_t)data)%4) == 0) + { + /* data is properly aligned so that we can cast it: */ + HASH_BLOCK_DATA_ORDER_ALIGNED (c,(const HASH_LONG *)data,sw); + sw*=HASH_CBLOCK; + data+=sw; + len-=sw; + } + else +#if !defined(HASH_BLOCK_DATA_ORDER) + while (sw--) + { + memcpy (p=c->data,data,HASH_CBLOCK); + HASH_BLOCK_DATA_ORDER_ALIGNED(c,p,1); + data+=HASH_CBLOCK; + len-=HASH_CBLOCK; + } +#endif +#endif +#if defined(HASH_BLOCK_DATA_ORDER) + { + HASH_BLOCK_DATA_ORDER(c,data,sw); + sw*=HASH_CBLOCK; + data+=sw; + len-=sw; + } +#endif + } + + if (len!=0) + { + p = c->data; + c->num = len; + ew=len>>2; /* words to copy */ + ec=len&0x03; + for (; ew; ew--,p++) + { + HOST_c2l(data,l); *p=l; + } + HOST_c2l_p(data,l,ec); + *p=l; + } + return 1; + } + + +void HASH_TRANSFORM (HASH_CTX *c, const unsigned char *data) + { +#if defined(HASH_BLOCK_DATA_ORDER_ALIGNED) + if ((((size_t)data)%4) == 0) + /* data is properly aligned so that we can cast it: */ + HASH_BLOCK_DATA_ORDER_ALIGNED (c,(const HASH_LONG *)data,1); + else +#if !defined(HASH_BLOCK_DATA_ORDER) + { + memcpy (c->data,data,HASH_CBLOCK); + HASH_BLOCK_DATA_ORDER_ALIGNED (c,c->data,1); + } +#endif +#endif +#if defined(HASH_BLOCK_DATA_ORDER) + HASH_BLOCK_DATA_ORDER (c,data,1); +#endif + } + + +int HASH_FINAL (unsigned char *md, HASH_CTX *c) + { + register HASH_LONG *p; + register unsigned long l; + register int i,j; + static const unsigned char end[4]={0x80,0x00,0x00,0x00}; + const unsigned char *cp=end; + + /* c->num should definitly have room for at least one more byte. */ + p=c->data; + i=c->num>>2; + j=c->num&0x03; + +#if 0 + /* purify often complains about the following line as an + * Uninitialized Memory Read. While this can be true, the + * following p_c2l macro will reset l when that case is true. + * This is because j&0x03 contains the number of 'valid' bytes + * already in p[i]. If and only if j&0x03 == 0, the UMR will + * occur but this is also the only time p_c2l will do + * l= *(cp++) instead of l|= *(cp++) + * Many thanks to Alex Tang <altitude@cic.net> for pickup this + * 'potential bug' */ +#ifdef PURIFY + if (j==0) p[i]=0; /* Yeah, but that's not the way to fix it:-) */ +#endif + l=p[i]; +#else + l = (j==0) ? 0 : p[i]; +#endif + HOST_p_c2l(cp,l,j); p[i++]=l; /* i is the next 'undefined word' */ + + if (i>(HASH_LBLOCK-2)) /* save room for Nl and Nh */ + { + if (i<HASH_LBLOCK) p[i]=0; + HASH_BLOCK_HOST_ORDER (c,p,1); + i=0; + } + for (; i<(HASH_LBLOCK-2); i++) + p[i]=0; + +#if defined(DATA_ORDER_IS_BIG_ENDIAN) + p[HASH_LBLOCK-2]=c->Nh; + p[HASH_LBLOCK-1]=c->Nl; +#elif defined(DATA_ORDER_IS_LITTLE_ENDIAN) + p[HASH_LBLOCK-2]=c->Nl; + p[HASH_LBLOCK-1]=c->Nh; +#endif + HASH_BLOCK_HOST_ORDER (c,p,1); + +#ifndef HASH_MAKE_STRING +#error "HASH_MAKE_STRING must be defined!" +#else + HASH_MAKE_STRING(c,md); +#endif + + c->num=0; + /* clear stuff, HASH_BLOCK may be leaving some stuff on the stack + * but I'm not worried :-) + OPENSSL_cleanse((void *)c,sizeof(HASH_CTX)); + */ + return 1; + } + +#ifndef MD32_REG_T +#define MD32_REG_T long +/* + * This comment was originaly written for MD5, which is why it + * discusses A-D. But it basically applies to all 32-bit digests, + * which is why it was moved to common header file. + * + * In case you wonder why A-D are declared as long and not + * as MD5_LONG. Doing so results in slight performance + * boost on LP64 architectures. The catch is we don't + * really care if 32 MSBs of a 64-bit register get polluted + * with eventual overflows as we *save* only 32 LSBs in + * *either* case. Now declaring 'em long excuses the compiler + * from keeping 32 MSBs zeroed resulting in 13% performance + * improvement under SPARC Solaris7/64 and 5% under AlphaLinux. + * Well, to be honest it should say that this *prevents* + * performance degradation. + * <appro@fy.chalmers.se> + * Apparently there're LP64 compilers that generate better + * code if A-D are declared int. Most notably GCC-x86_64 + * generates better code. + * <appro@fy.chalmers.se> + */ +#endif diff --git a/src/libutil/md5.c b/src/libutil/md5.c new file mode 100644 index 000000000000..b31640cdcced --- /dev/null +++ b/src/libutil/md5.c @@ -0,0 +1,365 @@ +/* Functions to compute MD5 message digest of files or memory blocks. + according to the definition of MD5 in RFC 1321 from April 1992. + Copyright (C) 1995,1996,1997,1999,2000,2001 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, write to the Free + Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA + 02111-1307 USA. */ + +/* Written by Ulrich Drepper <drepper@gnu.ai.mit.edu>, 1995. */ + +#include <sys/types.h> + +#include <stdlib.h> +#include <string.h> + +#include "md5.h" + + +static md5_uint32 SWAP(md5_uint32 n) +{ + static int checked = 0; + static int bigendian = 0; + static md5_uint32 test; + + if (!checked) { + test = 1; + if (* (char *) &test == 0) + bigendian = 1; + checked = 1; + } + + if (bigendian) + return (((n) << 24) | (((n) & 0xff00) << 8) | (((n) >> 8) & 0xff00) | ((n) >> 24)); + else + return n; +} + + +/* This array contains the bytes used to pad the buffer to the next + 64-byte boundary. (RFC 1321, 3.1: Step 1) */ +static const unsigned char fillbuf[64] = { 0x80, 0 /* , 0, 0, ... */ }; + + +/* Initialize structure containing state of computation. + (RFC 1321, 3.3: Step 3) */ +void +MD5_Init (ctx) + struct MD5_CTX *ctx; +{ + ctx->A = 0x67452301; + ctx->B = 0xefcdab89; + ctx->C = 0x98badcfe; + ctx->D = 0x10325476; + + ctx->total[0] = ctx->total[1] = 0; + ctx->buflen = 0; +} + +/* Put result from CTX in first 16 bytes following RESBUF. The result + must be in little endian byte order. + + IMPORTANT: On some systems it is required that RESBUF is correctly + aligned for a 32 bits value. */ +void * +md5_read_ctx (ctx, resbuf) + const struct MD5_CTX *ctx; + void *resbuf; +{ + ((md5_uint32 *) resbuf)[0] = SWAP (ctx->A); + ((md5_uint32 *) resbuf)[1] = SWAP (ctx->B); + ((md5_uint32 *) resbuf)[2] = SWAP (ctx->C); + ((md5_uint32 *) resbuf)[3] = SWAP (ctx->D); + + return resbuf; +} + +/* Process the remaining bytes in the internal buffer and the usual + prolog according to the standard and write the result to RESBUF. + + IMPORTANT: On some systems it is required that RESBUF is correctly + aligned for a 32 bits value. */ +void * +MD5_Final (resbuf, ctx) + void *resbuf; + struct MD5_CTX *ctx; +{ + /* Take yet unprocessed bytes into account. */ + md5_uint32 bytes = ctx->buflen; + size_t pad; + + /* Now count remaining bytes. */ + ctx->total[0] += bytes; + if (ctx->total[0] < bytes) + ++ctx->total[1]; + + pad = bytes >= 56 ? 64 + 56 - bytes : 56 - bytes; + memcpy (&ctx->buffer[bytes], fillbuf, pad); + + /* Put the 64-bit file length in *bits* at the end of the buffer. */ + *(md5_uint32 *) &ctx->buffer[bytes + pad] = SWAP (ctx->total[0] << 3); + *(md5_uint32 *) &ctx->buffer[bytes + pad + 4] = SWAP ((ctx->total[1] << 3) | + (ctx->total[0] >> 29)); + + /* Process last bytes. */ + md5_process_block (ctx->buffer, bytes + pad + 8, ctx); + + return md5_read_ctx (ctx, resbuf); +} + +void +MD5_Update (ctx, buffer, len) + struct MD5_CTX *ctx; + const void *buffer; + size_t len; +{ + /* When we already have some bits in our internal buffer concatenate + both inputs first. */ + if (ctx->buflen != 0) + { + size_t left_over = ctx->buflen; + size_t add = 128 - left_over > len ? len : 128 - left_over; + + memcpy (&ctx->buffer[left_over], buffer, add); + ctx->buflen += add; + + if (ctx->buflen > 64) + { + md5_process_block (ctx->buffer, ctx->buflen & ~63, ctx); + + ctx->buflen &= 63; + /* The regions in the following copy operation cannot overlap. */ + memcpy (ctx->buffer, &ctx->buffer[(left_over + add) & ~63], + ctx->buflen); + } + + buffer = (const char *) buffer + add; + len -= add; + } + + /* Process available complete blocks. */ + if (len >= 64) + { +#if !_STRING_ARCH_unaligned +/* To check alignment gcc has an appropriate operator. Other + compilers don't. */ +# if __GNUC__ >= 2 +# define UNALIGNED_P(p) (((md5_uintptr) p) % __alignof__ (md5_uint32) != 0) +# else +# define UNALIGNED_P(p) (((md5_uintptr) p) % sizeof (md5_uint32) != 0) +# endif + if (UNALIGNED_P (buffer)) + while (len > 64) + { + md5_process_block (memcpy (ctx->buffer, buffer, 64), 64, ctx); + buffer = (const char *) buffer + 64; + len -= 64; + } + else +#endif + { + md5_process_block (buffer, len & ~63, ctx); + buffer = (const char *) buffer + (len & ~63); + len &= 63; + } + } + + /* Move remaining bytes in internal buffer. */ + if (len > 0) + { + size_t left_over = ctx->buflen; + + memcpy (&ctx->buffer[left_over], buffer, len); + left_over += len; + if (left_over >= 64) + { + md5_process_block (ctx->buffer, 64, ctx); + left_over -= 64; + memcpy (ctx->buffer, &ctx->buffer[64], left_over); + } + ctx->buflen = left_over; + } +} + + +/* These are the four functions used in the four steps of the MD5 algorithm + and defined in the RFC 1321. The first function is a little bit optimized + (as found in Colin Plumbs public domain implementation). */ +/* #define FF(b, c, d) ((b & c) | (~b & d)) */ +#define FF(b, c, d) (d ^ (b & (c ^ d))) +#define FG(b, c, d) FF (d, b, c) +#define FH(b, c, d) (b ^ c ^ d) +#define FI(b, c, d) (c ^ (b | ~d)) + +/* Process LEN bytes of BUFFER, accumulating context into CTX. + It is assumed that LEN % 64 == 0. */ + +void +md5_process_block (buffer, len, ctx) + const void *buffer; + size_t len; + struct MD5_CTX *ctx; +{ + md5_uint32 correct_words[16]; + const md5_uint32 *words = buffer; + size_t nwords = len / sizeof (md5_uint32); + const md5_uint32 *endp = words + nwords; + md5_uint32 A = ctx->A; + md5_uint32 B = ctx->B; + md5_uint32 C = ctx->C; + md5_uint32 D = ctx->D; + + /* First increment the byte count. RFC 1321 specifies the possible + length of the file up to 2^64 bits. Here we only compute the + number of bytes. Do a double word increment. */ + ctx->total[0] += len; + if (ctx->total[0] < len) + ++ctx->total[1]; + + /* Process all bytes in the buffer with 64 bytes in each round of + the loop. */ + while (words < endp) + { + md5_uint32 *cwp = correct_words; + md5_uint32 A_save = A; + md5_uint32 B_save = B; + md5_uint32 C_save = C; + md5_uint32 D_save = D; + + /* First round: using the given function, the context and a constant + the next context is computed. Because the algorithms processing + unit is a 32-bit word and it is determined to work on words in + little endian byte order we perhaps have to change the byte order + before the computation. To reduce the work for the next steps + we store the swapped words in the array CORRECT_WORDS. */ + +#define OP(a, b, c, d, s, T) \ + do \ + { \ + a += FF (b, c, d) + (*cwp++ = SWAP (*words)) + T; \ + ++words; \ + CYCLIC (a, s); \ + a += b; \ + } \ + while (0) + + /* It is unfortunate that C does not provide an operator for + cyclic rotation. Hope the C compiler is smart enough. */ +#define CYCLIC(w, s) (w = (w << s) | (w >> (32 - s))) + + /* Before we start, one word to the strange constants. + They are defined in RFC 1321 as + + T[i] = (int) (4294967296.0 * fabs (sin (i))), i=1..64 + */ + + /* Round 1. */ + OP (A, B, C, D, 7, 0xd76aa478); + OP (D, A, B, C, 12, 0xe8c7b756); + OP (C, D, A, B, 17, 0x242070db); + OP (B, C, D, A, 22, 0xc1bdceee); + OP (A, B, C, D, 7, 0xf57c0faf); + OP (D, A, B, C, 12, 0x4787c62a); + OP (C, D, A, B, 17, 0xa8304613); + OP (B, C, D, A, 22, 0xfd469501); + OP (A, B, C, D, 7, 0x698098d8); + OP (D, A, B, C, 12, 0x8b44f7af); + OP (C, D, A, B, 17, 0xffff5bb1); + OP (B, C, D, A, 22, 0x895cd7be); + OP (A, B, C, D, 7, 0x6b901122); + OP (D, A, B, C, 12, 0xfd987193); + OP (C, D, A, B, 17, 0xa679438e); + OP (B, C, D, A, 22, 0x49b40821); + + /* For the second to fourth round we have the possibly swapped words + in CORRECT_WORDS. Redefine the macro to take an additional first + argument specifying the function to use. */ +#undef OP +#define OP(f, a, b, c, d, k, s, T) \ + do \ + { \ + a += f (b, c, d) + correct_words[k] + T; \ + CYCLIC (a, s); \ + a += b; \ + } \ + while (0) + + /* Round 2. */ + OP (FG, A, B, C, D, 1, 5, 0xf61e2562); + OP (FG, D, A, B, C, 6, 9, 0xc040b340); + OP (FG, C, D, A, B, 11, 14, 0x265e5a51); + OP (FG, B, C, D, A, 0, 20, 0xe9b6c7aa); + OP (FG, A, B, C, D, 5, 5, 0xd62f105d); + OP (FG, D, A, B, C, 10, 9, 0x02441453); + OP (FG, C, D, A, B, 15, 14, 0xd8a1e681); + OP (FG, B, C, D, A, 4, 20, 0xe7d3fbc8); + OP (FG, A, B, C, D, 9, 5, 0x21e1cde6); + OP (FG, D, A, B, C, 14, 9, 0xc33707d6); + OP (FG, C, D, A, B, 3, 14, 0xf4d50d87); + OP (FG, B, C, D, A, 8, 20, 0x455a14ed); + OP (FG, A, B, C, D, 13, 5, 0xa9e3e905); + OP (FG, D, A, B, C, 2, 9, 0xfcefa3f8); + OP (FG, C, D, A, B, 7, 14, 0x676f02d9); + OP (FG, B, C, D, A, 12, 20, 0x8d2a4c8a); + + /* Round 3. */ + OP (FH, A, B, C, D, 5, 4, 0xfffa3942); + OP (FH, D, A, B, C, 8, 11, 0x8771f681); + OP (FH, C, D, A, B, 11, 16, 0x6d9d6122); + OP (FH, B, C, D, A, 14, 23, 0xfde5380c); + OP (FH, A, B, C, D, 1, 4, 0xa4beea44); + OP (FH, D, A, B, C, 4, 11, 0x4bdecfa9); + OP (FH, C, D, A, B, 7, 16, 0xf6bb4b60); + OP (FH, B, C, D, A, 10, 23, 0xbebfbc70); + OP (FH, A, B, C, D, 13, 4, 0x289b7ec6); + OP (FH, D, A, B, C, 0, 11, 0xeaa127fa); + OP (FH, C, D, A, B, 3, 16, 0xd4ef3085); + OP (FH, B, C, D, A, 6, 23, 0x04881d05); + OP (FH, A, B, C, D, 9, 4, 0xd9d4d039); + OP (FH, D, A, B, C, 12, 11, 0xe6db99e5); + OP (FH, C, D, A, B, 15, 16, 0x1fa27cf8); + OP (FH, B, C, D, A, 2, 23, 0xc4ac5665); + + /* Round 4. */ + OP (FI, A, B, C, D, 0, 6, 0xf4292244); + OP (FI, D, A, B, C, 7, 10, 0x432aff97); + OP (FI, C, D, A, B, 14, 15, 0xab9423a7); + OP (FI, B, C, D, A, 5, 21, 0xfc93a039); + OP (FI, A, B, C, D, 12, 6, 0x655b59c3); + OP (FI, D, A, B, C, 3, 10, 0x8f0ccc92); + OP (FI, C, D, A, B, 10, 15, 0xffeff47d); + OP (FI, B, C, D, A, 1, 21, 0x85845dd1); + OP (FI, A, B, C, D, 8, 6, 0x6fa87e4f); + OP (FI, D, A, B, C, 15, 10, 0xfe2ce6e0); + OP (FI, C, D, A, B, 6, 15, 0xa3014314); + OP (FI, B, C, D, A, 13, 21, 0x4e0811a1); + OP (FI, A, B, C, D, 4, 6, 0xf7537e82); + OP (FI, D, A, B, C, 11, 10, 0xbd3af235); + OP (FI, C, D, A, B, 2, 15, 0x2ad7d2bb); + OP (FI, B, C, D, A, 9, 21, 0xeb86d391); + + /* Add the starting values of the context. */ + A += A_save; + B += B_save; + C += C_save; + D += D_save; + } + + /* Put checksum in context given as argument. */ + ctx->A = A; + ctx->B = B; + ctx->C = C; + ctx->D = D; +} diff --git a/src/libutil/md5.h b/src/libutil/md5.h new file mode 100644 index 000000000000..228d4972320f --- /dev/null +++ b/src/libutil/md5.h @@ -0,0 +1,82 @@ +/* Declaration of functions and data types used for MD5 sum computing + library functions. + Copyright (C) 1995,1996,1997,1999,2000,2001 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, write to the Free + Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA + 02111-1307 USA. */ + +#ifndef _MD5_H +#define _MD5_H 1 + +#include <inttypes.h> +typedef uint32_t md5_uint32; +typedef uintptr_t md5_uintptr; + +/* Structure to save state of computation between the single steps. */ +struct MD5_CTX +{ + md5_uint32 A; + md5_uint32 B; + md5_uint32 C; + md5_uint32 D; + + md5_uint32 total[2]; + md5_uint32 buflen; + char buffer[128] __attribute__ ((__aligned__ (__alignof__ (md5_uint32)))); +}; + +/* + * The following three functions are build up the low level used in + * the functions `md5_stream' and `md5_buffer'. + */ + +/* Initialize structure containing state of computation. + (RFC 1321, 3.3: Step 3) */ +extern void MD5_Init (struct MD5_CTX *ctx); + +/* Starting with the result of former calls of this function (or the + initialization function update the context for the next LEN bytes + starting at BUFFER. + It is necessary that LEN is a multiple of 64!!! */ +extern void md5_process_block (const void *buffer, size_t len, + struct MD5_CTX *ctx); + +/* Starting with the result of former calls of this function (or the + initialization function update the context for the next LEN bytes + starting at BUFFER. + It is NOT required that LEN is a multiple of 64. */ +extern void MD5_Update (struct MD5_CTX *ctx, const void *buffer, size_t len); + +/* Process the remaining bytes in the buffer and put result from CTX + in first 16 bytes following RESBUF. The result is always in little + endian byte order, so that a byte-wise output yields to the wanted + ASCII representation of the message digest. + + IMPORTANT: On some systems it is required that RESBUF is correctly + aligned for a 32 bits value. */ +extern void *MD5_Final (void *resbuf, struct MD5_CTX *ctx); + + +/* Put result from CTX in first 16 bytes following RESBUF. The result is + always in little endian byte order, so that a byte-wise output yields + to the wanted ASCII representation of the message digest. + + IMPORTANT: On some systems it is required that RESBUF is correctly + aligned for a 32 bits value. */ +extern void *md5_read_ctx (const struct MD5_CTX *ctx, void *resbuf); + + +#endif /* md5.h */ diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc new file mode 100644 index 000000000000..1f817a869539 --- /dev/null +++ b/src/libutil/serialise.cc @@ -0,0 +1,259 @@ +#include "serialise.hh" +#include "util.hh" + +#include <cstring> +#include <cerrno> + + +namespace nix { + + +BufferedSink::~BufferedSink() +{ + /* We can't call flush() here, because C++ for some insane reason + doesn't allow you to call virtual methods from a destructor. */ + assert(!bufPos); + if (buffer) delete[] buffer; +} + + +void BufferedSink::operator () (const unsigned char * data, size_t len) +{ + if (!buffer) buffer = new unsigned char[bufSize]; + + while (len) { + /* Optimisation: bypass the buffer if the data exceeds the + buffer size. */ + if (bufPos + len >= bufSize) { + flush(); + write(data, len); + break; + } + /* Otherwise, copy the bytes to the buffer. Flush the buffer + when it's full. */ + size_t n = bufPos + len > bufSize ? bufSize - bufPos : len; + memcpy(buffer + bufPos, data, n); + data += n; bufPos += n; len -= n; + if (bufPos == bufSize) flush(); + } +} + + +void BufferedSink::flush() +{ + if (bufPos == 0) return; + size_t n = bufPos; + bufPos = 0; // don't trigger the assert() in ~BufferedSink() + write(buffer, n); +} + + +FdSink::~FdSink() +{ + try { flush(); } catch (...) { ignoreException(); } +} + + +void FdSink::write(const unsigned char * data, size_t len) +{ + writeFull(fd, data, len); +} + + +void Source::operator () (unsigned char * data, size_t len) +{ + while (len) { + size_t n = read(data, len); + data += n; len -= n; + } +} + + +BufferedSource::~BufferedSource() +{ + if (buffer) delete[] buffer; +} + + +size_t BufferedSource::read(unsigned char * data, size_t len) +{ + if (!buffer) buffer = new unsigned char[bufSize]; + + if (!bufPosIn) bufPosIn = readUnbuffered(buffer, bufSize); + + /* Copy out the data in the buffer. */ + size_t n = len > bufPosIn - bufPosOut ? bufPosIn - bufPosOut : len; + memcpy(data, buffer + bufPosOut, n); + bufPosOut += n; + if (bufPosIn == bufPosOut) bufPosIn = bufPosOut = 0; + return n; +} + + +bool BufferedSource::hasData() +{ + return bufPosOut < bufPosIn; +} + + +size_t FdSource::readUnbuffered(unsigned char * data, size_t len) +{ + ssize_t n; + do { + checkInterrupt(); + n = ::read(fd, (char *) data, bufSize); + } while (n == -1 && errno == EINTR); + if (n == -1) throw SysError("reading from file"); + if (n == 0) throw EndOfFile("unexpected end-of-file"); + return n; +} + + +size_t StringSource::read(unsigned char * data, size_t len) +{ + if (pos == s.size()) throw EndOfFile("end of string reached"); + size_t n = s.copy((char *) data, len, pos); + pos += n; + return n; +} + + +void writePadding(size_t len, Sink & sink) +{ + if (len % 8) { + unsigned char zero[8]; + memset(zero, 0, sizeof(zero)); + sink(zero, 8 - (len % 8)); + } +} + + +void writeInt(unsigned int n, Sink & sink) +{ + unsigned char buf[8]; + memset(buf, 0, sizeof(buf)); + buf[0] = n & 0xff; + buf[1] = (n >> 8) & 0xff; + buf[2] = (n >> 16) & 0xff; + buf[3] = (n >> 24) & 0xff; + sink(buf, sizeof(buf)); +} + + +void writeLongLong(unsigned long long n, Sink & sink) +{ + unsigned char buf[8]; + buf[0] = n & 0xff; + buf[1] = (n >> 8) & 0xff; + buf[2] = (n >> 16) & 0xff; + buf[3] = (n >> 24) & 0xff; + buf[4] = (n >> 32) & 0xff; + buf[5] = (n >> 40) & 0xff; + buf[6] = (n >> 48) & 0xff; + buf[7] = (n >> 56) & 0xff; + sink(buf, sizeof(buf)); +} + + +void writeString(const unsigned char * buf, size_t len, Sink & sink) +{ + writeInt(len, sink); + sink(buf, len); + writePadding(len, sink); +} + + +void writeString(const string & s, Sink & sink) +{ + writeString((const unsigned char *) s.data(), s.size(), sink); +} + + +template<class T> void writeStrings(const T & ss, Sink & sink) +{ + writeInt(ss.size(), sink); + foreach (typename T::const_iterator, i, ss) + writeString(*i, sink); +} + +template void writeStrings(const Paths & ss, Sink & sink); +template void writeStrings(const PathSet & ss, Sink & sink); + + +void readPadding(size_t len, Source & source) +{ + if (len % 8) { + unsigned char zero[8]; + size_t n = 8 - (len % 8); + source(zero, n); + for (unsigned int i = 0; i < n; i++) + if (zero[i]) throw SerialisationError("non-zero padding"); + } +} + + +unsigned int readInt(Source & source) +{ + unsigned char buf[8]; + source(buf, sizeof(buf)); + if (buf[4] || buf[5] || buf[6] || buf[7]) + throw SerialisationError("implementation cannot deal with > 32-bit integers"); + return + buf[0] | + (buf[1] << 8) | + (buf[2] << 16) | + (buf[3] << 24); +} + + +unsigned long long readLongLong(Source & source) +{ + unsigned char buf[8]; + source(buf, sizeof(buf)); + return + ((unsigned long long) buf[0]) | + ((unsigned long long) buf[1] << 8) | + ((unsigned long long) buf[2] << 16) | + ((unsigned long long) buf[3] << 24) | + ((unsigned long long) buf[4] << 32) | + ((unsigned long long) buf[5] << 40) | + ((unsigned long long) buf[6] << 48) | + ((unsigned long long) buf[7] << 56); +} + + +size_t readString(unsigned char * buf, size_t max, Source & source) +{ + size_t len = readInt(source); + if (len > max) throw Error("string is too long"); + source(buf, len); + readPadding(len, source); + return len; +} + + +string readString(Source & source) +{ + size_t len = readInt(source); + unsigned char * buf = new unsigned char[len]; + AutoDeleteArray<unsigned char> d(buf); + source(buf, len); + readPadding(len, source); + return string((char *) buf, len); +} + + +template<class T> T readStrings(Source & source) +{ + unsigned int count = readInt(source); + T ss; + while (count--) + ss.insert(ss.end(), readString(source)); + return ss; +} + +template Paths readStrings(Source & source); +template PathSet readStrings(Source & source); + + +} diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh new file mode 100644 index 000000000000..e5a9df1d050a --- /dev/null +++ b/src/libutil/serialise.hh @@ -0,0 +1,133 @@ +#pragma once + +#include "types.hh" + + +namespace nix { + + +/* Abstract destination of binary data. */ +struct Sink +{ + virtual ~Sink() { } + virtual void operator () (const unsigned char * data, size_t len) = 0; +}; + + +/* A buffered abstract sink. */ +struct BufferedSink : Sink +{ + size_t bufSize, bufPos; + unsigned char * buffer; + + BufferedSink(size_t bufSize = 32 * 1024) + : bufSize(bufSize), bufPos(0), buffer(0) { } + ~BufferedSink(); + + void operator () (const unsigned char * data, size_t len); + + void flush(); + + virtual void write(const unsigned char * data, size_t len) = 0; +}; + + +/* Abstract source of binary data. */ +struct Source +{ + virtual ~Source() { } + + /* Store exactly ‘len’ bytes in the buffer pointed to by ‘data’. + It blocks until all the requested data is available, or throws + an error if it is not going to be available. */ + void operator () (unsigned char * data, size_t len); + + /* Store up to ‘len’ in the buffer pointed to by ‘data’, and + return the number of bytes stored. If blocks until at least + one byte is available. */ + virtual size_t read(unsigned char * data, size_t len) = 0; +}; + + +/* A buffered abstract source. */ +struct BufferedSource : Source +{ + size_t bufSize, bufPosIn, bufPosOut; + unsigned char * buffer; + + BufferedSource(size_t bufSize = 32 * 1024) + : bufSize(bufSize), bufPosIn(0), bufPosOut(0), buffer(0) { } + ~BufferedSource(); + + size_t read(unsigned char * data, size_t len); + + /* Underlying read call, to be overridden. */ + virtual size_t readUnbuffered(unsigned char * data, size_t len) = 0; + + bool hasData(); +}; + + +/* A sink that writes data to a file descriptor. */ +struct FdSink : BufferedSink +{ + int fd; + + FdSink() : fd(-1) { } + FdSink(int fd) : fd(fd) { } + ~FdSink(); + + void write(const unsigned char * data, size_t len); +}; + + +/* A source that reads data from a file descriptor. */ +struct FdSource : BufferedSource +{ + int fd; + FdSource() : fd(-1) { } + FdSource(int fd) : fd(fd) { } + size_t readUnbuffered(unsigned char * data, size_t len); +}; + + +/* A sink that writes data to a string. */ +struct StringSink : Sink +{ + string s; + void operator () (const unsigned char * data, size_t len) + { + s.append((const char *) data, len); + } +}; + + +/* A source that reads data from a string. */ +struct StringSource : Source +{ + const string & s; + size_t pos; + StringSource(const string & _s) : s(_s), pos(0) { } + size_t read(unsigned char * data, size_t len); +}; + + +void writePadding(size_t len, Sink & sink); +void writeInt(unsigned int n, Sink & sink); +void writeLongLong(unsigned long long n, Sink & sink); +void writeString(const unsigned char * buf, size_t len, Sink & sink); +void writeString(const string & s, Sink & sink); +template<class T> void writeStrings(const T & ss, Sink & sink); + +void readPadding(size_t len, Source & source); +unsigned int readInt(Source & source); +unsigned long long readLongLong(Source & source); +size_t readString(unsigned char * buf, size_t max, Source & source); +string readString(Source & source); +template<class T> T readStrings(Source & source); + + +MakeError(SerialisationError, Error) + + +} diff --git a/src/libutil/sha1.c b/src/libutil/sha1.c new file mode 100644 index 000000000000..d9d294d15540 --- /dev/null +++ b/src/libutil/sha1.c @@ -0,0 +1,369 @@ +/* $Id$ */ + +/* sha.c - Implementation of the Secure Hash Algorithm + * + * Copyright (C) 1995, A.M. Kuchling + * + * Distribute and use freely; there are no restrictions on further + * dissemination and usage except those imposed by the laws of your + * country of residence. + * + * Adapted to pike and some cleanup by Niels Möller. + */ + +/* $Id$ */ + +/* SHA: NIST's Secure Hash Algorithm */ + +/* Based on SHA code originally posted to sci.crypt by Peter Gutmann + in message <30ajo5$oe8@ccu2.auckland.ac.nz>. + Modified to test for endianness on creation of SHA objects by AMK. + Also, the original specification of SHA was found to have a weakness + by NSA/NIST. This code implements the fixed version of SHA. +*/ + +/* Here's the first paragraph of Peter Gutmann's posting: + +The following is my SHA (FIPS 180) code updated to allow use of the "fixed" +SHA, thanks to Jim Gillogly and an anonymous contributor for the information on +what's changed in the new version. The fix is a simple change which involves +adding a single rotate in the initial expansion function. It is unknown +whether this is an optimal solution to the problem which was discovered in the +SHA or whether it's simply a bandaid which fixes the problem with a minimum of +effort (for example the reengineering of a great many Capstone chips). +*/ + +#include "sha1.h" + +#include <string.h> + +void sha_copy(struct SHA_CTX *dest, struct SHA_CTX *src) +{ + unsigned int i; + + dest->count_l=src->count_l; + dest->count_h=src->count_h; + for(i=0; i<SHA_DIGESTLEN; i++) + dest->digest[i]=src->digest[i]; + for(i=0; i < src->index; i++) + dest->block[i] = src->block[i]; + dest->index = src->index; +} + + +/* The SHA f()-functions. The f1 and f3 functions can be optimized to + save one boolean operation each - thanks to Rich Schroeppel, + rcs@cs.arizona.edu for discovering this */ + +/*#define f1(x,y,z) ( ( x & y ) | ( ~x & z ) ) // Rounds 0-19 */ +#define f1(x,y,z) ( z ^ ( x & ( y ^ z ) ) ) /* Rounds 0-19 */ +#define f2(x,y,z) ( x ^ y ^ z ) /* Rounds 20-39 */ +/*#define f3(x,y,z) ( ( x & y ) | ( x & z ) | ( y & z ) ) // Rounds 40-59 */ +#define f3(x,y,z) ( ( x & y ) | ( z & ( x | y ) ) ) /* Rounds 40-59 */ +#define f4(x,y,z) ( x ^ y ^ z ) /* Rounds 60-79 */ + +/* The SHA Mysterious Constants */ + +#define K1 0x5A827999L /* Rounds 0-19 */ +#define K2 0x6ED9EBA1L /* Rounds 20-39 */ +#define K3 0x8F1BBCDCL /* Rounds 40-59 */ +#define K4 0xCA62C1D6L /* Rounds 60-79 */ + +/* SHA initial values */ + +#define h0init 0x67452301L +#define h1init 0xEFCDAB89L +#define h2init 0x98BADCFEL +#define h3init 0x10325476L +#define h4init 0xC3D2E1F0L + +/* 32-bit rotate left - kludged with shifts */ + +#define ROTL(n,X) ( ( (X) << (n) ) | ( (X) >> ( 32 - (n) ) ) ) + +/* The initial expanding function. The hash function is defined over an + 80-word expanded input array W, where the first 16 are copies of the input + data, and the remaining 64 are defined by + + W[ i ] = W[ i - 16 ] ^ W[ i - 14 ] ^ W[ i - 8 ] ^ W[ i - 3 ] + + This implementation generates these values on the fly in a circular + buffer - thanks to Colin Plumb, colin@nyx10.cs.du.edu for this + optimization. + + The updated SHA changes the expanding function by adding a rotate of 1 + bit. Thanks to Jim Gillogly, jim@rand.org, and an anonymous contributor + for this information */ + +#define expand(W,i) ( W[ i & 15 ] = \ + ROTL( 1, ( W[ i & 15 ] ^ W[ (i - 14) & 15 ] ^ \ + W[ (i - 8) & 15 ] ^ W[ (i - 3) & 15 ] ) ) ) + + +/* The prototype SHA sub-round. The fundamental sub-round is: + + a' = e + ROTL( 5, a ) + f( b, c, d ) + k + data; + b' = a; + c' = ROTL( 30, b ); + d' = c; + e' = d; + + but this is implemented by unrolling the loop 5 times and renaming the + variables ( e, a, b, c, d ) = ( a', b', c', d', e' ) each iteration. + This code is then replicated 20 times for each of the 4 functions, using + the next 20 values from the W[] array each time */ + +#define subRound(a, b, c, d, e, f, k, data) \ + ( e += ROTL( 5, a ) + f( b, c, d ) + k + data, b = ROTL( 30, b ) ) + +/* Initialize the SHA values */ + +void SHA1_Init(struct SHA_CTX *ctx) +{ + /* Set the h-vars to their initial values */ + ctx->digest[ 0 ] = h0init; + ctx->digest[ 1 ] = h1init; + ctx->digest[ 2 ] = h2init; + ctx->digest[ 3 ] = h3init; + ctx->digest[ 4 ] = h4init; + + /* Initialize bit count */ + ctx->count_l = ctx->count_h = 0; + + /* Initialize buffer */ + ctx->index = 0; +} + +/* Perform the SHA transformation. Note that this code, like MD5, seems to + break some optimizing compilers due to the complexity of the expressions + and the size of the basic block. It may be necessary to split it into + sections, e.g. based on the four subrounds + + Note that this function destroys the data area */ + +static void sha_transform(struct SHA_CTX *ctx, uint32_t *data ) +{ + uint32_t A, B, C, D, E; /* Local vars */ + + /* Set up first buffer and local data buffer */ + A = ctx->digest[0]; + B = ctx->digest[1]; + C = ctx->digest[2]; + D = ctx->digest[3]; + E = ctx->digest[4]; + + /* Heavy mangling, in 4 sub-rounds of 20 interations each. */ + subRound( A, B, C, D, E, f1, K1, data[ 0] ); + subRound( E, A, B, C, D, f1, K1, data[ 1] ); + subRound( D, E, A, B, C, f1, K1, data[ 2] ); + subRound( C, D, E, A, B, f1, K1, data[ 3] ); + subRound( B, C, D, E, A, f1, K1, data[ 4] ); + subRound( A, B, C, D, E, f1, K1, data[ 5] ); + subRound( E, A, B, C, D, f1, K1, data[ 6] ); + subRound( D, E, A, B, C, f1, K1, data[ 7] ); + subRound( C, D, E, A, B, f1, K1, data[ 8] ); + subRound( B, C, D, E, A, f1, K1, data[ 9] ); + subRound( A, B, C, D, E, f1, K1, data[10] ); + subRound( E, A, B, C, D, f1, K1, data[11] ); + subRound( D, E, A, B, C, f1, K1, data[12] ); + subRound( C, D, E, A, B, f1, K1, data[13] ); + subRound( B, C, D, E, A, f1, K1, data[14] ); + subRound( A, B, C, D, E, f1, K1, data[15] ); + subRound( E, A, B, C, D, f1, K1, expand( data, 16 ) ); + subRound( D, E, A, B, C, f1, K1, expand( data, 17 ) ); + subRound( C, D, E, A, B, f1, K1, expand( data, 18 ) ); + subRound( B, C, D, E, A, f1, K1, expand( data, 19 ) ); + + subRound( A, B, C, D, E, f2, K2, expand( data, 20 ) ); + subRound( E, A, B, C, D, f2, K2, expand( data, 21 ) ); + subRound( D, E, A, B, C, f2, K2, expand( data, 22 ) ); + subRound( C, D, E, A, B, f2, K2, expand( data, 23 ) ); + subRound( B, C, D, E, A, f2, K2, expand( data, 24 ) ); + subRound( A, B, C, D, E, f2, K2, expand( data, 25 ) ); + subRound( E, A, B, C, D, f2, K2, expand( data, 26 ) ); + subRound( D, E, A, B, C, f2, K2, expand( data, 27 ) ); + subRound( C, D, E, A, B, f2, K2, expand( data, 28 ) ); + subRound( B, C, D, E, A, f2, K2, expand( data, 29 ) ); + subRound( A, B, C, D, E, f2, K2, expand( data, 30 ) ); + subRound( E, A, B, C, D, f2, K2, expand( data, 31 ) ); + subRound( D, E, A, B, C, f2, K2, expand( data, 32 ) ); + subRound( C, D, E, A, B, f2, K2, expand( data, 33 ) ); + subRound( B, C, D, E, A, f2, K2, expand( data, 34 ) ); + subRound( A, B, C, D, E, f2, K2, expand( data, 35 ) ); + subRound( E, A, B, C, D, f2, K2, expand( data, 36 ) ); + subRound( D, E, A, B, C, f2, K2, expand( data, 37 ) ); + subRound( C, D, E, A, B, f2, K2, expand( data, 38 ) ); + subRound( B, C, D, E, A, f2, K2, expand( data, 39 ) ); + + subRound( A, B, C, D, E, f3, K3, expand( data, 40 ) ); + subRound( E, A, B, C, D, f3, K3, expand( data, 41 ) ); + subRound( D, E, A, B, C, f3, K3, expand( data, 42 ) ); + subRound( C, D, E, A, B, f3, K3, expand( data, 43 ) ); + subRound( B, C, D, E, A, f3, K3, expand( data, 44 ) ); + subRound( A, B, C, D, E, f3, K3, expand( data, 45 ) ); + subRound( E, A, B, C, D, f3, K3, expand( data, 46 ) ); + subRound( D, E, A, B, C, f3, K3, expand( data, 47 ) ); + subRound( C, D, E, A, B, f3, K3, expand( data, 48 ) ); + subRound( B, C, D, E, A, f3, K3, expand( data, 49 ) ); + subRound( A, B, C, D, E, f3, K3, expand( data, 50 ) ); + subRound( E, A, B, C, D, f3, K3, expand( data, 51 ) ); + subRound( D, E, A, B, C, f3, K3, expand( data, 52 ) ); + subRound( C, D, E, A, B, f3, K3, expand( data, 53 ) ); + subRound( B, C, D, E, A, f3, K3, expand( data, 54 ) ); + subRound( A, B, C, D, E, f3, K3, expand( data, 55 ) ); + subRound( E, A, B, C, D, f3, K3, expand( data, 56 ) ); + subRound( D, E, A, B, C, f3, K3, expand( data, 57 ) ); + subRound( C, D, E, A, B, f3, K3, expand( data, 58 ) ); + subRound( B, C, D, E, A, f3, K3, expand( data, 59 ) ); + + subRound( A, B, C, D, E, f4, K4, expand( data, 60 ) ); + subRound( E, A, B, C, D, f4, K4, expand( data, 61 ) ); + subRound( D, E, A, B, C, f4, K4, expand( data, 62 ) ); + subRound( C, D, E, A, B, f4, K4, expand( data, 63 ) ); + subRound( B, C, D, E, A, f4, K4, expand( data, 64 ) ); + subRound( A, B, C, D, E, f4, K4, expand( data, 65 ) ); + subRound( E, A, B, C, D, f4, K4, expand( data, 66 ) ); + subRound( D, E, A, B, C, f4, K4, expand( data, 67 ) ); + subRound( C, D, E, A, B, f4, K4, expand( data, 68 ) ); + subRound( B, C, D, E, A, f4, K4, expand( data, 69 ) ); + subRound( A, B, C, D, E, f4, K4, expand( data, 70 ) ); + subRound( E, A, B, C, D, f4, K4, expand( data, 71 ) ); + subRound( D, E, A, B, C, f4, K4, expand( data, 72 ) ); + subRound( C, D, E, A, B, f4, K4, expand( data, 73 ) ); + subRound( B, C, D, E, A, f4, K4, expand( data, 74 ) ); + subRound( A, B, C, D, E, f4, K4, expand( data, 75 ) ); + subRound( E, A, B, C, D, f4, K4, expand( data, 76 ) ); + subRound( D, E, A, B, C, f4, K4, expand( data, 77 ) ); + subRound( C, D, E, A, B, f4, K4, expand( data, 78 ) ); + subRound( B, C, D, E, A, f4, K4, expand( data, 79 ) ); + + /* Build message digest */ + ctx->digest[0] += A; + ctx->digest[1] += B; + ctx->digest[2] += C; + ctx->digest[3] += D; + ctx->digest[4] += E; +} + +#if 1 + +#ifndef EXTRACT_UCHAR +#define EXTRACT_UCHAR(p) (*(unsigned char *)(p)) +#endif + +#define STRING2INT(s) ((((((EXTRACT_UCHAR(s) << 8) \ + | EXTRACT_UCHAR(s+1)) << 8) \ + | EXTRACT_UCHAR(s+2)) << 8) \ + | EXTRACT_UCHAR(s+3)) +#else +uint32_t STRING2INT(unsigned char *s) +{ + uint32_t r; + unsigned int i; + + for (i = 0, r = 0; i < 4; i++, s++) + r = (r << 8) | *s; + return r; +} +#endif + +static void sha_block(struct SHA_CTX *ctx, const unsigned char *block) +{ + uint32_t data[SHA_DATALEN]; + unsigned int i; + + /* Update block count */ + if (!++ctx->count_l) + ++ctx->count_h; + + /* Endian independent conversion */ + for (i = 0; i<SHA_DATALEN; i++, block += 4) + data[i] = STRING2INT(block); + + sha_transform(ctx, data); +} + +void SHA1_Update(struct SHA_CTX *ctx, const unsigned char *buffer, uint32_t len) +{ + if (ctx->index) + { /* Try to fill partial block */ + unsigned left = SHA_DATASIZE - ctx->index; + if (len < left) + { + memcpy(ctx->block + ctx->index, buffer, len); + ctx->index += len; + return; /* Finished */ + } + else + { + memcpy(ctx->block + ctx->index, buffer, left); + sha_block(ctx, ctx->block); + buffer += left; + len -= left; + } + } + while (len >= SHA_DATASIZE) + { + sha_block(ctx, buffer); + buffer += SHA_DATASIZE; + len -= SHA_DATASIZE; + } + if ((ctx->index = len)) /* This assignment is intended */ + /* Buffer leftovers */ + memcpy(ctx->block, buffer, len); +} + +/* Final wrapup - pad to SHA_DATASIZE-byte boundary with the bit pattern + 1 0* (64-bit count of bits processed, MSB-first) */ + +void SHA1_Final(unsigned char *s, struct SHA_CTX *ctx) +{ + uint32_t data[SHA_DATALEN]; + unsigned int i; + unsigned int words; + + i = ctx->index; + /* Set the first char of padding to 0x80. This is safe since there is + always at least one byte free */ + ctx->block[i++] = 0x80; + + /* Fill rest of word */ + for( ; i & 3; i++) + ctx->block[i] = 0; + + /* i is now a multiple of the word size 4 */ + words = i >> 2; + for (i = 0; i < words; i++) + data[i] = STRING2INT(ctx->block + 4*i); + + if (words > (SHA_DATALEN-2)) + { /* No room for length in this block. Process it and + * pad with another one */ + for (i = words ; i < SHA_DATALEN; i++) + data[i] = 0; + sha_transform(ctx, data); + for (i = 0; i < (SHA_DATALEN-2); i++) + data[i] = 0; + } + else + for (i = words ; i < SHA_DATALEN - 2; i++) + data[i] = 0; + /* Theres 512 = 2^9 bits in one block */ + data[SHA_DATALEN-2] = (ctx->count_h << 9) | (ctx->count_l >> 23); + data[SHA_DATALEN-1] = (ctx->count_l << 9) | (ctx->index << 3); + sha_transform(ctx, data); + sha_digest(ctx, s); +} + +void sha_digest(struct SHA_CTX *ctx, unsigned char *s) +{ + unsigned int i; + + for (i = 0; i < SHA_DIGESTLEN; i++) + { + *s++ = ctx->digest[i] >> 24; + *s++ = 0xff & (ctx->digest[i] >> 16); + *s++ = 0xff & (ctx->digest[i] >> 8); + *s++ = 0xff & ctx->digest[i]; + } +} diff --git a/src/libutil/sha1.h b/src/libutil/sha1.h new file mode 100644 index 000000000000..715040dd48df --- /dev/null +++ b/src/libutil/sha1.h @@ -0,0 +1,28 @@ +#ifndef _SHA_H +#define _SHA_H + +#include <inttypes.h> + +/* The SHA block size and message digest sizes, in bytes */ + +#define SHA_DATASIZE 64 +#define SHA_DATALEN 16 +#define SHA_DIGESTSIZE 20 +#define SHA_DIGESTLEN 5 +/* The structure for storing SHA info */ + +struct SHA_CTX { + uint32_t digest[SHA_DIGESTLEN]; /* Message digest */ + uint32_t count_l, count_h; /* 64-bit block count */ + uint8_t block[SHA_DATASIZE]; /* SHA data buffer */ + unsigned int index; /* index into buffer */ +}; + +void SHA1_Init(struct SHA_CTX *ctx); +void SHA1_Update(struct SHA_CTX *ctx, const unsigned char *buffer, uint32_t len); +void SHA1_Final(unsigned char *s, struct SHA_CTX *ctx); +void sha_digest(struct SHA_CTX *ctx, unsigned char *s); +void sha_copy(struct SHA_CTX *dest, struct SHA_CTX *src); + + +#endif /* !_SHA_H */ diff --git a/src/libutil/sha256.c b/src/libutil/sha256.c new file mode 100644 index 000000000000..63ed0ba43011 --- /dev/null +++ b/src/libutil/sha256.c @@ -0,0 +1,238 @@ +/* crypto/sha/sha256.c */ +/* ==================================================================== + * Copyright (c) 2004 The OpenSSL Project. All rights reserved + * according to the OpenSSL license [found in ./md32_common.h]. + * ==================================================================== + */ + +#include <stdlib.h> +#include <string.h> + +#include "sha256.h" + +int SHA224_Init (SHA256_CTX *c) + { + c->h[0]=0xc1059ed8UL; c->h[1]=0x367cd507UL; + c->h[2]=0x3070dd17UL; c->h[3]=0xf70e5939UL; + c->h[4]=0xffc00b31UL; c->h[5]=0x68581511UL; + c->h[6]=0x64f98fa7UL; c->h[7]=0xbefa4fa4UL; + c->Nl=0; c->Nh=0; + c->num=0; c->md_len=SHA224_DIGEST_LENGTH; + return 1; + } + +int SHA256_Init (SHA256_CTX *c) + { + c->h[0]=0x6a09e667UL; c->h[1]=0xbb67ae85UL; + c->h[2]=0x3c6ef372UL; c->h[3]=0xa54ff53aUL; + c->h[4]=0x510e527fUL; c->h[5]=0x9b05688cUL; + c->h[6]=0x1f83d9abUL; c->h[7]=0x5be0cd19UL; + c->Nl=0; c->Nh=0; + c->num=0; c->md_len=SHA256_DIGEST_LENGTH; + return 1; + } + +unsigned char *SHA224(const unsigned char *d, size_t n, unsigned char *md) + { + SHA256_CTX c; + static unsigned char m[SHA224_DIGEST_LENGTH]; + + if (md == NULL) md=m; + SHA224_Init(&c); + SHA256_Update(&c,d,n); + SHA256_Final(md,&c); + return(md); + } + +unsigned char *SHA256(const unsigned char *d, size_t n, unsigned char *md) + { + SHA256_CTX c; + static unsigned char m[SHA256_DIGEST_LENGTH]; + + if (md == NULL) md=m; + SHA256_Init(&c); + SHA256_Update(&c,d,n); + SHA256_Final(md,&c); + return(md); + } + +int SHA224_Update(SHA256_CTX *c, const void *data, size_t len) +{ return SHA256_Update (c,data,len); } +int SHA224_Final (unsigned char *md, SHA256_CTX *c) +{ return SHA256_Final (md,c); } + +#define DATA_ORDER_IS_BIG_ENDIAN + +#define HASH_LONG uint32_t +#define HASH_LONG_LOG2 2 +#define HASH_CTX SHA256_CTX +#define HASH_CBLOCK SHA_CBLOCK +#define HASH_LBLOCK SHA_LBLOCK +/* + * Note that FIPS180-2 discusses "Truncation of the Hash Function Output." + * default: case below covers for it. It's not clear however if it's + * permitted to truncate to amount of bytes not divisible by 4. I bet not, + * but if it is, then default: case shall be extended. For reference. + * Idea behind separate cases for pre-defined lenghts is to let the + * compiler decide if it's appropriate to unroll small loops. + */ +#define HASH_MAKE_STRING(c,s) do { \ + unsigned long ll; \ + unsigned int n; \ + switch ((c)->md_len) \ + { case SHA224_DIGEST_LENGTH: \ + for (n=0;n<SHA224_DIGEST_LENGTH/4;n++) \ + { ll=(c)->h[n]; HOST_l2c(ll,(s)); } \ + break; \ + case SHA256_DIGEST_LENGTH: \ + for (n=0;n<SHA256_DIGEST_LENGTH/4;n++) \ + { ll=(c)->h[n]; HOST_l2c(ll,(s)); } \ + break; \ + default: \ + if ((c)->md_len > SHA256_DIGEST_LENGTH) \ + return 0; \ + for (n=0;n<(c)->md_len/4;n++) \ + { ll=(c)->h[n]; HOST_l2c(ll,(s)); } \ + break; \ + } \ + } while (0) + +#define HASH_UPDATE SHA256_Update +#define HASH_TRANSFORM SHA256_Transform +#define HASH_FINAL SHA256_Final +#define HASH_BLOCK_HOST_ORDER sha256_block_host_order +#define HASH_BLOCK_DATA_ORDER sha256_block_data_order +void sha256_block_host_order (SHA256_CTX *ctx, const void *in, size_t num); +void sha256_block_data_order (SHA256_CTX *ctx, const void *in, size_t num); + +#include "md32_common.h" + +static const uint32_t K256[64] = { + 0x428a2f98UL,0x71374491UL,0xb5c0fbcfUL,0xe9b5dba5UL, + 0x3956c25bUL,0x59f111f1UL,0x923f82a4UL,0xab1c5ed5UL, + 0xd807aa98UL,0x12835b01UL,0x243185beUL,0x550c7dc3UL, + 0x72be5d74UL,0x80deb1feUL,0x9bdc06a7UL,0xc19bf174UL, + 0xe49b69c1UL,0xefbe4786UL,0x0fc19dc6UL,0x240ca1ccUL, + 0x2de92c6fUL,0x4a7484aaUL,0x5cb0a9dcUL,0x76f988daUL, + 0x983e5152UL,0xa831c66dUL,0xb00327c8UL,0xbf597fc7UL, + 0xc6e00bf3UL,0xd5a79147UL,0x06ca6351UL,0x14292967UL, + 0x27b70a85UL,0x2e1b2138UL,0x4d2c6dfcUL,0x53380d13UL, + 0x650a7354UL,0x766a0abbUL,0x81c2c92eUL,0x92722c85UL, + 0xa2bfe8a1UL,0xa81a664bUL,0xc24b8b70UL,0xc76c51a3UL, + 0xd192e819UL,0xd6990624UL,0xf40e3585UL,0x106aa070UL, + 0x19a4c116UL,0x1e376c08UL,0x2748774cUL,0x34b0bcb5UL, + 0x391c0cb3UL,0x4ed8aa4aUL,0x5b9cca4fUL,0x682e6ff3UL, + 0x748f82eeUL,0x78a5636fUL,0x84c87814UL,0x8cc70208UL, + 0x90befffaUL,0xa4506cebUL,0xbef9a3f7UL,0xc67178f2UL }; + +/* + * FIPS specification refers to right rotations, while our ROTATE macro + * is left one. This is why you might notice that rotation coefficients + * differ from those observed in FIPS document by 32-N... + */ +#define Sigma0(x) (ROTATE((x),30) ^ ROTATE((x),19) ^ ROTATE((x),10)) +#define Sigma1(x) (ROTATE((x),26) ^ ROTATE((x),21) ^ ROTATE((x),7)) +#define sigma0(x) (ROTATE((x),25) ^ ROTATE((x),14) ^ ((x)>>3)) +#define sigma1(x) (ROTATE((x),15) ^ ROTATE((x),13) ^ ((x)>>10)) + +#define Ch(x,y,z) (((x) & (y)) ^ ((~(x)) & (z))) +#define Maj(x,y,z) (((x) & (y)) ^ ((x) & (z)) ^ ((y) & (z))) + +#define ROUND_00_15(i,a,b,c,d,e,f,g,h) do { \ + T1 += h + Sigma1(e) + Ch(e,f,g) + K256[i]; \ + h = Sigma0(a) + Maj(a,b,c); \ + d += T1; h += T1; } while (0) + +#define ROUND_16_63(i,a,b,c,d,e,f,g,h,X) do { \ + s0 = X[(i+1)&0x0f]; s0 = sigma0(s0); \ + s1 = X[(i+14)&0x0f]; s1 = sigma1(s1); \ + T1 = X[(i)&0x0f] += s0 + s1 + X[(i+9)&0x0f]; \ + ROUND_00_15(i,a,b,c,d,e,f,g,h); } while (0) + +static void sha256_block (SHA256_CTX *ctx, const void *in, size_t num, int host) + { + uint32_t a,b,c,d,e,f,g,h,s0,s1,T1; + uint32_t X[16]; + int i; + const unsigned char *data=in; + + while (num--) { + + a = ctx->h[0]; b = ctx->h[1]; c = ctx->h[2]; d = ctx->h[3]; + e = ctx->h[4]; f = ctx->h[5]; g = ctx->h[6]; h = ctx->h[7]; + + if (host) + { + const uint32_t *W=(const uint32_t *)data; + + T1 = X[0] = W[0]; ROUND_00_15(0,a,b,c,d,e,f,g,h); + T1 = X[1] = W[1]; ROUND_00_15(1,h,a,b,c,d,e,f,g); + T1 = X[2] = W[2]; ROUND_00_15(2,g,h,a,b,c,d,e,f); + T1 = X[3] = W[3]; ROUND_00_15(3,f,g,h,a,b,c,d,e); + T1 = X[4] = W[4]; ROUND_00_15(4,e,f,g,h,a,b,c,d); + T1 = X[5] = W[5]; ROUND_00_15(5,d,e,f,g,h,a,b,c); + T1 = X[6] = W[6]; ROUND_00_15(6,c,d,e,f,g,h,a,b); + T1 = X[7] = W[7]; ROUND_00_15(7,b,c,d,e,f,g,h,a); + T1 = X[8] = W[8]; ROUND_00_15(8,a,b,c,d,e,f,g,h); + T1 = X[9] = W[9]; ROUND_00_15(9,h,a,b,c,d,e,f,g); + T1 = X[10] = W[10]; ROUND_00_15(10,g,h,a,b,c,d,e,f); + T1 = X[11] = W[11]; ROUND_00_15(11,f,g,h,a,b,c,d,e); + T1 = X[12] = W[12]; ROUND_00_15(12,e,f,g,h,a,b,c,d); + T1 = X[13] = W[13]; ROUND_00_15(13,d,e,f,g,h,a,b,c); + T1 = X[14] = W[14]; ROUND_00_15(14,c,d,e,f,g,h,a,b); + T1 = X[15] = W[15]; ROUND_00_15(15,b,c,d,e,f,g,h,a); + + data += SHA256_CBLOCK; + } + else + { + uint32_t l; + + HOST_c2l(data,l); T1 = X[0] = l; ROUND_00_15(0,a,b,c,d,e,f,g,h); + HOST_c2l(data,l); T1 = X[1] = l; ROUND_00_15(1,h,a,b,c,d,e,f,g); + HOST_c2l(data,l); T1 = X[2] = l; ROUND_00_15(2,g,h,a,b,c,d,e,f); + HOST_c2l(data,l); T1 = X[3] = l; ROUND_00_15(3,f,g,h,a,b,c,d,e); + HOST_c2l(data,l); T1 = X[4] = l; ROUND_00_15(4,e,f,g,h,a,b,c,d); + HOST_c2l(data,l); T1 = X[5] = l; ROUND_00_15(5,d,e,f,g,h,a,b,c); + HOST_c2l(data,l); T1 = X[6] = l; ROUND_00_15(6,c,d,e,f,g,h,a,b); + HOST_c2l(data,l); T1 = X[7] = l; ROUND_00_15(7,b,c,d,e,f,g,h,a); + HOST_c2l(data,l); T1 = X[8] = l; ROUND_00_15(8,a,b,c,d,e,f,g,h); + HOST_c2l(data,l); T1 = X[9] = l; ROUND_00_15(9,h,a,b,c,d,e,f,g); + HOST_c2l(data,l); T1 = X[10] = l; ROUND_00_15(10,g,h,a,b,c,d,e,f); + HOST_c2l(data,l); T1 = X[11] = l; ROUND_00_15(11,f,g,h,a,b,c,d,e); + HOST_c2l(data,l); T1 = X[12] = l; ROUND_00_15(12,e,f,g,h,a,b,c,d); + HOST_c2l(data,l); T1 = X[13] = l; ROUND_00_15(13,d,e,f,g,h,a,b,c); + HOST_c2l(data,l); T1 = X[14] = l; ROUND_00_15(14,c,d,e,f,g,h,a,b); + HOST_c2l(data,l); T1 = X[15] = l; ROUND_00_15(15,b,c,d,e,f,g,h,a); + } + + for (i=16;i<64;i+=8) + { + ROUND_16_63(i+0,a,b,c,d,e,f,g,h,X); + ROUND_16_63(i+1,h,a,b,c,d,e,f,g,X); + ROUND_16_63(i+2,g,h,a,b,c,d,e,f,X); + ROUND_16_63(i+3,f,g,h,a,b,c,d,e,X); + ROUND_16_63(i+4,e,f,g,h,a,b,c,d,X); + ROUND_16_63(i+5,d,e,f,g,h,a,b,c,X); + ROUND_16_63(i+6,c,d,e,f,g,h,a,b,X); + ROUND_16_63(i+7,b,c,d,e,f,g,h,a,X); + } + + ctx->h[0] += a; ctx->h[1] += b; ctx->h[2] += c; ctx->h[3] += d; + ctx->h[4] += e; ctx->h[5] += f; ctx->h[6] += g; ctx->h[7] += h; + + } + } + +/* + * Idea is to trade couple of cycles for some space. On IA-32 we save + * about 4K in "big footprint" case. In "small footprint" case any gain + * is appreciated:-) + */ +void HASH_BLOCK_HOST_ORDER (SHA256_CTX *ctx, const void *in, size_t num) +{ sha256_block (ctx,in,num,1); } + +void HASH_BLOCK_DATA_ORDER (SHA256_CTX *ctx, const void *in, size_t num) +{ sha256_block (ctx,in,num,0); } + + diff --git a/src/libutil/sha256.h b/src/libutil/sha256.h new file mode 100644 index 000000000000..0686b84f0e08 --- /dev/null +++ b/src/libutil/sha256.h @@ -0,0 +1,35 @@ +#ifndef _SHA256_H +#define _SHA256_H 1 + +#include <inttypes.h> + +#define SHA_LBLOCK 16 +#define SHA_CBLOCK (SHA_LBLOCK*4) /* SHA treats input data as a + * contiguous array of 32 bit + * wide big-endian values. */ + +#define SHA256_CBLOCK (SHA_LBLOCK*4) /* SHA-256 treats input data as a + * contiguous array of 32 bit + * wide big-endian values. */ +#define SHA224_DIGEST_LENGTH 28 +#define SHA256_DIGEST_LENGTH 32 + +typedef struct SHA256state_st + { + uint32_t h[8]; + uint32_t Nl,Nh; + uint32_t data[SHA_LBLOCK]; + unsigned int num,md_len; + } SHA256_CTX; + +int SHA224_Init(SHA256_CTX *c); +int SHA224_Update(SHA256_CTX *c, const void *data, size_t len); +int SHA224_Final(unsigned char *md, SHA256_CTX *c); +unsigned char *SHA224(const unsigned char *d, size_t n,unsigned char *md); +int SHA256_Init(SHA256_CTX *c); +int SHA256_Update(SHA256_CTX *c, const void *data, size_t len); +int SHA256_Final(unsigned char *md, SHA256_CTX *c); +unsigned char *SHA256(const unsigned char *d, size_t n,unsigned char *md); +void SHA256_Transform(SHA256_CTX *c, const unsigned char *data); + +#endif diff --git a/src/libutil/types.hh b/src/libutil/types.hh new file mode 100644 index 000000000000..a2ab1cc0ee83 --- /dev/null +++ b/src/libutil/types.hh @@ -0,0 +1,77 @@ +#pragma once + +#include "config.h" + +#include <string> +#include <list> +#include <set> + +#include <boost/format.hpp> + + +namespace nix { + + +/* Inherit some names from other namespaces for convenience. */ +using std::string; +using std::list; +using std::set; +using std::vector; +using boost::format; + + +/* BaseError should generally not be caught, as it has Interrupted as + a subclass. Catch Error instead. */ +class BaseError : public std::exception +{ +protected: + string prefix_; // used for location traces etc. + string err; +public: + unsigned int status; // exit status + BaseError(const format & f, unsigned int status = 1); + ~BaseError() throw () { }; + const char * what() const throw () { return err.c_str(); } + const string & msg() const throw () { return err; } + const string & prefix() const throw () { return prefix_; } + BaseError & addPrefix(const format & f); +}; + +#define MakeError(newClass, superClass) \ + class newClass : public superClass \ + { \ + public: \ + newClass(const format & f, unsigned int status = 1) : superClass(f, status) { }; \ + }; + +MakeError(Error, BaseError) + +class SysError : public Error +{ +public: + int errNo; + SysError(const format & f); +}; + + +typedef list<string> Strings; +typedef set<string> StringSet; + + +/* Paths are just strings. */ +typedef string Path; +typedef list<Path> Paths; +typedef set<Path> PathSet; + + +typedef enum { + lvlError = 0, + lvlInfo, + lvlTalkative, + lvlChatty, + lvlDebug, + lvlVomit +} Verbosity; + + +} diff --git a/src/libutil/util.cc b/src/libutil/util.cc new file mode 100644 index 000000000000..9437a2f99a3c --- /dev/null +++ b/src/libutil/util.cc @@ -0,0 +1,1145 @@ +#include "config.h" + +#include <iostream> +#include <cerrno> +#include <cstdio> +#include <cstdlib> +#include <sstream> +#include <cstring> + +#include <sys/wait.h> +#include <unistd.h> +#include <fcntl.h> +#include <limits.h> + +#ifdef __APPLE__ +#include <sys/syscall.h> +#endif + +#include "util.hh" + + +extern char * * environ; + + +namespace nix { + + +BaseError::BaseError(const format & f, unsigned int status) + : status(status) +{ + err = f.str(); +} + + +BaseError & BaseError::addPrefix(const format & f) +{ + prefix_ = f.str() + prefix_; + return *this; +} + + +SysError::SysError(const format & f) + : Error(format("%1%: %2%") % f.str() % strerror(errno)) + , errNo(errno) +{ +} + + +string getEnv(const string & key, const string & def) +{ + char * value = getenv(key.c_str()); + return value ? string(value) : def; +} + + +Path absPath(Path path, Path dir) +{ + if (path[0] != '/') { + if (dir == "") { +#ifdef __GNU__ + /* GNU (aka. GNU/Hurd) doesn't have any limitation on path + lengths and doesn't define `PATH_MAX'. */ + char *buf = getcwd(NULL, 0); + if (buf == NULL) +#else + char buf[PATH_MAX]; + if (!getcwd(buf, sizeof(buf))) +#endif + throw SysError("cannot get cwd"); + dir = buf; +#ifdef __GNU__ + free(buf); +#endif + } + path = dir + "/" + path; + } + return canonPath(path); +} + + +Path canonPath(const Path & path, bool resolveSymlinks) +{ + string s; + + if (path[0] != '/') + throw Error(format("not an absolute path: `%1%'") % path); + + string::const_iterator i = path.begin(), end = path.end(); + string temp; + + /* Count the number of times we follow a symlink and stop at some + arbitrary (but high) limit to prevent infinite loops. */ + unsigned int followCount = 0, maxFollow = 1024; + + while (1) { + + /* Skip slashes. */ + while (i != end && *i == '/') i++; + if (i == end) break; + + /* Ignore `.'. */ + if (*i == '.' && (i + 1 == end || i[1] == '/')) + i++; + + /* If `..', delete the last component. */ + else if (*i == '.' && i + 1 < end && i[1] == '.' && + (i + 2 == end || i[2] == '/')) + { + if (!s.empty()) s.erase(s.rfind('/')); + i += 2; + } + + /* Normal component; copy it. */ + else { + s += '/'; + while (i != end && *i != '/') s += *i++; + + /* If s points to a symlink, resolve it and restart (since + the symlink target might contain new symlinks). */ + if (resolveSymlinks && isLink(s)) { + if (++followCount >= maxFollow) + throw Error(format("infinite symlink recursion in path `%1%'") % path); + temp = absPath(readLink(s), dirOf(s)) + + string(i, end); + i = temp.begin(); /* restart */ + end = temp.end(); + s = ""; + /* !!! potential for infinite loop */ + } + } + } + + return s.empty() ? "/" : s; +} + + +Path dirOf(const Path & path) +{ + Path::size_type pos = path.rfind('/'); + if (pos == string::npos) + throw Error(format("invalid file name `%1%'") % path); + return pos == 0 ? "/" : Path(path, 0, pos); +} + + +string baseNameOf(const Path & path) +{ + Path::size_type pos = path.rfind('/'); + if (pos == string::npos) + throw Error(format("invalid file name `%1%'") % path); + return string(path, pos + 1); +} + + +bool isInDir(const Path & path, const Path & dir) +{ + return path[0] == '/' + && string(path, 0, dir.size()) == dir + && path.size() >= dir.size() + 2 + && path[dir.size()] == '/'; +} + + +struct stat lstat(const Path & path) +{ + struct stat st; + if (lstat(path.c_str(), &st)) + throw SysError(format("getting status of `%1%'") % path); + return st; +} + + +bool pathExists(const Path & path) +{ + int res; + struct stat st; + res = lstat(path.c_str(), &st); + if (!res) return true; + if (errno != ENOENT && errno != ENOTDIR) + throw SysError(format("getting status of %1%") % path); + return false; +} + + +Path readLink(const Path & path) +{ + checkInterrupt(); + struct stat st = lstat(path); + if (!S_ISLNK(st.st_mode)) + throw Error(format("`%1%' is not a symlink") % path); + char buf[st.st_size]; + if (readlink(path.c_str(), buf, st.st_size) != st.st_size) + throw SysError(format("reading symbolic link `%1%'") % path); + return string(buf, st.st_size); +} + + +bool isLink(const Path & path) +{ + struct stat st = lstat(path); + return S_ISLNK(st.st_mode); +} + + +Strings readDirectory(const Path & path) +{ + Strings names; + + AutoCloseDir dir = opendir(path.c_str()); + if (!dir) throw SysError(format("opening directory `%1%'") % path); + + struct dirent * dirent; + while (errno = 0, dirent = readdir(dir)) { /* sic */ + checkInterrupt(); + string name = dirent->d_name; + if (name == "." || name == "..") continue; + names.push_back(name); + } + if (errno) throw SysError(format("reading directory `%1%'") % path); + + return names; +} + + +string readFile(int fd) +{ + struct stat st; + if (fstat(fd, &st) == -1) + throw SysError("statting file"); + + unsigned char * buf = new unsigned char[st.st_size]; + AutoDeleteArray<unsigned char> d(buf); + readFull(fd, buf, st.st_size); + + return string((char *) buf, st.st_size); +} + + +string readFile(const Path & path, bool drain) +{ + AutoCloseFD fd = open(path.c_str(), O_RDONLY); + if (fd == -1) + throw SysError(format("opening file `%1%'") % path); + return drain ? drainFD(fd) : readFile(fd); +} + + +void writeFile(const Path & path, const string & s) +{ + AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT, 0666); + if (fd == -1) + throw SysError(format("opening file `%1%'") % path); + writeFull(fd, (unsigned char *) s.data(), s.size()); +} + + +string readLine(int fd) +{ + string s; + while (1) { + checkInterrupt(); + char ch; + ssize_t rd = read(fd, &ch, 1); + if (rd == -1) { + if (errno != EINTR) + throw SysError("reading a line"); + } else if (rd == 0) + throw EndOfFile("unexpected EOF reading a line"); + else { + if (ch == '\n') return s; + s += ch; + } + } +} + + +void writeLine(int fd, string s) +{ + s += '\n'; + writeFull(fd, (const unsigned char *) s.data(), s.size()); +} + + +static void _computePathSize(const Path & path, + unsigned long long & bytes, unsigned long long & blocks) +{ + checkInterrupt(); + + struct stat st = lstat(path); + + bytes += st.st_size; + blocks += st.st_blocks; + + if (S_ISDIR(st.st_mode)) { + Strings names = readDirectory(path); + + for (Strings::iterator i = names.begin(); i != names.end(); ++i) + _computePathSize(path + "/" + *i, bytes, blocks); + } +} + + +void computePathSize(const Path & path, + unsigned long long & bytes, unsigned long long & blocks) +{ + bytes = 0; + blocks = 0; + _computePathSize(path, bytes, blocks); +} + + +static void _deletePath(const Path & path, unsigned long long & bytesFreed) +{ + checkInterrupt(); + + printMsg(lvlVomit, format("%1%") % path); + + struct stat st = lstat(path); + + if (!S_ISDIR(st.st_mode) && st.st_nlink == 1) + bytesFreed += st.st_blocks * 512; + + if (S_ISDIR(st.st_mode)) { + Strings names = readDirectory(path); + + /* Make the directory writable. */ + if (!(st.st_mode & S_IWUSR)) { + if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1) + throw SysError(format("making `%1%' writable") % path); + } + + for (Strings::iterator i = names.begin(); i != names.end(); ++i) + _deletePath(path + "/" + *i, bytesFreed); + } + + if (remove(path.c_str()) == -1) + throw SysError(format("cannot unlink `%1%'") % path); +} + + +void deletePath(const Path & path) +{ + unsigned long long dummy; + deletePath(path, dummy); +} + + +void deletePath(const Path & path, unsigned long long & bytesFreed) +{ + startNest(nest, lvlDebug, + format("recursively deleting path `%1%'") % path); + bytesFreed = 0; + _deletePath(path, bytesFreed); +} + + +void makePathReadOnly(const Path & path) +{ + checkInterrupt(); + + struct stat st = lstat(path); + + if (!S_ISLNK(st.st_mode) && (st.st_mode & S_IWUSR)) { + if (chmod(path.c_str(), st.st_mode & ~S_IWUSR) == -1) + throw SysError(format("making `%1%' read-only") % path); + } + + if (S_ISDIR(st.st_mode)) { + Strings names = readDirectory(path); + for (Strings::iterator i = names.begin(); i != names.end(); ++i) + makePathReadOnly(path + "/" + *i); + } +} + + +static Path tempName(Path tmpRoot, const Path & prefix, bool includePid, + int & counter) +{ + tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR", "/tmp") : tmpRoot, true); + if (includePid) + return (format("%1%/%2%-%3%-%4%") % tmpRoot % prefix % getpid() % counter++).str(); + else + return (format("%1%/%2%-%3%") % tmpRoot % prefix % counter++).str(); +} + + +Path createTempDir(const Path & tmpRoot, const Path & prefix, + bool includePid, bool useGlobalCounter, mode_t mode) +{ + static int globalCounter = 0; + int localCounter = 0; + int & counter(useGlobalCounter ? globalCounter : localCounter); + + while (1) { + checkInterrupt(); + Path tmpDir = tempName(tmpRoot, prefix, includePid, counter); + if (mkdir(tmpDir.c_str(), mode) == 0) { + /* Explicitly set the group of the directory. This is to + work around around problems caused by BSD's group + ownership semantics (directories inherit the group of + the parent). For instance, the group of /tmp on + FreeBSD is "wheel", so all directories created in /tmp + will be owned by "wheel"; but if the user is not in + "wheel", then "tar" will fail to unpack archives that + have the setgid bit set on directories. */ + if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0) + throw SysError(format("setting group of directory `%1%'") % tmpDir); + return tmpDir; + } + if (errno != EEXIST) + throw SysError(format("creating directory `%1%'") % tmpDir); + } +} + + +Paths createDirs(const Path & path) +{ + Paths created; + if (path == "/") return created; + + struct stat st; + if (lstat(path.c_str(), &st) == -1) { + created = createDirs(dirOf(path)); + if (mkdir(path.c_str(), 0777) == -1 && errno != EEXIST) + throw SysError(format("creating directory `%1%'") % path); + st = lstat(path); + created.push_back(path); + } + + if (!S_ISDIR(st.st_mode)) throw Error(format("`%1%' is not a directory") % path); + + return created; +} + + +LogType logType = ltPretty; +Verbosity verbosity = lvlInfo; + +static int nestingLevel = 0; + + +Nest::Nest() +{ + nest = false; +} + + +Nest::~Nest() +{ + close(); +} + + +static string escVerbosity(Verbosity level) +{ + return int2String((int) level); +} + + +void Nest::open(Verbosity level, const format & f) +{ + if (level <= verbosity) { + if (logType == ltEscapes) + std::cerr << "\033[" << escVerbosity(level) << "p" + << f.str() << "\n"; + else + printMsg_(level, f); + nest = true; + nestingLevel++; + } +} + + +void Nest::close() +{ + if (nest) { + nestingLevel--; + if (logType == ltEscapes) + std::cerr << "\033[q"; + nest = false; + } +} + + +void printMsg_(Verbosity level, const format & f) +{ + checkInterrupt(); + if (level > verbosity) return; + string prefix; + if (logType == ltPretty) + for (int i = 0; i < nestingLevel; i++) + prefix += "| "; + else if (logType == ltEscapes && level != lvlInfo) + prefix = "\033[" + escVerbosity(level) + "s"; + string s = (format("%1%%2%\n") % prefix % f.str()).str(); + writeToStderr(s); +} + + +void warnOnce(bool & haveWarned, const format & f) +{ + if (!haveWarned) { + printMsg(lvlError, format("warning: %1%") % f.str()); + haveWarned = true; + } +} + + +void writeToStderr(const string & s) +{ + try { + _writeToStderr((const unsigned char *) s.data(), s.size()); + } catch (SysError & e) { + /* Ignore failing writes to stderr if we're in an exception + handler, otherwise throw an exception. We need to ignore + write errors in exception handlers to ensure that cleanup + code runs to completion if the other side of stderr has + been closed unexpectedly. */ + if (!std::uncaught_exception()) throw; + } +} + + +static void defaultWriteToStderr(const unsigned char * buf, size_t count) +{ + writeFull(STDERR_FILENO, buf, count); +} + + +void (*_writeToStderr) (const unsigned char * buf, size_t count) = defaultWriteToStderr; + + +void readFull(int fd, unsigned char * buf, size_t count) +{ + while (count) { + checkInterrupt(); + ssize_t res = read(fd, (char *) buf, count); + if (res == -1) { + if (errno == EINTR) continue; + throw SysError("reading from file"); + } + if (res == 0) throw EndOfFile("unexpected end-of-file"); + count -= res; + buf += res; + } +} + + +void writeFull(int fd, const unsigned char * buf, size_t count) +{ + while (count) { + checkInterrupt(); + ssize_t res = write(fd, (char *) buf, count); + if (res == -1) { + if (errno == EINTR) continue; + throw SysError("writing to file"); + } + count -= res; + buf += res; + } +} + + +string drainFD(int fd) +{ + string result; + unsigned char buffer[4096]; + while (1) { + checkInterrupt(); + ssize_t rd = read(fd, buffer, sizeof buffer); + if (rd == -1) { + if (errno != EINTR) + throw SysError("reading from file"); + } + else if (rd == 0) break; + else result.append((char *) buffer, rd); + } + return result; +} + + + +////////////////////////////////////////////////////////////////////// + + +AutoDelete::AutoDelete(const string & p, bool recursive) : path(p) +{ + del = true; + this->recursive = recursive; +} + +AutoDelete::~AutoDelete() +{ + try { + if (del) { + if (recursive) + deletePath(path); + else { + if (remove(path.c_str()) == -1) + throw SysError(format("cannot unlink `%1%'") % path); + } + } + } catch (...) { + ignoreException(); + } +} + +void AutoDelete::cancel() +{ + del = false; +} + + + +////////////////////////////////////////////////////////////////////// + + +AutoCloseFD::AutoCloseFD() +{ + fd = -1; +} + + +AutoCloseFD::AutoCloseFD(int fd) +{ + this->fd = fd; +} + + +AutoCloseFD::AutoCloseFD(const AutoCloseFD & fd) +{ + /* Copying an AutoCloseFD isn't allowed (who should get to close + it?). But as an edge case, allow copying of closed + AutoCloseFDs. This is necessary due to tiresome reasons + involving copy constructor use on default object values in STL + containers (like when you do `map[value]' where value isn't in + the map yet). */ + this->fd = fd.fd; + if (this->fd != -1) abort(); +} + + +AutoCloseFD::~AutoCloseFD() +{ + try { + close(); + } catch (...) { + ignoreException(); + } +} + + +void AutoCloseFD::operator =(int fd) +{ + if (this->fd != fd) close(); + this->fd = fd; +} + + +AutoCloseFD::operator int() const +{ + return fd; +} + + +void AutoCloseFD::close() +{ + if (fd != -1) { + if (::close(fd) == -1) + /* This should never happen. */ + throw SysError(format("closing file descriptor %1%") % fd); + fd = -1; + } +} + + +bool AutoCloseFD::isOpen() +{ + return fd != -1; +} + + +/* Pass responsibility for closing this fd to the caller. */ +int AutoCloseFD::borrow() +{ + int oldFD = fd; + fd = -1; + return oldFD; +} + + +void Pipe::create() +{ + int fds[2]; + if (pipe(fds) != 0) throw SysError("creating pipe"); + readSide = fds[0]; + writeSide = fds[1]; + closeOnExec(readSide); + closeOnExec(writeSide); +} + + + +////////////////////////////////////////////////////////////////////// + + +AutoCloseDir::AutoCloseDir() +{ + dir = 0; +} + + +AutoCloseDir::AutoCloseDir(DIR * dir) +{ + this->dir = dir; +} + + +AutoCloseDir::~AutoCloseDir() +{ + close(); +} + + +void AutoCloseDir::operator =(DIR * dir) +{ + this->dir = dir; +} + + +AutoCloseDir::operator DIR *() +{ + return dir; +} + + +void AutoCloseDir::close() +{ + if (dir) { + closedir(dir); + dir = 0; + } +} + + +////////////////////////////////////////////////////////////////////// + + +Pid::Pid() +{ + pid = -1; + separatePG = false; + killSignal = SIGKILL; +} + + +Pid::~Pid() +{ + kill(); +} + + +void Pid::operator =(pid_t pid) +{ + if (this->pid != pid) kill(); + this->pid = pid; + killSignal = SIGKILL; // reset signal to default +} + + +Pid::operator pid_t() +{ + return pid; +} + + +void Pid::kill() +{ + if (pid == -1 || pid == 0) return; + + printMsg(lvlError, format("killing process %1%") % pid); + + /* Send the requested signal to the child. If it has its own + process group, send the signal to every process in the child + process group (which hopefully includes *all* its children). */ + if (::kill(separatePG ? -pid : pid, killSignal) != 0) + printMsg(lvlError, (SysError(format("killing process %1%") % pid).msg())); + + /* Wait until the child dies, disregarding the exit status. */ + int status; + while (waitpid(pid, &status, 0) == -1) { + checkInterrupt(); + if (errno != EINTR) { + printMsg(lvlError, + (SysError(format("waiting for process %1%") % pid).msg())); + break; + } + } + + pid = -1; +} + + +int Pid::wait(bool block) +{ + assert(pid != -1); + while (1) { + int status; + int res = waitpid(pid, &status, block ? 0 : WNOHANG); + if (res == pid) { + pid = -1; + return status; + } + if (res == 0 && !block) return -1; + if (errno != EINTR) + throw SysError("cannot get child exit status"); + checkInterrupt(); + } +} + + +void Pid::setSeparatePG(bool separatePG) +{ + this->separatePG = separatePG; +} + + +void Pid::setKillSignal(int signal) +{ + this->killSignal = signal; +} + + +void killUser(uid_t uid) +{ + debug(format("killing all processes running under uid `%1%'") % uid); + + assert(uid != 0); /* just to be safe... */ + + /* The system call kill(-1, sig) sends the signal `sig' to all + users to which the current process can send signals. So we + fork a process, switch to uid, and send a mass kill. */ + + Pid pid; + pid = fork(); + switch (pid) { + + case -1: + throw SysError("unable to fork"); + + case 0: + try { /* child */ + + if (setuid(uid) == -1) + throw SysError("setting uid"); + + while (true) { +#ifdef __APPLE__ + /* OSX's kill syscall takes a third parameter that, among other + things, determines if kill(-1, signo) affects the calling + process. In the OSX libc, it's set to true, which means + "follow POSIX", which we don't want here + */ + if (syscall(SYS_kill, -1, SIGKILL, false) == 0) break; +#else + if (kill(-1, SIGKILL) == 0) break; +#endif + if (errno == ESRCH) break; /* no more processes */ + if (errno != EINTR) + throw SysError(format("cannot kill processes for uid `%1%'") % uid); + } + + } catch (std::exception & e) { + writeToStderr((format("killing processes belonging to uid `%1%': %2%\n") % uid % e.what()).str()); + _exit(1); + } + _exit(0); + } + + /* parent */ + int status = pid.wait(true); + if (status != 0) + throw Error(format("cannot kill processes for uid `%1%': %2%") % uid % statusToString(status)); + + /* !!! We should really do some check to make sure that there are + no processes left running under `uid', but there is no portable + way to do so (I think). The most reliable way may be `ps -eo + uid | grep -q $uid'. */ +} + + +////////////////////////////////////////////////////////////////////// + + +string runProgram(Path program, bool searchPath, const Strings & args) +{ + checkInterrupt(); + + std::vector<const char *> cargs; /* careful with c_str()! */ + cargs.push_back(program.c_str()); + for (Strings::const_iterator i = args.begin(); i != args.end(); ++i) + cargs.push_back(i->c_str()); + cargs.push_back(0); + + /* Create a pipe. */ + Pipe pipe; + pipe.create(); + + /* Fork. */ + Pid pid; + pid = maybeVfork(); + + switch (pid) { + + case -1: + throw SysError("unable to fork"); + + case 0: /* child */ + try { + if (dup2(pipe.writeSide, STDOUT_FILENO) == -1) + throw SysError("dupping stdout"); + + if (searchPath) + execvp(program.c_str(), (char * *) &cargs[0]); + else + execv(program.c_str(), (char * *) &cargs[0]); + throw SysError(format("executing `%1%'") % program); + + } catch (std::exception & e) { + writeToStderr("error: " + string(e.what()) + "\n"); + } + _exit(1); + } + + /* Parent. */ + + pipe.writeSide.close(); + + string result = drainFD(pipe.readSide); + + /* Wait for the child to finish. */ + int status = pid.wait(true); + if (!statusOk(status)) + throw Error(format("program `%1%' %2%") + % program % statusToString(status)); + + return result; +} + + +void closeMostFDs(const set<int> & exceptions) +{ + int maxFD = 0; + maxFD = sysconf(_SC_OPEN_MAX); + for (int fd = 0; fd < maxFD; ++fd) + if (fd != STDIN_FILENO && fd != STDOUT_FILENO && fd != STDERR_FILENO + && exceptions.find(fd) == exceptions.end()) + close(fd); /* ignore result */ +} + + +void closeOnExec(int fd) +{ + int prev; + if ((prev = fcntl(fd, F_GETFD, 0)) == -1 || + fcntl(fd, F_SETFD, prev | FD_CLOEXEC) == -1) + throw SysError("setting close-on-exec flag"); +} + + +#if HAVE_VFORK +pid_t (*maybeVfork)() = vfork; +#else +pid_t (*maybeVfork)() = fork; +#endif + + +////////////////////////////////////////////////////////////////////// + + +volatile sig_atomic_t _isInterrupted = 0; + +void _interrupted() +{ + /* Block user interrupts while an exception is being handled. + Throwing an exception while another exception is being handled + kills the program! */ + if (!std::uncaught_exception()) { + _isInterrupted = 0; + throw Interrupted("interrupted by the user"); + } +} + + + +////////////////////////////////////////////////////////////////////// + + +template<class C> C tokenizeString(const string & s, const string & separators) +{ + C result; + string::size_type pos = s.find_first_not_of(separators, 0); + while (pos != string::npos) { + string::size_type end = s.find_first_of(separators, pos + 1); + if (end == string::npos) end = s.size(); + string token(s, pos, end - pos); + result.insert(result.end(), token); + pos = s.find_first_not_of(separators, end); + } + return result; +} + +template Strings tokenizeString(const string & s, const string & separators); +template StringSet tokenizeString(const string & s, const string & separators); +template vector<string> tokenizeString(const string & s, const string & separators); + + +string concatStringsSep(const string & sep, const Strings & ss) +{ + string s; + foreach (Strings::const_iterator, i, ss) { + if (s.size() != 0) s += sep; + s += *i; + } + return s; +} + + +string concatStringsSep(const string & sep, const StringSet & ss) +{ + string s; + foreach (StringSet::const_iterator, i, ss) { + if (s.size() != 0) s += sep; + s += *i; + } + return s; +} + + +string chomp(const string & s) +{ + size_t i = s.find_last_not_of(" \n\r\t"); + return i == string::npos ? "" : string(s, 0, i + 1); +} + + +string statusToString(int status) +{ + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { + if (WIFEXITED(status)) + return (format("failed with exit code %1%") % WEXITSTATUS(status)).str(); + else if (WIFSIGNALED(status)) { + int sig = WTERMSIG(status); +#if HAVE_STRSIGNAL + const char * description = strsignal(sig); + return (format("failed due to signal %1% (%2%)") % sig % description).str(); +#else + return (format("failed due to signal %1%") % sig).str(); +#endif + } + else + return "died abnormally"; + } else return "succeeded"; +} + + +bool statusOk(int status) +{ + return WIFEXITED(status) && WEXITSTATUS(status) == 0; +} + + +bool hasSuffix(const string & s, const string & suffix) +{ + return s.size() >= suffix.size() && string(s, s.size() - suffix.size()) == suffix; +} + + +void expect(std::istream & str, const string & s) +{ + char s2[s.size()]; + str.read(s2, s.size()); + if (string(s2, s.size()) != s) + throw Error(format("expected string `%1%'") % s); +} + + +string parseString(std::istream & str) +{ + string res; + expect(str, "\""); + int c; + while ((c = str.get()) != '"') + if (c == '\\') { + c = str.get(); + if (c == 'n') res += '\n'; + else if (c == 'r') res += '\r'; + else if (c == 't') res += '\t'; + else res += c; + } + else res += c; + return res; +} + + +bool endOfList(std::istream & str) +{ + if (str.peek() == ',') { + str.get(); + return false; + } + if (str.peek() == ']') { + str.get(); + return true; + } + return false; +} + + +string decodeOctalEscaped(const string & s) +{ + string r; + for (string::const_iterator i = s.begin(); i != s.end(); ) { + if (*i != '\\') { r += *i++; continue; } + unsigned char c = 0; + ++i; + while (i != s.end() && *i >= '0' && *i < '8') + c = c * 8 + (*i++ - '0'); + r += c; + } + return r; +} + + +void ignoreException() +{ + try { + throw; + } catch (std::exception & e) { + printMsg(lvlError, format("error (ignored): %1%") % e.what()); + } +} + + +} diff --git a/src/libutil/util.hh b/src/libutil/util.hh new file mode 100644 index 000000000000..2335cbf98820 --- /dev/null +++ b/src/libutil/util.hh @@ -0,0 +1,353 @@ +#pragma once + +#include "types.hh" + +#include <sys/types.h> +#include <sys/stat.h> +#include <dirent.h> +#include <unistd.h> +#include <signal.h> + +#include <cstdio> + + +namespace nix { + + +#define foreach(it_type, it, collection) \ + for (it_type it = (collection).begin(); it != (collection).end(); ++it) + +#define foreach_reverse(it_type, it, collection) \ + for (it_type it = (collection).rbegin(); it != (collection).rend(); ++it) + + +/* Return an environment variable. */ +string getEnv(const string & key, const string & def = ""); + +/* Return an absolutized path, resolving paths relative to the + specified directory, or the current directory otherwise. The path + is also canonicalised. */ +Path absPath(Path path, Path dir = ""); + +/* Canonicalise a path by removing all `.' or `..' components and + double or trailing slashes. Optionally resolves all symlink + components such that each component of the resulting path is *not* + a symbolic link. */ +Path canonPath(const Path & path, bool resolveSymlinks = false); + +/* Return the directory part of the given canonical path, i.e., + everything before the final `/'. If the path is the root or an + immediate child thereof (e.g., `/foo'), this means an empty string + is returned. */ +Path dirOf(const Path & path); + +/* Return the base name of the given canonical path, i.e., everything + following the final `/'. */ +string baseNameOf(const Path & path); + +/* Check whether a given path is a descendant of the given + directory. */ +bool isInDir(const Path & path, const Path & dir); + +/* Get status of `path'. */ +struct stat lstat(const Path & path); + +/* Return true iff the given path exists. */ +bool pathExists(const Path & path); + +/* Read the contents (target) of a symbolic link. The result is not + in any way canonicalised. */ +Path readLink(const Path & path); + +bool isLink(const Path & path); + +/* Read the contents of a directory. The entries `.' and `..' are + removed. */ +Strings readDirectory(const Path & path); + +/* Read the contents of a file into a string. */ +string readFile(int fd); +string readFile(const Path & path, bool drain = false); + +/* Write a string to a file. */ +void writeFile(const Path & path, const string & s); + +/* Read a line from a file descriptor. */ +string readLine(int fd); + +/* Write a line to a file descriptor. */ +void writeLine(int fd, string s); + +/* Compute the sum of the sizes of all files in `path'. */ +void computePathSize(const Path & path, + unsigned long long & bytes, unsigned long long & blocks); + +/* Delete a path; i.e., in the case of a directory, it is deleted + recursively. Don't use this at home, kids. The second variant + returns the number of bytes and blocks freed. */ +void deletePath(const Path & path); + +void deletePath(const Path & path, unsigned long long & bytesFreed); + +/* Make a path read-only recursively. */ +void makePathReadOnly(const Path & path); + +/* Create a temporary directory. */ +Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix", + bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755); + +/* Create a directory and all its parents, if necessary. Returns the + list of created directories, in order of creation. */ +Paths createDirs(const Path & path); + + +template<class T, class A> +T singleton(const A & a) +{ + T t; + t.insert(a); + return t; +} + + +/* Messages. */ + + +typedef enum { + ltPretty, /* nice, nested output */ + ltEscapes, /* nesting indicated using escape codes (for log2xml) */ + ltFlat /* no nesting */ +} LogType; + +extern LogType logType; +extern Verbosity verbosity; /* suppress msgs > this */ + +class Nest +{ +private: + bool nest; +public: + Nest(); + ~Nest(); + void open(Verbosity level, const format & f); + void close(); +}; + +void printMsg_(Verbosity level, const format & f); + +#define startNest(varName, level, f) \ + Nest varName; \ + if (level <= verbosity) { \ + varName.open(level, (f)); \ + } + +#define printMsg(level, f) \ + do { \ + if (level <= verbosity) { \ + printMsg_(level, (f)); \ + } \ + } while (0) + +#define debug(f) printMsg(lvlDebug, f) + +void warnOnce(bool & haveWarned, const format & f); + +void writeToStderr(const string & s); + +extern void (*_writeToStderr) (const unsigned char * buf, size_t count); + + +/* Wrappers arount read()/write() that read/write exactly the + requested number of bytes. */ +void readFull(int fd, unsigned char * buf, size_t count); +void writeFull(int fd, const unsigned char * buf, size_t count); + +MakeError(EndOfFile, Error) + + +/* Read a file descriptor until EOF occurs. */ +string drainFD(int fd); + + + +/* Automatic cleanup of resources. */ + + +template <class T> +struct AutoDeleteArray +{ + T * p; + AutoDeleteArray(T * p) : p(p) { } + ~AutoDeleteArray() + { + delete [] p; + } +}; + + +class AutoDelete +{ + Path path; + bool del; + bool recursive; +public: + AutoDelete(const Path & p, bool recursive = true); + ~AutoDelete(); + void cancel(); +}; + + +class AutoCloseFD +{ + int fd; +public: + AutoCloseFD(); + AutoCloseFD(int fd); + AutoCloseFD(const AutoCloseFD & fd); + ~AutoCloseFD(); + void operator =(int fd); + operator int() const; + void close(); + bool isOpen(); + int borrow(); +}; + + +class Pipe +{ +public: + AutoCloseFD readSide, writeSide; + void create(); +}; + + +class AutoCloseDir +{ + DIR * dir; +public: + AutoCloseDir(); + AutoCloseDir(DIR * dir); + ~AutoCloseDir(); + void operator =(DIR * dir); + operator DIR *(); + void close(); +}; + + +class Pid +{ + pid_t pid; + bool separatePG; + int killSignal; +public: + Pid(); + ~Pid(); + void operator =(pid_t pid); + operator pid_t(); + void kill(); + int wait(bool block); + void setSeparatePG(bool separatePG); + void setKillSignal(int signal); +}; + + +/* Kill all processes running under the specified uid by sending them + a SIGKILL. */ +void killUser(uid_t uid); + + +/* Run a program and return its stdout in a string (i.e., like the + shell backtick operator). */ +string runProgram(Path program, bool searchPath = false, + const Strings & args = Strings()); + +/* Close all file descriptors except stdin, stdout, stderr, and those + listed in the given set. Good practice in child processes. */ +void closeMostFDs(const set<int> & exceptions); + +/* Set the close-on-exec flag for the given file descriptor. */ +void closeOnExec(int fd); + +/* Call vfork() if available, otherwise fork(). */ +extern pid_t (*maybeVfork)(); + + +/* User interruption. */ + +extern volatile sig_atomic_t _isInterrupted; + +void _interrupted(); + +void inline checkInterrupt() +{ + if (_isInterrupted) _interrupted(); +} + +MakeError(Interrupted, BaseError) + + +/* String tokenizer. */ +template<class C> C tokenizeString(const string & s, const string & separators = " \t\n\r"); + + +/* Concatenate the given strings with a separator between the + elements. */ +string concatStringsSep(const string & sep, const Strings & ss); +string concatStringsSep(const string & sep, const StringSet & ss); + + +/* Remove trailing whitespace from a string. */ +string chomp(const string & s); + + +/* Convert the exit status of a child as returned by wait() into an + error string. */ +string statusToString(int status); + +bool statusOk(int status); + + +/* Parse a string into an integer. */ +template<class N> bool string2Int(const string & s, N & n) +{ + std::istringstream str(s); + str >> n; + return str && str.get() == EOF; +} + +template<class N> string int2String(N n) +{ + std::ostringstream str; + str << n; + return str.str(); +} + + +/* Return true iff `s' ends in `suffix'. */ +bool hasSuffix(const string & s, const string & suffix); + + +/* Read string `s' from stream `str'. */ +void expect(std::istream & str, const string & s); + + +/* Read a C-style string from stream `str'. */ +string parseString(std::istream & str); + + +/* Utility function used to parse legacy ATerms. */ +bool endOfList(std::istream & str); + + +/* Escape a string that contains octal-encoded escape codes such as + used in /etc/fstab and /proc/mounts (e.g. "foo\040bar" decodes to + "foo bar"). */ +string decodeOctalEscaped(const string & s); + + +/* Exception handling in destructors: print an error message, then + ignore the exception. */ +void ignoreException(); + + +} diff --git a/src/libutil/xml-writer.cc b/src/libutil/xml-writer.cc new file mode 100644 index 000000000000..16dfb6ddc903 --- /dev/null +++ b/src/libutil/xml-writer.cc @@ -0,0 +1,131 @@ +#include <assert.h> + +#include "xml-writer.hh" + + +namespace nix { + + +XMLWriter::XMLWriter(bool indent, std::ostream & output) + : output(output), indent(indent) +{ + output << "<?xml version='1.0' encoding='utf-8'?>" << std::endl; + closed = false; +} + + +XMLWriter::~XMLWriter() +{ + close(); +} + + +void XMLWriter::close() +{ + if (closed) return; + while (!pendingElems.empty()) closeElement(); + closed = true; +} + + +void XMLWriter::indent_(unsigned int depth) +{ + if (!indent) return; + output << string(depth * 2, ' '); +} + + +void XMLWriter::openElement(const string & name, + const XMLAttrs & attrs) +{ + assert(!closed); + indent_(pendingElems.size()); + output << "<" << name; + writeAttrs(attrs); + output << ">"; + if (indent) output << std::endl; + pendingElems.push_back(name); +} + + +void XMLWriter::closeElement() +{ + assert(!pendingElems.empty()); + indent_(pendingElems.size() - 1); + output << "</" << pendingElems.back() << ">"; + if (indent) output << std::endl; + pendingElems.pop_back(); + if (pendingElems.empty()) closed = true; +} + + +void XMLWriter::writeEmptyElement(const string & name, + const XMLAttrs & attrs) +{ + assert(!closed); + indent_(pendingElems.size()); + output << "<" << name; + writeAttrs(attrs); + output << " />"; + if (indent) output << std::endl; +} + + +void XMLWriter::writeCharData(const string & data) +{ + assert(!pendingElems.empty()); + for (unsigned int i = 0; i < data.size(); ++i) { + char c = data[i]; + if (c == '<') output << "<"; + else if (c == '&') output << "&"; + else output << c; + } +} + + +void XMLWriter::writeAttrs(const XMLAttrs & attrs) +{ + for (XMLAttrs::const_iterator i = attrs.begin(); i != attrs.end(); ++i) { + output << " " << i->first << "=\""; + for (unsigned int j = 0; j < i->second.size(); ++j) { + char c = i->second[j]; + if (c == '"') output << """; + else if (c == '<') output << "<"; + else if (c == '>') output << ">"; + else if (c == '&') output << "&"; + /* Escape newlines to prevent attribute normalisation (see + XML spec, section 3.3.3. */ + else if (c == '\n') output << "
"; + else output << c; + } + output << "\""; + } +} + + +#if 0 +int main(int argc, char * * argv) +{ + XMLWriter doc(cout); + + // OpenElement e(doc, "foo"); + + doc.openElement("foo"); + + doc.writeCharData("dit is een test &\n"); + doc.writeCharData("<foo>\n"); + + for (int i = 0; i < 5; ++i) { + XMLAttrs attrs; + attrs["a"] = "b"; + attrs["bla"] = "<foo>'&\">"; + XMLOpenElement e(doc, "item", attrs); + doc.writeCharData("x"); + } + + return 0; +} +#endif + + +} diff --git a/src/libutil/xml-writer.hh b/src/libutil/xml-writer.hh new file mode 100644 index 000000000000..fee2eb495eaf --- /dev/null +++ b/src/libutil/xml-writer.hh @@ -0,0 +1,71 @@ +#pragma once + +#include <iostream> +#include <string> +#include <list> +#include <map> + + +namespace nix { + +using std::string; +using std::map; +using std::list; + + +typedef map<string, string> XMLAttrs; + + +class XMLWriter +{ +private: + + std::ostream & output; + + bool indent; + bool closed; + + list<string> pendingElems; + +public: + + XMLWriter(bool indent, std::ostream & output); + ~XMLWriter(); + + void close(); + + void openElement(const string & name, + const XMLAttrs & attrs = XMLAttrs()); + void closeElement(); + + void writeEmptyElement(const string & name, + const XMLAttrs & attrs = XMLAttrs()); + + void writeCharData(const string & data); + +private: + void writeAttrs(const XMLAttrs & attrs); + + void indent_(unsigned int depth); +}; + + +class XMLOpenElement +{ +private: + XMLWriter & writer; +public: + XMLOpenElement(XMLWriter & writer, const string & name, + const XMLAttrs & attrs = XMLAttrs()) + : writer(writer) + { + writer.openElement(name, attrs); + } + ~XMLOpenElement() + { + writer.closeElement(); + } +}; + + +} diff --git a/src/nix-daemon/local.mk b/src/nix-daemon/local.mk new file mode 100644 index 000000000000..db071a3f7c7e --- /dev/null +++ b/src/nix-daemon/local.mk @@ -0,0 +1,9 @@ +programs += nix-daemon + +nix-daemon_DIR := $(d) + +nix-daemon_SOURCES := $(d)/nix-daemon.cc + +nix-daemon_LIBS = libmain libstore libutil libformat + +$(eval $(call install-symlink, nix-daemon, $(bindir)/nix-worker)) diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc new file mode 100644 index 000000000000..58ef2109b313 --- /dev/null +++ b/src/nix-daemon/nix-daemon.cc @@ -0,0 +1,932 @@ +#include "shared.hh" +#include "local-store.hh" +#include "util.hh" +#include "serialise.hh" +#include "worker-protocol.hh" +#include "archive.hh" +#include "affinity.hh" +#include "globals.hh" + +#include <cstring> +#include <unistd.h> +#include <signal.h> +#include <sys/types.h> +#include <sys/wait.h> +#include <sys/stat.h> +#include <sys/socket.h> +#include <sys/un.h> +#include <fcntl.h> +#include <errno.h> + +using namespace nix; + + +/* On platforms that have O_ASYNC, we can detect when a client + disconnects and immediately kill any ongoing builds. On platforms + that lack it, we only notice the disconnection the next time we try + to write to the client. So if you have a builder that never + generates output on stdout/stderr, the daemon will never notice + that the client has disconnected until the builder terminates. + + GNU/Hurd does have O_ASYNC, but its Unix-domain socket translator + (pflocal) does not implement F_SETOWN. See + <http://lists.gnu.org/archive/html/bug-guix/2013-07/msg00021.html> for + details.*/ +#if defined O_ASYNC && !defined __GNU__ +#define HAVE_HUP_NOTIFICATION +#ifndef SIGPOLL +#define SIGPOLL SIGIO +#endif +#endif + + +static FdSource from(STDIN_FILENO); +static FdSink to(STDOUT_FILENO); + +bool canSendStderr; +pid_t myPid; + + + +/* This function is called anytime we want to write something to + stderr. If we're in a state where the protocol allows it (i.e., + when canSendStderr), send the message to the client over the + socket. */ +static void tunnelStderr(const unsigned char * buf, size_t count) +{ + /* Don't send the message to the client if we're a child of the + process handling the connection. Otherwise we could screw up + the protocol. It's up to the parent to redirect stderr and + send it to the client somehow (e.g., as in build.cc). */ + if (canSendStderr && myPid == getpid()) { + try { + writeInt(STDERR_NEXT, to); + writeString(buf, count, to); + to.flush(); + } catch (...) { + /* Write failed; that means that the other side is + gone. */ + canSendStderr = false; + throw; + } + } else + writeFull(STDERR_FILENO, buf, count); +} + + +/* Return true if the remote side has closed its end of the + connection, false otherwise. Should not be called on any socket on + which we expect input! */ +static bool isFarSideClosed(int socket) +{ + struct timeval timeout; + timeout.tv_sec = timeout.tv_usec = 0; + + fd_set fds; + FD_ZERO(&fds); + FD_SET(socket, &fds); + + while (select(socket + 1, &fds, 0, 0, &timeout) == -1) + if (errno != EINTR) throw SysError("select()"); + + if (!FD_ISSET(socket, &fds)) return false; + + /* Destructive read to determine whether the select() marked the + socket as readable because there is actual input or because + we've reached EOF (i.e., a read of size 0 is available). */ + char c; + int rd; + if ((rd = read(socket, &c, 1)) > 0) + throw Error("EOF expected (protocol error?)"); + else if (rd == -1 && errno != ECONNRESET) + throw SysError("expected connection reset or EOF"); + + return true; +} + + +/* A SIGPOLL signal is received when data is available on the client + communication socket, or when the client has closed its side of the + socket. This handler is enabled at precisely those moments in the + protocol when we're doing work and the client is supposed to be + quiet. Thus, if we get a SIGPOLL signal, it means that the client + has quit. So we should quit as well. + + Too bad most operating systems don't support the POLL_HUP value for + si_code in siginfo_t. That would make most of the SIGPOLL + complexity unnecessary, i.e., we could just enable SIGPOLL all the + time and wouldn't have to worry about races. */ +static void sigPollHandler(int sigNo) +{ + using namespace std; + try { + /* Check that the far side actually closed. We're still + getting spurious signals every once in a while. I.e., + there is no input available, but we get a signal with + POLL_IN set. Maybe it's delayed or something. */ + if (isFarSideClosed(from.fd)) { + if (!blockInt) { + _isInterrupted = 1; + blockInt = 1; + canSendStderr = false; + const char * s = "SIGPOLL\n"; + write(STDERR_FILENO, s, strlen(s)); + } + } else { + const char * s = "spurious SIGPOLL\n"; + write(STDERR_FILENO, s, strlen(s)); + } + } + catch (Error & e) { + /* Shouldn't happen. */ + string s = "impossible: " + e.msg() + '\n'; + write(STDERR_FILENO, s.data(), s.size()); + throw; + } +} + + +static void setSigPollAction(bool enable) +{ +#ifdef HAVE_HUP_NOTIFICATION + struct sigaction act, oact; + act.sa_handler = enable ? sigPollHandler : SIG_IGN; + sigfillset(&act.sa_mask); + act.sa_flags = 0; + if (sigaction(SIGPOLL, &act, &oact)) + throw SysError("setting handler for SIGPOLL"); +#endif +} + + +/* startWork() means that we're starting an operation for which we + want to send out stderr to the client. */ +static void startWork() +{ + canSendStderr = true; + + /* Handle client death asynchronously. */ + setSigPollAction(true); + + /* Of course, there is a race condition here: the socket could + have closed between when we last read from / wrote to it, and + between the time we set the handler for SIGPOLL. In that case + we won't get the signal. So do a non-blocking select() to find + out if any input is available on the socket. If there is, it + has to be the 0-byte read that indicates that the socket has + closed. */ + if (isFarSideClosed(from.fd)) { + _isInterrupted = 1; + checkInterrupt(); + } +} + + +/* stopWork() means that we're done; stop sending stderr to the + client. */ +static void stopWork(bool success = true, const string & msg = "", unsigned int status = 0) +{ + /* Stop handling async client death; we're going to a state where + we're either sending or receiving from the client, so we'll be + notified of client death anyway. */ + setSigPollAction(false); + + canSendStderr = false; + + if (success) + writeInt(STDERR_LAST, to); + else { + writeInt(STDERR_ERROR, to); + writeString(msg, to); + if (status != 0) writeInt(status, to); + } +} + + +struct TunnelSink : Sink +{ + Sink & to; + TunnelSink(Sink & to) : to(to) { } + virtual void operator () (const unsigned char * data, size_t len) + { + writeInt(STDERR_WRITE, to); + writeString(data, len, to); + } +}; + + +struct TunnelSource : BufferedSource +{ + Source & from; + TunnelSource(Source & from) : from(from) { } + size_t readUnbuffered(unsigned char * data, size_t len) + { + /* Careful: we're going to receive data from the client now, + so we have to disable the SIGPOLL handler. */ + setSigPollAction(false); + canSendStderr = false; + + writeInt(STDERR_READ, to); + writeInt(len, to); + to.flush(); + size_t n = readString(data, len, from); + + startWork(); + if (n == 0) throw EndOfFile("unexpected end-of-file"); + return n; + } +}; + + +/* If the NAR archive contains a single file at top-level, then save + the contents of the file to `s'. Otherwise barf. */ +struct RetrieveRegularNARSink : ParseSink +{ + bool regular; + string s; + + RetrieveRegularNARSink() : regular(true) { } + + void createDirectory(const Path & path) + { + regular = false; + } + + void receiveContents(unsigned char * data, unsigned int len) + { + s.append((const char *) data, len); + } + + void createSymlink(const Path & path, const string & target) + { + regular = false; + } +}; + + +/* Adapter class of a Source that saves all data read to `s'. */ +struct SavingSourceAdapter : Source +{ + Source & orig; + string s; + SavingSourceAdapter(Source & orig) : orig(orig) { } + size_t read(unsigned char * data, size_t len) + { + size_t n = orig.read(data, len); + s.append((const char *) data, n); + return n; + } +}; + + +static void performOp(bool trusted, unsigned int clientVersion, + Source & from, Sink & to, unsigned int op) +{ + switch (op) { + +#if 0 + case wopQuit: { + /* Close the database. */ + store.reset((StoreAPI *) 0); + writeInt(1, to); + break; + } +#endif + + case wopIsValidPath: { + Path path = readStorePath(from); + startWork(); + bool result = store->isValidPath(path); + stopWork(); + writeInt(result, to); + break; + } + + case wopQueryValidPaths: { + PathSet paths = readStorePaths<PathSet>(from); + startWork(); + PathSet res = store->queryValidPaths(paths); + stopWork(); + writeStrings(res, to); + break; + } + + case wopHasSubstitutes: { + Path path = readStorePath(from); + startWork(); + PathSet res = store->querySubstitutablePaths(singleton<PathSet>(path)); + stopWork(); + writeInt(res.find(path) != res.end(), to); + break; + } + + case wopQuerySubstitutablePaths: { + PathSet paths = readStorePaths<PathSet>(from); + startWork(); + PathSet res = store->querySubstitutablePaths(paths); + stopWork(); + writeStrings(res, to); + break; + } + + case wopQueryPathHash: { + Path path = readStorePath(from); + startWork(); + Hash hash = store->queryPathHash(path); + stopWork(); + writeString(printHash(hash), to); + break; + } + + case wopQueryReferences: + case wopQueryReferrers: + case wopQueryValidDerivers: + case wopQueryDerivationOutputs: { + Path path = readStorePath(from); + startWork(); + PathSet paths; + if (op == wopQueryReferences) + store->queryReferences(path, paths); + else if (op == wopQueryReferrers) + store->queryReferrers(path, paths); + else if (op == wopQueryValidDerivers) + paths = store->queryValidDerivers(path); + else paths = store->queryDerivationOutputs(path); + stopWork(); + writeStrings(paths, to); + break; + } + + case wopQueryDerivationOutputNames: { + Path path = readStorePath(from); + startWork(); + StringSet names; + names = store->queryDerivationOutputNames(path); + stopWork(); + writeStrings(names, to); + break; + } + + case wopQueryDeriver: { + Path path = readStorePath(from); + startWork(); + Path deriver = store->queryDeriver(path); + stopWork(); + writeString(deriver, to); + break; + } + + case wopQueryPathFromHashPart: { + string hashPart = readString(from); + startWork(); + Path path = store->queryPathFromHashPart(hashPart); + stopWork(); + writeString(path, to); + break; + } + + case wopAddToStore: { + string baseName = readString(from); + bool fixed = readInt(from) == 1; /* obsolete */ + bool recursive = readInt(from) == 1; + string s = readString(from); + /* Compatibility hack. */ + if (!fixed) { + s = "sha256"; + recursive = true; + } + HashType hashAlgo = parseHashType(s); + + SavingSourceAdapter savedNAR(from); + RetrieveRegularNARSink savedRegular; + + if (recursive) { + /* Get the entire NAR dump from the client and save it to + a string so that we can pass it to + addToStoreFromDump(). */ + ParseSink sink; /* null sink; just parse the NAR */ + parseDump(sink, savedNAR); + } else + parseDump(savedRegular, from); + + startWork(); + if (!savedRegular.regular) throw Error("regular file expected"); + Path path = dynamic_cast<LocalStore *>(store.get()) + ->addToStoreFromDump(recursive ? savedNAR.s : savedRegular.s, baseName, recursive, hashAlgo); + stopWork(); + + writeString(path, to); + break; + } + + case wopAddTextToStore: { + string suffix = readString(from); + string s = readString(from); + PathSet refs = readStorePaths<PathSet>(from); + startWork(); + Path path = store->addTextToStore(suffix, s, refs); + stopWork(); + writeString(path, to); + break; + } + + case wopExportPath: { + Path path = readStorePath(from); + bool sign = readInt(from) == 1; + startWork(); + TunnelSink sink(to); + store->exportPath(path, sign, sink); + stopWork(); + writeInt(1, to); + break; + } + + case wopImportPaths: { + startWork(); + TunnelSource source(from); + Paths paths = store->importPaths(true, source); + stopWork(); + writeStrings(paths, to); + break; + } + + case wopBuildPaths: { + PathSet drvs = readStorePaths<PathSet>(from); + startWork(); + store->buildPaths(drvs); + stopWork(); + writeInt(1, to); + break; + } + + case wopEnsurePath: { + Path path = readStorePath(from); + startWork(); + store->ensurePath(path); + stopWork(); + writeInt(1, to); + break; + } + + case wopAddTempRoot: { + Path path = readStorePath(from); + startWork(); + store->addTempRoot(path); + stopWork(); + writeInt(1, to); + break; + } + + case wopAddIndirectRoot: { + Path path = absPath(readString(from)); + startWork(); + store->addIndirectRoot(path); + stopWork(); + writeInt(1, to); + break; + } + + case wopSyncWithGC: { + startWork(); + store->syncWithGC(); + stopWork(); + writeInt(1, to); + break; + } + + case wopFindRoots: { + startWork(); + Roots roots = store->findRoots(); + stopWork(); + writeInt(roots.size(), to); + for (Roots::iterator i = roots.begin(); i != roots.end(); ++i) { + writeString(i->first, to); + writeString(i->second, to); + } + break; + } + + case wopCollectGarbage: { + GCOptions options; + options.action = (GCOptions::GCAction) readInt(from); + options.pathsToDelete = readStorePaths<PathSet>(from); + options.ignoreLiveness = readInt(from); + options.maxFreed = readLongLong(from); + readInt(from); // obsolete field + if (GET_PROTOCOL_MINOR(clientVersion) >= 5) { + /* removed options */ + readInt(from); + readInt(from); + } + + GCResults results; + + startWork(); + if (options.ignoreLiveness) + throw Error("you are not allowed to ignore liveness"); + store->collectGarbage(options, results); + stopWork(); + + writeStrings(results.paths, to); + writeLongLong(results.bytesFreed, to); + writeLongLong(0, to); // obsolete + + break; + } + + case wopSetOptions: { + settings.keepFailed = readInt(from) != 0; + settings.keepGoing = readInt(from) != 0; + settings.tryFallback = readInt(from) != 0; + verbosity = (Verbosity) readInt(from); + settings.maxBuildJobs = readInt(from); + settings.maxSilentTime = readInt(from); + if (GET_PROTOCOL_MINOR(clientVersion) >= 2) + settings.useBuildHook = readInt(from) != 0; + if (GET_PROTOCOL_MINOR(clientVersion) >= 4) { + settings.buildVerbosity = (Verbosity) readInt(from); + logType = (LogType) readInt(from); + settings.printBuildTrace = readInt(from) != 0; + } + if (GET_PROTOCOL_MINOR(clientVersion) >= 6) + settings.buildCores = readInt(from); + if (GET_PROTOCOL_MINOR(clientVersion) >= 10) + settings.useSubstitutes = readInt(from) != 0; + if (GET_PROTOCOL_MINOR(clientVersion) >= 12) { + unsigned int n = readInt(from); + for (unsigned int i = 0; i < n; i++) { + string name = readString(from); + string value = readString(from); + if (name == "build-timeout") + string2Int(value, settings.buildTimeout); + else + settings.set(trusted ? name : "untrusted-" + name, value); + } + } + startWork(); + stopWork(); + break; + } + + case wopQuerySubstitutablePathInfo: { + Path path = absPath(readString(from)); + startWork(); + SubstitutablePathInfos infos; + store->querySubstitutablePathInfos(singleton<PathSet>(path), infos); + stopWork(); + SubstitutablePathInfos::iterator i = infos.find(path); + if (i == infos.end()) + writeInt(0, to); + else { + writeInt(1, to); + writeString(i->second.deriver, to); + writeStrings(i->second.references, to); + writeLongLong(i->second.downloadSize, to); + if (GET_PROTOCOL_MINOR(clientVersion) >= 7) + writeLongLong(i->second.narSize, to); + } + break; + } + + case wopQuerySubstitutablePathInfos: { + PathSet paths = readStorePaths<PathSet>(from); + startWork(); + SubstitutablePathInfos infos; + store->querySubstitutablePathInfos(paths, infos); + stopWork(); + writeInt(infos.size(), to); + foreach (SubstitutablePathInfos::iterator, i, infos) { + writeString(i->first, to); + writeString(i->second.deriver, to); + writeStrings(i->second.references, to); + writeLongLong(i->second.downloadSize, to); + writeLongLong(i->second.narSize, to); + } + break; + } + + case wopQueryAllValidPaths: { + startWork(); + PathSet paths = store->queryAllValidPaths(); + stopWork(); + writeStrings(paths, to); + break; + } + + case wopQueryFailedPaths: { + startWork(); + PathSet paths = store->queryFailedPaths(); + stopWork(); + writeStrings(paths, to); + break; + } + + case wopClearFailedPaths: { + PathSet paths = readStrings<PathSet>(from); + startWork(); + store->clearFailedPaths(paths); + stopWork(); + writeInt(1, to); + break; + } + + case wopQueryPathInfo: { + Path path = readStorePath(from); + startWork(); + ValidPathInfo info = store->queryPathInfo(path); + stopWork(); + writeString(info.deriver, to); + writeString(printHash(info.hash), to); + writeStrings(info.references, to); + writeInt(info.registrationTime, to); + writeLongLong(info.narSize, to); + break; + } + + default: + throw Error(format("invalid operation %1%") % op); + } +} + + +static void processConnection(bool trusted) +{ + canSendStderr = false; + myPid = getpid(); + _writeToStderr = tunnelStderr; + +#ifdef HAVE_HUP_NOTIFICATION + /* Allow us to receive SIGPOLL for events on the client socket. */ + setSigPollAction(false); + if (fcntl(from.fd, F_SETOWN, getpid()) == -1) + throw SysError("F_SETOWN"); + if (fcntl(from.fd, F_SETFL, fcntl(from.fd, F_GETFL, 0) | O_ASYNC) == -1) + throw SysError("F_SETFL"); +#endif + + /* Exchange the greeting. */ + unsigned int magic = readInt(from); + if (magic != WORKER_MAGIC_1) throw Error("protocol mismatch"); + writeInt(WORKER_MAGIC_2, to); + writeInt(PROTOCOL_VERSION, to); + to.flush(); + unsigned int clientVersion = readInt(from); + + if (GET_PROTOCOL_MINOR(clientVersion) >= 14 && readInt(from)) + setAffinityTo(readInt(from)); + + bool reserveSpace = true; + if (GET_PROTOCOL_MINOR(clientVersion) >= 11) + reserveSpace = readInt(from) != 0; + + /* Send startup error messages to the client. */ + startWork(); + + try { + + /* If we can't accept clientVersion, then throw an error + *here* (not above). */ + +#if 0 + /* Prevent users from doing something very dangerous. */ + if (geteuid() == 0 && + querySetting("build-users-group", "") == "") + throw Error("if you run `nix-daemon' as root, then you MUST set `build-users-group'!"); +#endif + + /* Open the store. */ + store = boost::shared_ptr<StoreAPI>(new LocalStore(reserveSpace)); + + stopWork(); + to.flush(); + + } catch (Error & e) { + stopWork(false, e.msg()); + to.flush(); + return; + } + + /* Process client requests. */ + unsigned int opCount = 0; + + while (true) { + WorkerOp op; + try { + op = (WorkerOp) readInt(from); + } catch (EndOfFile & e) { + break; + } + + opCount++; + + try { + performOp(trusted, clientVersion, from, to, op); + } catch (Error & e) { + /* If we're not in a state where we can send replies, then + something went wrong processing the input of the + client. This can happen especially if I/O errors occur + during addTextToStore() / importPath(). If that + happens, just send the error message and exit. */ + bool errorAllowed = canSendStderr; + if (!errorAllowed) printMsg(lvlError, format("error processing client input: %1%") % e.msg()); + stopWork(false, e.msg(), GET_PROTOCOL_MINOR(clientVersion) >= 8 ? e.status : 0); + if (!errorAllowed) break; + } catch (std::bad_alloc & e) { + if (canSendStderr) + stopWork(false, "Nix daemon out of memory", GET_PROTOCOL_MINOR(clientVersion) >= 8 ? 1 : 0); + throw; + } + + to.flush(); + + assert(!canSendStderr); + }; + + printMsg(lvlError, format("%1% operations") % opCount); +} + + +static void sigChldHandler(int sigNo) +{ + /* Reap all dead children. */ + while (waitpid(-1, 0, WNOHANG) > 0) ; +} + + +static void setSigChldAction(bool autoReap) +{ + struct sigaction act, oact; + act.sa_handler = autoReap ? sigChldHandler : SIG_DFL; + sigfillset(&act.sa_mask); + act.sa_flags = 0; + if (sigaction(SIGCHLD, &act, &oact)) + throw SysError("setting SIGCHLD handler"); +} + + +#define SD_LISTEN_FDS_START 3 + + +static void daemonLoop() +{ + /* Get rid of children automatically; don't let them become + zombies. */ + setSigChldAction(true); + + AutoCloseFD fdSocket; + + /* Handle socket-based activation by systemd. */ + if (getEnv("LISTEN_FDS") != "") { + if (getEnv("LISTEN_PID") != int2String(getpid()) || getEnv("LISTEN_FDS") != "1") + throw Error("unexpected systemd environment variables"); + fdSocket = SD_LISTEN_FDS_START; + } + + /* Otherwise, create and bind to a Unix domain socket. */ + else { + + /* Create and bind to a Unix domain socket. */ + fdSocket = socket(PF_UNIX, SOCK_STREAM, 0); + if (fdSocket == -1) + throw SysError("cannot create Unix domain socket"); + + string socketPath = settings.nixDaemonSocketFile; + + createDirs(dirOf(socketPath)); + + /* Urgh, sockaddr_un allows path names of only 108 characters. + So chdir to the socket directory so that we can pass a + relative path name. */ + chdir(dirOf(socketPath).c_str()); + Path socketPathRel = "./" + baseNameOf(socketPath); + + struct sockaddr_un addr; + addr.sun_family = AF_UNIX; + if (socketPathRel.size() >= sizeof(addr.sun_path)) + throw Error(format("socket path `%1%' is too long") % socketPathRel); + strcpy(addr.sun_path, socketPathRel.c_str()); + + unlink(socketPath.c_str()); + + /* Make sure that the socket is created with 0666 permission + (everybody can connect --- provided they have access to the + directory containing the socket). */ + mode_t oldMode = umask(0111); + int res = bind(fdSocket, (struct sockaddr *) &addr, sizeof(addr)); + umask(oldMode); + if (res == -1) + throw SysError(format("cannot bind to socket `%1%'") % socketPath); + + chdir("/"); /* back to the root */ + + if (listen(fdSocket, 5) == -1) + throw SysError(format("cannot listen on socket `%1%'") % socketPath); + } + + closeOnExec(fdSocket); + + /* Loop accepting connections. */ + while (1) { + + try { + /* Important: the server process *cannot* open the SQLite + database, because it doesn't like forks very much. */ + assert(!store); + + /* Accept a connection. */ + struct sockaddr_un remoteAddr; + socklen_t remoteAddrLen = sizeof(remoteAddr); + + AutoCloseFD remote = accept(fdSocket, + (struct sockaddr *) &remoteAddr, &remoteAddrLen); + checkInterrupt(); + if (remote == -1) { + if (errno == EINTR) + continue; + else + throw SysError("accepting connection"); + } + + closeOnExec(remote); + + /* Get the identity of the caller, if possible. */ + uid_t clientUid = -1; + pid_t clientPid = -1; + bool trusted = false; + +#if defined(SO_PEERCRED) + ucred cred; + socklen_t credLen = sizeof(cred); + if (getsockopt(remote, SOL_SOCKET, SO_PEERCRED, &cred, &credLen) != -1) { + clientPid = cred.pid; + clientUid = cred.uid; + if (clientUid == 0) trusted = true; + } +#endif + + printMsg(lvlInfo, format("accepted connection from pid %1%, uid %2%") % clientPid % clientUid); + + /* Fork a child to handle the connection. */ + pid_t child; + child = fork(); + + switch (child) { + + case -1: + throw SysError("unable to fork"); + + case 0: + try { /* child */ + + /* Background the daemon. */ + if (setsid() == -1) + throw SysError(format("creating a new session")); + + /* Restore normal handling of SIGCHLD. */ + setSigChldAction(false); + + /* For debugging, stuff the pid into argv[1]. */ + if (clientPid != -1 && argvSaved[1]) { + string processName = int2String(clientPid); + strncpy(argvSaved[1], processName.c_str(), strlen(argvSaved[1])); + } + + /* Handle the connection. */ + from.fd = remote; + to.fd = remote; + processConnection(trusted); + + } catch (std::exception & e) { + writeToStderr("unexpected Nix daemon error: " + string(e.what()) + "\n"); + } + exit(0); + } + + } catch (Interrupted & e) { + throw; + } catch (Error & e) { + printMsg(lvlError, format("error processing connection: %1%") % e.msg()); + } + } +} + + +void run(Strings args) +{ + for (Strings::iterator i = args.begin(); i != args.end(); ) { + string arg = *i++; + if (arg == "--daemon") /* ignored for backwards compatibility */; + } + + chdir("/"); + daemonLoop(); +} + + +void printHelp() +{ + showManPage("nix-daemon"); +} + + +string programId = "nix-daemon"; diff --git a/src/nix-env/local.mk b/src/nix-env/local.mk new file mode 100644 index 000000000000..e80719cd76f7 --- /dev/null +++ b/src/nix-env/local.mk @@ -0,0 +1,7 @@ +programs += nix-env + +nix-env_DIR := $(d) + +nix-env_SOURCES := $(wildcard $(d)/*.cc) + +nix-env_LIBS = libexpr libmain libstore libutil libformat diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc new file mode 100644 index 000000000000..e2781e540bf2 --- /dev/null +++ b/src/nix-env/nix-env.cc @@ -0,0 +1,1426 @@ +#include "profiles.hh" +#include "names.hh" +#include "globals.hh" +#include "misc.hh" +#include "shared.hh" +#include "eval.hh" +#include "get-drvs.hh" +#include "attr-path.hh" +#include "common-opts.hh" +#include "xml-writer.hh" +#include "store-api.hh" +#include "user-env.hh" +#include "util.hh" +#include "value-to-json.hh" + +#include <cerrno> +#include <ctime> +#include <algorithm> +#include <iostream> +#include <sstream> + +#include <sys/types.h> +#include <sys/stat.h> +#include <unistd.h> + + +using namespace nix; +using std::cout; + + +typedef enum { + srcNixExprDrvs, + srcNixExprs, + srcStorePaths, + srcProfile, + srcAttrPath, + srcUnknown +} InstallSourceType; + + +struct InstallSourceInfo +{ + InstallSourceType type; + Path nixExprPath; /* for srcNixExprDrvs, srcNixExprs */ + Path profile; /* for srcProfile */ + string systemFilter; /* for srcNixExprDrvs */ + Bindings autoArgs; +}; + + +struct Globals +{ + InstallSourceInfo instSource; + Path profile; + EvalState state; + bool dryRun; + bool preserveInstalled; + bool removeAll; + string forceName; + bool prebuiltOnly; +}; + + +typedef void (* Operation) (Globals & globals, + Strings args, Strings opFlags, Strings opArgs); + + +void printHelp() +{ + showManPage("nix-env"); +} + + +static string needArg(Strings::iterator & i, + Strings & args, const string & arg) +{ + if (i == args.end()) throw UsageError( + format("`%1%' requires an argument") % arg); + return *i++; +} + + +static bool parseInstallSourceOptions(Globals & globals, + Strings::iterator & i, Strings & args, const string & arg) +{ + if (arg == "--from-expression" || arg == "-E") + globals.instSource.type = srcNixExprs; + else if (arg == "--from-profile") { + globals.instSource.type = srcProfile; + globals.instSource.profile = needArg(i, args, arg); + } + else if (arg == "--attr" || arg == "-A") + globals.instSource.type = srcAttrPath; + else return false; + return true; +} + + +static bool isNixExpr(const Path & path, struct stat & st) +{ + return S_ISREG(st.st_mode) || (S_ISDIR(st.st_mode) && pathExists(path + "/default.nix")); +} + + +static void getAllExprs(EvalState & state, + const Path & path, StringSet & attrs, Value & v) +{ + Strings names = readDirectory(path); + StringSet namesSorted(names.begin(), names.end()); + + foreach (StringSet::iterator, i, namesSorted) { + /* Ignore the manifest.nix used by profiles. This is + necessary to prevent it from showing up in channels (which + are implemented using profiles). */ + if (*i == "manifest.nix") continue; + + Path path2 = path + "/" + *i; + + struct stat st; + if (stat(path2.c_str(), &st) == -1) + continue; // ignore dangling symlinks in ~/.nix-defexpr + + if (isNixExpr(path2, st) && (!S_ISREG(st.st_mode) || hasSuffix(path2, ".nix"))) { + /* Strip off the `.nix' filename suffix (if applicable), + otherwise the attribute cannot be selected with the + `-A' option. Useful if you want to stick a Nix + expression directly in ~/.nix-defexpr. */ + string attrName = *i; + if (hasSuffix(attrName, ".nix")) + attrName = string(attrName, 0, attrName.size() - 4); + if (attrs.find(attrName) != attrs.end()) { + printMsg(lvlError, format("warning: name collision in input Nix expressions, skipping `%1%'") % path2); + continue; + } + attrs.insert(attrName); + /* Load the expression on demand. */ + Value & vFun(*state.allocValue()); + Value & vArg(*state.allocValue()); + state.getBuiltin("import", vFun); + mkString(vArg, path2); + mkApp(*state.allocAttr(v, state.symbols.create(attrName)), vFun, vArg); + } + else if (S_ISDIR(st.st_mode)) + /* `path2' is a directory (with no default.nix in it); + recurse into it. */ + getAllExprs(state, path2, attrs, v); + } +} + + +static void loadSourceExpr(EvalState & state, const Path & path, Value & v) +{ + struct stat st; + if (stat(path.c_str(), &st) == -1) + throw SysError(format("getting information about `%1%'") % path); + + if (isNixExpr(path, st)) { + state.evalFile(path, v); + return; + } + + /* The path is a directory. Put the Nix expressions in the + directory in a set, with the file name of each expression as + the attribute name. Recurse into subdirectories (but keep the + set flat, not nested, to make it easier for a user to have a + ~/.nix-defexpr directory that includes some system-wide + directory). */ + if (S_ISDIR(st.st_mode)) { + state.mkAttrs(v, 16); + state.mkList(*state.allocAttr(v, state.symbols.create("_combineChannels")), 0); + StringSet attrs; + getAllExprs(state, path, attrs, v); + v.attrs->sort(); + } +} + + +static void loadDerivations(EvalState & state, Path nixExprPath, + string systemFilter, Bindings & autoArgs, + const string & pathPrefix, DrvInfos & elems) +{ + Value vRoot; + loadSourceExpr(state, nixExprPath, vRoot); + + Value & v(*findAlongAttrPath(state, pathPrefix, autoArgs, vRoot)); + + getDerivations(state, v, pathPrefix, autoArgs, elems, true); + + /* Filter out all derivations not applicable to the current + system. */ + for (DrvInfos::iterator i = elems.begin(), j; i != elems.end(); i = j) { + j = i; j++; + if (systemFilter != "*" && i->system != systemFilter) + elems.erase(i); + } +} + + +static Path getHomeDir() +{ + Path homeDir(getEnv("HOME", "")); + if (homeDir == "") throw Error("HOME environment variable not set"); + return homeDir; +} + + +static Path getDefNixExprPath() +{ + return getHomeDir() + "/.nix-defexpr"; +} + + +static int getPriority(EvalState & state, DrvInfo & drv) +{ + return drv.queryMetaInt("priority", 0); +} + + +static int comparePriorities(EvalState & state, DrvInfo & drv1, DrvInfo & drv2) +{ + return getPriority(state, drv2) - getPriority(state, drv1); +} + + +// FIXME: this function is rather slow since it checks a single path +// at a time. +static bool isPrebuilt(EvalState & state, DrvInfo & elem) +{ + Path path = elem.queryOutPath(); + if (store->isValidPath(path)) return true; + PathSet ps = store->querySubstitutablePaths(singleton<PathSet>(path)); + return ps.find(path) != ps.end(); +} + + +static void checkSelectorUse(DrvNames & selectors) +{ + /* Check that all selectors have been used. */ + foreach (DrvNames::iterator, i, selectors) + if (i->hits == 0 && i->fullName != "*") + throw Error(format("selector `%1%' matches no derivations") % i->fullName); +} + + +static DrvInfos filterBySelector(EvalState & state, const DrvInfos & allElems, + const Strings & args, bool newestOnly) +{ + DrvNames selectors = drvNamesFromArgs(args); + if (selectors.empty()) + selectors.push_back(DrvName("*")); + + DrvInfos elems; + set<unsigned int> done; + + foreach (DrvNames::iterator, i, selectors) { + typedef list<std::pair<DrvInfo, unsigned int> > Matches; + Matches matches; + unsigned int n = 0; + for (DrvInfos::const_iterator j = allElems.begin(); + j != allElems.end(); ++j, ++n) + { + DrvName drvName(j->name); + if (i->matches(drvName)) { + i->hits++; + matches.push_back(std::pair<DrvInfo, unsigned int>(*j, n)); + } + } + + /* If `newestOnly', if a selector matches multiple derivations + with the same name, pick the one matching the current + system. If there are still multiple derivations, pick the + one with the highest priority. If there are still multiple + derivations, pick the one with the highest version. + Finally, if there are still multiple derivations, + arbitrarily pick the first one. */ + if (newestOnly) { + + /* Map from package names to derivations. */ + typedef map<string, std::pair<DrvInfo, unsigned int> > Newest; + Newest newest; + StringSet multiple; + + for (Matches::iterator j = matches.begin(); j != matches.end(); ++j) { + DrvName drvName(j->first.name); + int d = 1; + + Newest::iterator k = newest.find(drvName.name); + + if (k != newest.end()) { + d = j->first.system == k->second.first.system ? 0 : + j->first.system == settings.thisSystem ? 1 : + k->second.first.system == settings.thisSystem ? -1 : 0; + if (d == 0) + d = comparePriorities(state, j->first, k->second.first); + if (d == 0) + d = compareVersions(drvName.version, DrvName(k->second.first.name).version); + } + + if (d > 0) { + newest.erase(drvName.name); + newest.insert(Newest::value_type(drvName.name, *j)); + multiple.erase(j->first.name); + } else if (d == 0) { + multiple.insert(j->first.name); + } + } + + matches.clear(); + for (Newest::iterator j = newest.begin(); j != newest.end(); ++j) { + if (multiple.find(j->second.first.name) != multiple.end()) + printMsg(lvlInfo, + format("warning: there are multiple derivations named `%1%'; using the first one") + % j->second.first.name); + matches.push_back(j->second); + } + } + + /* Insert only those elements in the final list that we + haven't inserted before. */ + for (Matches::iterator j = matches.begin(); j != matches.end(); ++j) + if (done.find(j->second) == done.end()) { + done.insert(j->second); + elems.push_back(j->first); + } + } + + checkSelectorUse(selectors); + + return elems; +} + + +static bool isPath(const string & s) +{ + return s.find('/') != string::npos; +} + + +static void queryInstSources(EvalState & state, + InstallSourceInfo & instSource, const Strings & args, + DrvInfos & elems, bool newestOnly) +{ + InstallSourceType type = instSource.type; + if (type == srcUnknown && args.size() > 0 && isPath(args.front())) + type = srcStorePaths; + + switch (type) { + + /* Get the available user environment elements from the + derivations specified in a Nix expression, including only + those with names matching any of the names in `args'. */ + case srcUnknown: + case srcNixExprDrvs: { + + /* Load the derivations from the (default or specified) + Nix expression. */ + DrvInfos allElems; + loadDerivations(state, instSource.nixExprPath, + instSource.systemFilter, instSource.autoArgs, "", allElems); + + elems = filterBySelector(state, allElems, args, newestOnly); + + break; + } + + /* Get the available user environment elements from the Nix + expressions specified on the command line; these should be + functions that take the default Nix expression file as + argument, e.g., if the file is `./foo.nix', then the + argument `x: x.bar' is equivalent to `(x: x.bar) + (import ./foo.nix)' = `(import ./foo.nix).bar'. */ + case srcNixExprs: { + + Value vArg; + loadSourceExpr(state, instSource.nixExprPath, vArg); + + foreach (Strings::const_iterator, i, args) { + Expr * eFun = state.parseExprFromString(*i, absPath(".")); + Value vFun, vTmp; + state.eval(eFun, vFun); + mkApp(vTmp, vFun, vArg); + getDerivations(state, vTmp, "", instSource.autoArgs, elems, true); + } + + break; + } + + /* The available user environment elements are specified as a + list of store paths (which may or may not be + derivations). */ + case srcStorePaths: { + + foreach (Strings::const_iterator, i, args) { + Path path = followLinksToStorePath(*i); + + string name = baseNameOf(path); + string::size_type dash = name.find('-'); + if (dash != string::npos) + name = string(name, dash + 1); + + DrvInfo elem(state, name, "", "", 0); + + if (isDerivation(path)) { + elem.setDrvPath(path); + elem.setOutPath(findOutput(derivationFromPath(*store, path), "out")); + if (name.size() >= drvExtension.size() && + string(name, name.size() - drvExtension.size()) == drvExtension) + name = string(name, 0, name.size() - drvExtension.size()); + } + else elem.setOutPath(path); + + elems.push_back(elem); + } + + break; + } + + /* Get the available user environment elements from another + user environment. These are then filtered as in the + `srcNixExprDrvs' case. */ + case srcProfile: { + elems = filterBySelector(state, + queryInstalled(state, instSource.profile), + args, newestOnly); + break; + } + + case srcAttrPath: { + Value vRoot; + loadSourceExpr(state, instSource.nixExprPath, vRoot); + foreach (Strings::const_iterator, i, args) { + Value & v(*findAlongAttrPath(state, *i, instSource.autoArgs, vRoot)); + getDerivations(state, v, "", instSource.autoArgs, elems, true); + } + break; + } + } +} + + +static void printMissing(EvalState & state, DrvInfos & elems) +{ + PathSet targets; + foreach (DrvInfos::iterator, i, elems) { + Path drvPath = i->queryDrvPath(); + if (drvPath != "") + targets.insert(drvPath); + else + targets.insert(i->queryOutPath()); + } + + printMissing(*store, targets); +} + + +static bool keep(DrvInfo & drv) +{ + return drv.queryMetaBool("keep", false); +} + + +static void installDerivations(Globals & globals, + const Strings & args, const Path & profile) +{ + debug(format("installing derivations")); + + /* Get the set of user environment elements to be installed. */ + DrvInfos newElems, newElemsTmp; + queryInstSources(globals.state, globals.instSource, args, newElemsTmp, true); + + /* If --prebuilt-only is given, filter out source-only packages. */ + foreach (DrvInfos::iterator, i, newElemsTmp) + if (!globals.prebuiltOnly || isPrebuilt(globals.state, *i)) + newElems.push_back(*i); + + StringSet newNames; + for (DrvInfos::iterator i = newElems.begin(); i != newElems.end(); ++i) { + /* `forceName' is a hack to get package names right in some + one-click installs, namely those where the name used in the + path is not the one we want (e.g., `java-front' versus + `java-front-0.9pre15899'). */ + if (globals.forceName != "") + i->name = globals.forceName; + newNames.insert(DrvName(i->name).name); + } + + + while (true) { + string lockToken = optimisticLockProfile(profile); + + DrvInfos allElems(newElems); + + /* Add in the already installed derivations, unless they have + the same name as a to-be-installed element. */ + if (!globals.removeAll) { + DrvInfos installedElems = queryInstalled(globals.state, profile); + + foreach (DrvInfos::iterator, i, installedElems) { + DrvName drvName(i->name); + if (!globals.preserveInstalled && + newNames.find(drvName.name) != newNames.end() && + !keep(*i)) + printMsg(lvlInfo, format("replacing old `%1%'") % i->name); + else + allElems.push_back(*i); + } + + foreach (DrvInfos::iterator, i, newElems) + printMsg(lvlInfo, format("installing `%1%'") % i->name); + } + + printMissing(globals.state, newElems); + + if (globals.dryRun) return; + + if (createUserEnv(globals.state, allElems, + profile, settings.envKeepDerivations, lockToken)) break; + } +} + + +static void opInstall(Globals & globals, + Strings args, Strings opFlags, Strings opArgs) +{ + for (Strings::iterator i = opFlags.begin(); i != opFlags.end(); ) { + string arg = *i++; + if (parseInstallSourceOptions(globals, i, opFlags, arg)) ; + else if (arg == "--preserve-installed" || arg == "-P") + globals.preserveInstalled = true; + else if (arg == "--remove-all" || arg == "-r") + globals.removeAll = true; + else throw UsageError(format("unknown flag `%1%'") % arg); + } + + installDerivations(globals, opArgs, globals.profile); +} + + +typedef enum { utLt, utLeq, utEq, utAlways } UpgradeType; + + +static void upgradeDerivations(Globals & globals, + const Strings & args, UpgradeType upgradeType) +{ + debug(format("upgrading derivations")); + + /* Upgrade works as follows: we take all currently installed + derivations, and for any derivation matching any selector, look + for a derivation in the input Nix expression that has the same + name and a higher version number. */ + + while (true) { + string lockToken = optimisticLockProfile(globals.profile); + + DrvInfos installedElems = queryInstalled(globals.state, globals.profile); + + /* Fetch all derivations from the input file. */ + DrvInfos availElems; + queryInstSources(globals.state, globals.instSource, args, availElems, false); + + /* Go through all installed derivations. */ + DrvInfos newElems; + foreach (DrvInfos::iterator, i, installedElems) { + DrvName drvName(i->name); + + try { + + if (keep(*i)) { + newElems.push_back(*i); + continue; + } + + /* Find the derivation in the input Nix expression + with the same name that satisfies the version + constraints specified by upgradeType. If there are + multiple matches, take the one with the highest + priority. If there are still multiple matches, + take the one with the highest version. */ + DrvInfos::iterator bestElem = availElems.end(); + DrvName bestName; + foreach (DrvInfos::iterator, j, availElems) { + DrvName newName(j->name); + if (newName.name == drvName.name) { + int d = comparePriorities(globals.state, *i, *j); + if (d == 0) d = compareVersions(drvName.version, newName.version); + if ((upgradeType == utLt && d < 0) || + (upgradeType == utLeq && d <= 0) || + (upgradeType == utEq && d == 0) || + upgradeType == utAlways) + { + int d2 = -1; + if (bestElem != availElems.end()) { + d2 = comparePriorities(globals.state, *bestElem, *j); + if (d2 == 0) d2 = compareVersions(bestName.version, newName.version); + } + if (d2 < 0 && (!globals.prebuiltOnly || isPrebuilt(globals.state, *j))) { + bestElem = j; + bestName = newName; + } + } + } + } + + if (bestElem != availElems.end() && + i->queryOutPath() != + bestElem->queryOutPath()) + { + printMsg(lvlInfo, + format("upgrading `%1%' to `%2%'") + % i->name % bestElem->name); + newElems.push_back(*bestElem); + } else newElems.push_back(*i); + + } catch (Error & e) { + e.addPrefix(format("while trying to find an upgrade for `%1%':\n") % i->name); + throw; + } + } + + printMissing(globals.state, newElems); + + if (globals.dryRun) return; + + if (createUserEnv(globals.state, newElems, + globals.profile, settings.envKeepDerivations, lockToken)) break; + } +} + + +static void opUpgrade(Globals & globals, + Strings args, Strings opFlags, Strings opArgs) +{ + UpgradeType upgradeType = utLt; + for (Strings::iterator i = opFlags.begin(); i != opFlags.end(); ) { + string arg = *i++; + if (parseInstallSourceOptions(globals, i, opFlags, arg)) ; + else if (arg == "--lt") upgradeType = utLt; + else if (arg == "--leq") upgradeType = utLeq; + else if (arg == "--eq") upgradeType = utEq; + else if (arg == "--always") upgradeType = utAlways; + else throw UsageError(format("unknown flag `%1%'") % arg); + } + + upgradeDerivations(globals, opArgs, upgradeType); +} + + +static void setMetaFlag(EvalState & state, DrvInfo & drv, + const string & name, const string & value) +{ + Value * v = state.allocValue(); + mkString(*v, value.c_str()); + drv.setMeta(name, v); +} + + +static void opSetFlag(Globals & globals, + Strings args, Strings opFlags, Strings opArgs) +{ + if (opFlags.size() > 0) + throw UsageError(format("unknown flag `%1%'") % opFlags.front()); + if (opArgs.size() < 2) + throw UsageError("not enough arguments to `--set-flag'"); + + Strings::iterator arg = opArgs.begin(); + string flagName = *arg++; + string flagValue = *arg++; + DrvNames selectors = drvNamesFromArgs(Strings(arg, opArgs.end())); + + while (true) { + string lockToken = optimisticLockProfile(globals.profile); + + DrvInfos installedElems = queryInstalled(globals.state, globals.profile); + + /* Update all matching derivations. */ + foreach (DrvInfos::iterator, i, installedElems) { + DrvName drvName(i->name); + foreach (DrvNames::iterator, j, selectors) + if (j->matches(drvName)) { + printMsg(lvlInfo, format("setting flag on `%1%'") % i->name); + j->hits++; + setMetaFlag(globals.state, *i, flagName, flagValue); + break; + } + } + + checkSelectorUse(selectors); + + /* Write the new user environment. */ + if (createUserEnv(globals.state, installedElems, + globals.profile, settings.envKeepDerivations, lockToken)) break; + } +} + + +static void opSet(Globals & globals, + Strings args, Strings opFlags, Strings opArgs) +{ + for (Strings::iterator i = opFlags.begin(); i != opFlags.end(); ) { + string arg = *i++; + if (parseInstallSourceOptions(globals, i, opFlags, arg)) ; + else throw UsageError(format("unknown flag `%1%'") % arg); + } + + DrvInfos elems; + queryInstSources(globals.state, globals.instSource, opArgs, elems, true); + + if (elems.size() != 1) + throw Error("--set requires exactly one derivation"); + + DrvInfo & drv(elems.front()); + + if (drv.queryDrvPath() != "") { + PathSet paths = singleton<PathSet>(drv.queryDrvPath()); + printMissing(*store, paths); + if (globals.dryRun) return; + store->buildPaths(paths, globals.state.repair); + } + else { + printMissing(*store, singleton<PathSet>(drv.queryOutPath())); + if (globals.dryRun) return; + store->ensurePath(drv.queryOutPath()); + } + + debug(format("switching to new user environment")); + Path generation = createGeneration(globals.profile, drv.queryOutPath()); + switchLink(globals.profile, generation); +} + + +static void uninstallDerivations(Globals & globals, Strings & selectors, + Path & profile) +{ + while (true) { + string lockToken = optimisticLockProfile(profile); + + DrvInfos installedElems = queryInstalled(globals.state, profile); + DrvInfos newElems; + + foreach (DrvInfos::iterator, i, installedElems) { + DrvName drvName(i->name); + bool found = false; + foreach (Strings::iterator, j, selectors) + /* !!! the repeated calls to followLinksToStorePath() + are expensive, should pre-compute them. */ + if ((isPath(*j) && i->queryOutPath() == followLinksToStorePath(*j)) + || DrvName(*j).matches(drvName)) + { + printMsg(lvlInfo, format("uninstalling `%1%'") % i->name); + found = true; + break; + } + if (!found) newElems.push_back(*i); + } + + if (globals.dryRun) return; + + if (createUserEnv(globals.state, newElems, + profile, settings.envKeepDerivations, lockToken)) break; + } +} + + +static void opUninstall(Globals & globals, + Strings args, Strings opFlags, Strings opArgs) +{ + if (opFlags.size() > 0) + throw UsageError(format("unknown flag `%1%'") % opFlags.front()); + uninstallDerivations(globals, opArgs, globals.profile); +} + + +static bool cmpChars(char a, char b) +{ + return toupper(a) < toupper(b); +} + + +static bool cmpElemByName(const DrvInfo & a, const DrvInfo & b) +{ + return lexicographical_compare( + a.name.begin(), a.name.end(), + b.name.begin(), b.name.end(), cmpChars); +} + + +typedef list<Strings> Table; + + +void printTable(Table & table) +{ + unsigned int nrColumns = table.size() > 0 ? table.front().size() : 0; + + vector<unsigned int> widths; + widths.resize(nrColumns); + + foreach (Table::iterator, i, table) { + assert(i->size() == nrColumns); + Strings::iterator j; + unsigned int column; + for (j = i->begin(), column = 0; j != i->end(); ++j, ++column) + if (j->size() > widths[column]) widths[column] = j->size(); + } + + foreach (Table::iterator, i, table) { + Strings::iterator j; + unsigned int column; + for (j = i->begin(), column = 0; j != i->end(); ++j, ++column) { + string s = *j; + replace(s.begin(), s.end(), '\n', ' '); + cout << s; + if (column < nrColumns - 1) + cout << string(widths[column] - s.size() + 2, ' '); + } + cout << std::endl; + } +} + + +/* This function compares the version of an element against the + versions in the given set of elements. `cvLess' means that only + lower versions are in the set, `cvEqual' means that at most an + equal version is in the set, and `cvGreater' means that there is at + least one element with a higher version in the set. `cvUnavail' + means that there are no elements with the same name in the set. */ + +typedef enum { cvLess, cvEqual, cvGreater, cvUnavail } VersionDiff; + +static VersionDiff compareVersionAgainstSet( + const DrvInfo & elem, const DrvInfos & elems, string & version) +{ + DrvName name(elem.name); + + VersionDiff diff = cvUnavail; + version = "?"; + + for (DrvInfos::const_iterator i = elems.begin(); i != elems.end(); ++i) { + DrvName name2(i->name); + if (name.name == name2.name) { + int d = compareVersions(name.version, name2.version); + if (d < 0) { + diff = cvGreater; + version = name2.version; + } + else if (diff != cvGreater && d == 0) { + diff = cvEqual; + version = name2.version; + } + else if (diff != cvGreater && diff != cvEqual && d > 0) { + diff = cvLess; + if (version == "" || compareVersions(version, name2.version) < 0) + version = name2.version; + } + } + } + + return diff; +} + + +static string colorString(const string & s) +{ + if (!isatty(STDOUT_FILENO)) return s; + return "\e[1;31m" + s + "\e[0m"; +} + + +static void queryJSON(Globals & globals, vector<DrvInfo> & elems) +{ + JSONObject topObj(cout); + foreach (vector<DrvInfo>::iterator, i, elems) { + topObj.attr(i->attrPath); + JSONObject pkgObj(cout); + + pkgObj.attr("name", i->name); + pkgObj.attr("system", i->system); + + pkgObj.attr("meta"); + JSONObject metaObj(cout); + StringSet metaNames = i->queryMetaNames(); + foreach (StringSet::iterator, j, metaNames) { + metaObj.attr(*j); + Value * v = i->queryMeta(*j); + if (!v) + printMsg(lvlError, format("derivation `%1%' has invalid meta attribute `%2%'") % i->name % *j); + else { + PathSet context; + printValueAsJSON(globals.state, true, *v, cout, context); + } + } + } +} + + +static void opQuery(Globals & globals, + Strings args, Strings opFlags, Strings opArgs) +{ + Strings remaining; + string attrPath; + + bool printStatus = false; + bool printName = true; + bool printAttrPath = false; + bool printSystem = false; + bool printDrvPath = false; + bool printOutPath = false; + bool printDescription = false; + bool printMeta = false; + bool compareVersions = false; + bool xmlOutput = false; + bool jsonOutput = false; + + enum { sInstalled, sAvailable } source = sInstalled; + + settings.readOnlyMode = true; /* makes evaluation a bit faster */ + + for (Strings::iterator i = args.begin(); i != args.end(); ) { + string arg = *i++; + if (arg == "--status" || arg == "-s") printStatus = true; + else if (arg == "--no-name") printName = false; + else if (arg == "--system") printSystem = true; + else if (arg == "--description") printDescription = true; + else if (arg == "--compare-versions" || arg == "-c") compareVersions = true; + else if (arg == "--drv-path") printDrvPath = true; + else if (arg == "--out-path") printOutPath = true; + else if (arg == "--meta") printMeta = true; + else if (arg == "--installed") source = sInstalled; + else if (arg == "--available" || arg == "-a") source = sAvailable; + else if (arg == "--xml") xmlOutput = true; + else if (arg == "--json") jsonOutput = true; + else if (arg == "--attr-path" || arg == "-P") printAttrPath = true; + else if (arg == "--attr" || arg == "-A") + attrPath = needArg(i, args, arg); + else if (arg[0] == '-') + throw UsageError(format("unknown flag `%1%'") % arg); + else remaining.push_back(arg); + } + + + /* Obtain derivation information from the specified source. */ + DrvInfos availElems, installedElems; + + if (source == sInstalled || compareVersions || printStatus) + installedElems = queryInstalled(globals.state, globals.profile); + + if (source == sAvailable || compareVersions) + loadDerivations(globals.state, globals.instSource.nixExprPath, + globals.instSource.systemFilter, globals.instSource.autoArgs, + attrPath, availElems); + + DrvInfos elems_ = filterBySelector(globals.state, + source == sInstalled ? installedElems : availElems, + remaining, false); + + DrvInfos & otherElems(source == sInstalled ? availElems : installedElems); + + + /* Sort them by name. */ + /* !!! */ + vector<DrvInfo> elems; + for (DrvInfos::iterator i = elems_.begin(); i != elems_.end(); ++i) + elems.push_back(*i); + sort(elems.begin(), elems.end(), cmpElemByName); + + + /* We only need to know the installed paths when we are querying + the status of the derivation. */ + PathSet installed; /* installed paths */ + + if (printStatus) { + for (DrvInfos::iterator i = installedElems.begin(); + i != installedElems.end(); ++i) + installed.insert(i->queryOutPath()); + } + + + /* Query which paths have substitutes. */ + PathSet validPaths, substitutablePaths; + if (printStatus || globals.prebuiltOnly) { + PathSet paths; + foreach (vector<DrvInfo>::iterator, i, elems) + try { + paths.insert(i->queryOutPath()); + } catch (AssertionError & e) { + printMsg(lvlTalkative, format("skipping derivation named `%1%' which gives an assertion failure") % i->name); + i->setFailed(); + } + validPaths = store->queryValidPaths(paths); + substitutablePaths = store->querySubstitutablePaths(paths); + } + + + /* Print the desired columns, or XML output. */ + if (jsonOutput) { + queryJSON(globals, elems); + return; + } + + Table table; + std::ostringstream dummy; + XMLWriter xml(true, *(xmlOutput ? &cout : &dummy)); + XMLOpenElement xmlRoot(xml, "items"); + + foreach (vector<DrvInfo>::iterator, i, elems) { + try { + if (i->hasFailed()) continue; + + startNest(nest, lvlDebug, format("outputting query result `%1%'") % i->attrPath); + + if (globals.prebuiltOnly && + validPaths.find(i->queryOutPath()) == validPaths.end() && + substitutablePaths.find(i->queryOutPath()) == substitutablePaths.end()) + continue; + + /* For table output. */ + Strings columns; + + /* For XML output. */ + XMLAttrs attrs; + + if (printStatus) { + Path outPath = i->queryOutPath(); + bool hasSubs = substitutablePaths.find(outPath) != substitutablePaths.end(); + bool isInstalled = installed.find(outPath) != installed.end(); + bool isValid = validPaths.find(outPath) != validPaths.end(); + if (xmlOutput) { + attrs["installed"] = isInstalled ? "1" : "0"; + attrs["valid"] = isValid ? "1" : "0"; + attrs["substitutable"] = hasSubs ? "1" : "0"; + } else + columns.push_back( + (string) (isInstalled ? "I" : "-") + + (isValid ? "P" : "-") + + (hasSubs ? "S" : "-")); + } + + if (xmlOutput) + attrs["attrPath"] = i->attrPath; + else if (printAttrPath) + columns.push_back(i->attrPath); + + if (xmlOutput) + attrs["name"] = i->name; + else if (printName) + columns.push_back(i->name); + + if (compareVersions) { + /* Compare this element against the versions of the + same named packages in either the set of available + elements, or the set of installed elements. !!! + This is O(N * M), should be O(N * lg M). */ + string version; + VersionDiff diff = compareVersionAgainstSet(*i, otherElems, version); + + char ch; + switch (diff) { + case cvLess: ch = '>'; break; + case cvEqual: ch = '='; break; + case cvGreater: ch = '<'; break; + case cvUnavail: ch = '-'; break; + default: abort(); + } + + if (xmlOutput) { + if (diff != cvUnavail) { + attrs["versionDiff"] = ch; + attrs["maxComparedVersion"] = version; + } + } else { + string column = (string) "" + ch + " " + version; + if (diff == cvGreater) column = colorString(column); + columns.push_back(column); + } + } + + if (xmlOutput) { + if (i->system != "") attrs["system"] = i->system; + } + else if (printSystem) + columns.push_back(i->system); + + if (printDrvPath) { + string drvPath = i->queryDrvPath(); + if (xmlOutput) { + if (drvPath != "") attrs["drvPath"] = drvPath; + } else + columns.push_back(drvPath == "" ? "-" : drvPath); + } + + if (printOutPath && !xmlOutput) { + DrvInfo::Outputs outputs = i->queryOutputs(); + string s; + foreach (DrvInfo::Outputs::iterator, j, outputs) { + if (!s.empty()) s += ';'; + if (j->first != "out") { s += j->first; s += "="; } + s += j->second; + } + columns.push_back(s); + } + + if (printDescription) { + string descr = i->queryMetaString("description"); + if (xmlOutput) { + if (descr != "") attrs["description"] = descr; + } else + columns.push_back(descr); + } + + if (xmlOutput) { + if (printOutPath || printMeta) { + XMLOpenElement item(xml, "item", attrs); + if (printOutPath) { + DrvInfo::Outputs outputs = i->queryOutputs(); + foreach (DrvInfo::Outputs::iterator, j, outputs) { + XMLAttrs attrs2; + attrs2["name"] = j->first; + attrs2["path"] = j->second; + xml.writeEmptyElement("output", attrs2); + } + } + if (printMeta) { + StringSet metaNames = i->queryMetaNames(); + foreach (StringSet::iterator, j, metaNames) { + XMLAttrs attrs2; + attrs2["name"] = *j; + Value * v = i->queryMeta(*j); + if (!v) + printMsg(lvlError, format("derivation `%1%' has invalid meta attribute `%2%'") % i->name % *j); + else { + if (v->type == tString) { + attrs2["type"] = "string"; + attrs2["value"] = v->string.s; + xml.writeEmptyElement("meta", attrs2); + } else if (v->type == tInt) { + attrs2["type"] = "int"; + attrs2["value"] = (format("%1%") % v->integer).str(); + xml.writeEmptyElement("meta", attrs2); + } else if (v->type == tBool) { + attrs2["type"] = "bool"; + attrs2["value"] = v->boolean ? "true" : "false"; + xml.writeEmptyElement("meta", attrs2); + } else if (v->type == tList) { + attrs2["type"] = "strings"; + XMLOpenElement m(xml, "meta", attrs2); + for (unsigned int j = 0; j < v->list.length; ++j) { + if (v->list.elems[j]->type != tString) continue; + XMLAttrs attrs3; + attrs3["value"] = v->list.elems[j]->string.s; + xml.writeEmptyElement("string", attrs3); + } + } + } + } + } + } else + xml.writeEmptyElement("item", attrs); + } else + table.push_back(columns); + + cout.flush(); + + } catch (AssertionError & e) { + printMsg(lvlTalkative, format("skipping derivation named `%1%' which gives an assertion failure") % i->name); + } catch (Error & e) { + e.addPrefix(format("while querying the derivation named `%1%':\n") % i->name); + throw; + } + } + + if (!xmlOutput) printTable(table); +} + + +static void opSwitchProfile(Globals & globals, + Strings args, Strings opFlags, Strings opArgs) +{ + if (opFlags.size() > 0) + throw UsageError(format("unknown flag `%1%'") % opFlags.front()); + if (opArgs.size() != 1) + throw UsageError(format("exactly one argument expected")); + + Path profile = absPath(opArgs.front()); + Path profileLink = getHomeDir() + "/.nix-profile"; + + switchLink(profileLink, profile); +} + + +static const int prevGen = -2; + + +static void switchGeneration(Globals & globals, int dstGen) +{ + PathLocks lock; + lockProfile(lock, globals.profile); + + int curGen; + Generations gens = findGenerations(globals.profile, curGen); + + Generation dst; + for (Generations::iterator i = gens.begin(); i != gens.end(); ++i) + if ((dstGen == prevGen && i->number < curGen) || + (dstGen >= 0 && i->number == dstGen)) + dst = *i; + + if (!dst) { + if (dstGen == prevGen) + throw Error(format("no generation older than the current (%1%) exists") + % curGen); + else + throw Error(format("generation %1% does not exist") % dstGen); + } + + printMsg(lvlInfo, format("switching from generation %1% to %2%") + % curGen % dst.number); + + if (globals.dryRun) return; + + switchLink(globals.profile, dst.path); +} + + +static void opSwitchGeneration(Globals & globals, + Strings args, Strings opFlags, Strings opArgs) +{ + if (opFlags.size() > 0) + throw UsageError(format("unknown flag `%1%'") % opFlags.front()); + if (opArgs.size() != 1) + throw UsageError(format("exactly one argument expected")); + + int dstGen; + if (!string2Int(opArgs.front(), dstGen)) + throw UsageError(format("expected a generation number")); + + switchGeneration(globals, dstGen); +} + + +static void opRollback(Globals & globals, + Strings args, Strings opFlags, Strings opArgs) +{ + if (opFlags.size() > 0) + throw UsageError(format("unknown flag `%1%'") % opFlags.front()); + if (opArgs.size() != 0) + throw UsageError(format("no arguments expected")); + + switchGeneration(globals, prevGen); +} + + +static void opListGenerations(Globals & globals, + Strings args, Strings opFlags, Strings opArgs) +{ + if (opFlags.size() > 0) + throw UsageError(format("unknown flag `%1%'") % opFlags.front()); + if (opArgs.size() != 0) + throw UsageError(format("no arguments expected")); + + PathLocks lock; + lockProfile(lock, globals.profile); + + int curGen; + Generations gens = findGenerations(globals.profile, curGen); + + for (Generations::iterator i = gens.begin(); i != gens.end(); ++i) { + tm t; + if (!localtime_r(&i->creationTime, &t)) throw Error("cannot convert time"); + cout << format("%|4| %|4|-%|02|-%|02| %|02|:%|02|:%|02| %||\n") + % i->number + % (t.tm_year + 1900) % (t.tm_mon + 1) % t.tm_mday + % t.tm_hour % t.tm_min % t.tm_sec + % (i->number == curGen ? "(current)" : ""); + } +} + + +static void deleteGeneration2(Globals & globals, unsigned int gen) +{ + if (globals.dryRun) + printMsg(lvlInfo, format("would remove generation %1%") % gen); + else { + printMsg(lvlInfo, format("removing generation %1%") % gen); + deleteGeneration(globals.profile, gen); + } + +} + + +static void opDeleteGenerations(Globals & globals, + Strings args, Strings opFlags, Strings opArgs) +{ + if (opFlags.size() > 0) + throw UsageError(format("unknown flag `%1%'") % opFlags.front()); + + PathLocks lock; + lockProfile(lock, globals.profile); + + int curGen; + Generations gens = findGenerations(globals.profile, curGen); + + for (Strings::iterator i = opArgs.begin(); i != opArgs.end(); ++i) { + + if (*i == "old") { + for (Generations::iterator j = gens.begin(); j != gens.end(); ++j) + if (j->number != curGen) + deleteGeneration2(globals, j->number); + } + + else { + int n; + if (!string2Int(*i, n) || n < 0) + throw UsageError(format("invalid generation specifier `%1%'") % *i); + bool found = false; + for (Generations::iterator j = gens.begin(); j != gens.end(); ++j) { + if (j->number == n) { + deleteGeneration2(globals, j->number); + found = true; + break; + } + } + if (!found) + printMsg(lvlError, format("generation %1% does not exist") % n); + } + } +} + + +void run(Strings args) +{ + Strings opFlags, opArgs, remaining; + Operation op = 0; + + Globals globals; + + globals.instSource.type = srcUnknown; + globals.instSource.nixExprPath = getDefNixExprPath(); + globals.instSource.systemFilter = "*"; + + globals.dryRun = false; + globals.preserveInstalled = false; + globals.removeAll = false; + globals.prebuiltOnly = false; + + for (Strings::iterator i = args.begin(); i != args.end(); ) { + string arg = *i++; + + Operation oldOp = op; + + if (arg == "--install" || arg == "-i") + op = opInstall; + else if (parseOptionArg(arg, i, args.end(), + globals.state, globals.instSource.autoArgs)) + ; + else if (parseSearchPathArg(arg, i, args.end(), globals.state)) + ; + else if (arg == "--force-name") // undocumented flag for nix-install-package + globals.forceName = needArg(i, args, arg); + else if (arg == "--uninstall" || arg == "-e") + op = opUninstall; + else if (arg == "--upgrade" || arg == "-u") + op = opUpgrade; + else if (arg == "--set-flag") + op = opSetFlag; + else if (arg == "--set") + op = opSet; + else if (arg == "--query" || arg == "-q") + op = opQuery; + else if (arg == "--profile" || arg == "-p") + globals.profile = absPath(needArg(i, args, arg)); + else if (arg == "--file" || arg == "-f") + globals.instSource.nixExprPath = lookupFileArg(globals.state, needArg(i, args, arg)); + else if (arg == "--switch-profile" || arg == "-S") + op = opSwitchProfile; + else if (arg == "--switch-generation" || arg == "-G") + op = opSwitchGeneration; + else if (arg == "--rollback") + op = opRollback; + else if (arg == "--list-generations") + op = opListGenerations; + else if (arg == "--delete-generations") + op = opDeleteGenerations; + else if (arg == "--dry-run") { + printMsg(lvlInfo, "(dry run; not doing anything)"); + globals.dryRun = true; + } + else if (arg == "--system-filter") + globals.instSource.systemFilter = needArg(i, args, arg); + else if (arg == "--prebuilt-only" || arg == "-b") + globals.prebuiltOnly = true; + else if (arg == "--repair") + globals.state.repair = true; + else { + remaining.push_back(arg); + if (arg[0] == '-') { + opFlags.push_back(arg); + if (arg == "--from-profile") { /* !!! hack */ + if (i != args.end()) opFlags.push_back(*i++); + } + } else opArgs.push_back(arg); + } + + if (oldOp && oldOp != op) + throw UsageError("only one operation may be specified"); + } + + if (!op) throw UsageError("no operation specified"); + + if (globals.profile == "") + globals.profile = getEnv("NIX_PROFILE", ""); + + if (globals.profile == "") { + Path profileLink = getHomeDir() + "/.nix-profile"; + globals.profile = pathExists(profileLink) + ? absPath(readLink(profileLink), dirOf(profileLink)) + : canonPath(settings.nixStateDir + "/profiles/default"); + } + + store = openStore(); + + op(globals, remaining, opFlags, opArgs); + + globals.state.printStats(); +} + + +string programId = "nix-env"; diff --git a/src/nix-env/profiles.cc b/src/nix-env/profiles.cc new file mode 100644 index 000000000000..c327993594e8 --- /dev/null +++ b/src/nix-env/profiles.cc @@ -0,0 +1,148 @@ +#include "profiles.hh" +#include "store-api.hh" +#include "util.hh" + +#include <sys/types.h> +#include <sys/stat.h> +#include <unistd.h> +#include <errno.h> +#include <stdio.h> + + +namespace nix { + + +static bool cmpGensByNumber(const Generation & a, const Generation & b) +{ + return a.number < b.number; +} + + +/* Parse a generation name of the format + `<profilename>-<number>-link'. */ +static int parseName(const string & profileName, const string & name) +{ + if (string(name, 0, profileName.size() + 1) != profileName + "-") return -1; + string s = string(name, profileName.size() + 1); + string::size_type p = s.find("-link"); + if (p == string::npos) return -1; + int n; + if (string2Int(string(s, 0, p), n) && n >= 0) + return n; + else + return -1; +} + + + +Generations findGenerations(Path profile, int & curGen) +{ + Generations gens; + + Path profileDir = dirOf(profile); + string profileName = baseNameOf(profile); + + Strings names = readDirectory(profileDir); + for (Strings::iterator i = names.begin(); i != names.end(); ++i) { + int n; + if ((n = parseName(profileName, *i)) != -1) { + Generation gen; + gen.path = profileDir + "/" + *i; + gen.number = n; + struct stat st; + if (lstat(gen.path.c_str(), &st) != 0) + throw SysError(format("statting `%1%'") % gen.path); + gen.creationTime = st.st_mtime; + gens.push_back(gen); + } + } + + gens.sort(cmpGensByNumber); + + curGen = pathExists(profile) + ? parseName(profileName, readLink(profile)) + : -1; + + return gens; +} + + +static void makeName(const Path & profile, unsigned int num, + Path & outLink) +{ + Path prefix = (format("%1%-%2%") % profile % num).str(); + outLink = prefix + "-link"; +} + + +Path createGeneration(Path profile, Path outPath) +{ + /* The new generation number should be higher than old the + previous ones. */ + int dummy; + Generations gens = findGenerations(profile, dummy); + unsigned int num = gens.size() > 0 ? gens.back().number : 0; + + /* Create the new generation. Note that addPermRoot() blocks if + the garbage collector is running to prevent the stuff we've + built from moving from the temporary roots (which the GC knows) + to the permanent roots (of which the GC would have a stale + view). If we didn't do it this way, the GC might remove the + user environment etc. we've just built. */ + Path generation; + makeName(profile, num + 1, generation); + addPermRoot(*store, outPath, generation, false, true); + + return generation; +} + + +static void removeFile(const Path & path) +{ + if (remove(path.c_str()) == -1) + throw SysError(format("cannot unlink `%1%'") % path); +} + + +void deleteGeneration(const Path & profile, unsigned int gen) +{ + Path generation; + makeName(profile, gen, generation); + removeFile(generation); +} + + +void switchLink(Path link, Path target) +{ + /* Hacky. */ + if (dirOf(target) == dirOf(link)) target = baseNameOf(target); + + Path tmp = canonPath(dirOf(link) + "/.new_" + baseNameOf(link)); + if (symlink(target.c_str(), tmp.c_str()) != 0) + throw SysError(format("creating symlink `%1%'") % tmp); + /* The rename() system call is supposed to be essentially atomic + on Unix. That is, if we have links `current -> X' and + `new_current -> Y', and we rename new_current to current, a + process accessing current will see X or Y, but never a + file-not-found or other error condition. This is sufficient to + atomically switch user environments. */ + if (rename(tmp.c_str(), link.c_str()) != 0) + throw SysError(format("renaming `%1%' to `%2%'") % tmp % link); +} + + +void lockProfile(PathLocks & lock, const Path & profile) +{ + lock.lockPaths(singleton<PathSet>(profile), + (format("waiting for lock on profile `%1%'") % profile).str()); + lock.setDeletion(true); +} + + +string optimisticLockProfile(const Path & profile) +{ + return pathExists(profile) ? readLink(profile) : ""; +} + + +} diff --git a/src/nix-env/profiles.hh b/src/nix-env/profiles.hh new file mode 100644 index 000000000000..30d2376d998c --- /dev/null +++ b/src/nix-env/profiles.hh @@ -0,0 +1,55 @@ +#pragma once + +#include "types.hh" +#include "pathlocks.hh" + +#include <time.h> + + +namespace nix { + + +struct Generation +{ + int number; + Path path; + time_t creationTime; + Generation() + { + number = -1; + } + operator bool() const + { + return number != -1; + } +}; + +typedef list<Generation> Generations; + + +/* Returns the list of currently present generations for the specified + profile, sorted by generation number. */ +Generations findGenerations(Path profile, int & curGen); + +Path createGeneration(Path profile, Path outPath); + +void deleteGeneration(const Path & profile, unsigned int gen); + +void switchLink(Path link, Path target); + +/* Ensure exclusive access to a profile. Any command that modifies + the profile first acquires this lock. */ +void lockProfile(PathLocks & lock, const Path & profile); + +/* Optimistic locking is used by long-running operations like `nix-env + -i'. Instead of acquiring the exclusive lock for the entire + duration of the operation, we just perform the operation + optimistically (without an exclusive lock), and check at the end + whether the profile changed while we were busy (i.e., the symlink + target changed). If so, the operation is restarted. Restarting is + generally cheap, since the build results are still in the Nix + store. Most of the time, only the user environment has to be + rebuilt. */ +string optimisticLockProfile(const Path & profile); + +} diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc new file mode 100644 index 000000000000..3a73f2647a6b --- /dev/null +++ b/src/nix-env/user-env.cc @@ -0,0 +1,148 @@ +#include "user-env.hh" +#include "util.hh" +#include "derivations.hh" +#include "store-api.hh" +#include "globals.hh" +#include "shared.hh" +#include "eval.hh" +#include "eval-inline.hh" +#include "profiles.hh" + + +namespace nix { + + +DrvInfos queryInstalled(EvalState & state, const Path & userEnv) +{ + DrvInfos elems; + Path manifestFile = userEnv + "/manifest.nix"; + if (pathExists(manifestFile)) { + Value v; + state.evalFile(manifestFile, v); + Bindings bindings; + getDerivations(state, v, "", bindings, elems, false); + } + return elems; +} + + +bool createUserEnv(EvalState & state, DrvInfos & elems, + const Path & profile, bool keepDerivations, + const string & lockToken) +{ + /* Build the components in the user environment, if they don't + exist already. */ + PathSet drvsToBuild; + foreach (DrvInfos::iterator, i, elems) + if (i->queryDrvPath() != "") + drvsToBuild.insert(i->queryDrvPath()); + + debug(format("building user environment dependencies")); + store->buildPaths(drvsToBuild, state.repair); + + /* Construct the whole top level derivation. */ + PathSet references; + Value manifest; + state.mkList(manifest, elems.size()); + unsigned int n = 0; + foreach (DrvInfos::iterator, i, elems) { + /* Create a pseudo-derivation containing the name, system, + output paths, and optionally the derivation path, as well + as the meta attributes. */ + Path drvPath = keepDerivations ? i->queryDrvPath() : ""; + + Value & v(*state.allocValue()); + manifest.list.elems[n++] = &v; + state.mkAttrs(v, 16); + + mkString(*state.allocAttr(v, state.sType), "derivation"); + mkString(*state.allocAttr(v, state.sName), i->name); + if (!i->system.empty()) + mkString(*state.allocAttr(v, state.sSystem), i->system); + mkString(*state.allocAttr(v, state.sOutPath), i->queryOutPath()); + if (drvPath != "") + mkString(*state.allocAttr(v, state.sDrvPath), i->queryDrvPath()); + + // Copy each output. + DrvInfo::Outputs outputs = i->queryOutputs(); + Value & vOutputs = *state.allocAttr(v, state.sOutputs); + state.mkList(vOutputs, outputs.size()); + unsigned int m = 0; + foreach (DrvInfo::Outputs::iterator, j, outputs) { + mkString(*(vOutputs.list.elems[m++] = state.allocValue()), j->first); + Value & vOutputs = *state.allocAttr(v, state.symbols.create(j->first)); + state.mkAttrs(vOutputs, 2); + mkString(*state.allocAttr(vOutputs, state.sOutPath), j->second); + + /* This is only necessary when installing store paths, e.g., + `nix-env -i /nix/store/abcd...-foo'. */ + store->addTempRoot(j->second); + store->ensurePath(j->second); + + references.insert(j->second); + } + + // Copy the meta attributes. + Value & vMeta = *state.allocAttr(v, state.sMeta); + state.mkAttrs(vMeta, 16); + StringSet metaNames = i->queryMetaNames(); + foreach (StringSet::iterator, j, metaNames) { + Value * v = i->queryMeta(*j); + if (!v) continue; + vMeta.attrs->push_back(Attr(state.symbols.create(*j), v)); + } + v.attrs->sort(); + + if (drvPath != "") references.insert(drvPath); + } + + /* Also write a copy of the list of user environment elements to + the store; we need it for future modifications of the + environment. */ + Path manifestFile = store->addTextToStore("env-manifest.nix", + (format("%1%") % manifest).str(), references); + + /* Get the environment builder expression. */ + Value envBuilder; + state.evalFile(state.findFile("nix/buildenv.nix"), envBuilder); + + /* Construct a Nix expression that calls the user environment + builder with the manifest as argument. */ + Value args, topLevel; + state.mkAttrs(args, 3); + mkString(*state.allocAttr(args, state.symbols.create("manifest")), + manifestFile, singleton<PathSet>(manifestFile)); + args.attrs->push_back(Attr(state.symbols.create("derivations"), &manifest)); + args.attrs->sort(); + mkApp(topLevel, envBuilder, args); + + /* Evaluate it. */ + debug("evaluating user environment builder"); + state.forceValue(topLevel); + PathSet context; + Path topLevelDrv = state.coerceToPath(*topLevel.attrs->find(state.sDrvPath)->value, context); + Path topLevelOut = state.coerceToPath(*topLevel.attrs->find(state.sOutPath)->value, context); + + /* Realise the resulting store expression. */ + debug("building user environment"); + store->buildPaths(singleton<PathSet>(topLevelDrv), state.repair); + + /* Switch the current user environment to the output path. */ + PathLocks lock; + lockProfile(lock, profile); + + Path lockTokenCur = optimisticLockProfile(profile); + if (lockToken != lockTokenCur) { + printMsg(lvlError, format("profile `%1%' changed while we were busy; restarting") % profile); + return false; + } + + debug(format("switching to new user environment")); + Path generation = createGeneration(profile, topLevelOut); + switchLink(profile, generation); + + return true; +} + + +} diff --git a/src/nix-env/user-env.hh b/src/nix-env/user-env.hh new file mode 100644 index 000000000000..f188efe9b4a9 --- /dev/null +++ b/src/nix-env/user-env.hh @@ -0,0 +1,13 @@ +#pragma once + +#include "get-drvs.hh" + +namespace nix { + +DrvInfos queryInstalled(EvalState & state, const Path & userEnv); + +bool createUserEnv(EvalState & state, DrvInfos & elems, + const Path & profile, bool keepDerivations, + const string & lockToken); + +} diff --git a/src/nix-hash/local.mk b/src/nix-hash/local.mk new file mode 100644 index 000000000000..7c290ca8466e --- /dev/null +++ b/src/nix-hash/local.mk @@ -0,0 +1,7 @@ +programs += nix-hash + +nix-hash_DIR := $(d) + +nix-hash_SOURCES := $(d)/nix-hash.cc + +nix-hash_LIBS = libmain libstore libutil libformat diff --git a/src/nix-hash/nix-hash.cc b/src/nix-hash/nix-hash.cc new file mode 100644 index 000000000000..af3dda4ad7b8 --- /dev/null +++ b/src/nix-hash/nix-hash.cc @@ -0,0 +1,64 @@ +#include "hash.hh" +#include "shared.hh" + +#include <iostream> + + +using namespace nix; + + +void printHelp() +{ + showManPage("nix-hash"); +} + + +void run(Strings args) +{ + HashType ht = htMD5; + bool flat = false; + bool base32 = false; + bool truncate = false; + enum { opHash, opTo32, opTo16 } op = opHash; + + Strings ss; + + for (Strings::iterator i = args.begin(); + i != args.end(); i++) + { + if (*i == "--flat") flat = true; + else if (*i == "--base32") base32 = true; + else if (*i == "--truncate") truncate = true; + else if (*i == "--type") { + ++i; + if (i == args.end()) throw UsageError("`--type' requires an argument"); + ht = parseHashType(*i); + if (ht == htUnknown) + throw UsageError(format("unknown hash type `%1%'") % *i); + } + else if (*i == "--to-base16") op = opTo16; + else if (*i == "--to-base32") op = opTo32; + else ss.push_back(*i); + } + + if (op == opHash) { + foreach (Strings::iterator, i, ss) { + Hash h = flat ? hashFile(ht, *i) : hashPath(ht, *i).first; + if (truncate && h.hashSize > 20) h = compressHash(h, 20); + std::cout << format("%1%\n") % + (base32 ? printHash32(h) : printHash(h)); + } + } + + else { + foreach (Strings::iterator, i, ss) { + Hash h = parseHash16or32(ht, *i); + std::cout << format("%1%\n") % + (op == opTo16 ? printHash(h) : printHash32(h)); + } + } +} + + +string programId = "nix-hash"; + diff --git a/src/nix-instantiate/local.mk b/src/nix-instantiate/local.mk new file mode 100644 index 000000000000..7d1bc5ec9dfb --- /dev/null +++ b/src/nix-instantiate/local.mk @@ -0,0 +1,7 @@ +programs += nix-instantiate + +nix-instantiate_DIR := $(d) + +nix-instantiate_SOURCES := $(d)/nix-instantiate.cc + +nix-instantiate_LIBS = libexpr libmain libstore libutil libformat diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc new file mode 100644 index 000000000000..b7f944062cec --- /dev/null +++ b/src/nix-instantiate/nix-instantiate.cc @@ -0,0 +1,176 @@ +#include "globals.hh" +#include "shared.hh" +#include "eval.hh" +#include "eval-inline.hh" +#include "get-drvs.hh" +#include "attr-path.hh" +#include "value-to-xml.hh" +#include "util.hh" +#include "store-api.hh" +#include "common-opts.hh" +#include "misc.hh" + +#include <map> +#include <iostream> + + +using namespace nix; + + +void printHelp() +{ + showManPage("nix-instantiate"); +} + + +static Expr * parseStdin(EvalState & state) +{ + startNest(nest, lvlTalkative, format("parsing standard input")); + return state.parseExprFromString(drainFD(0), absPath(".")); +} + + +static Path gcRoot; +static int rootNr = 0; +static bool indirectRoot = false; + + +void processExpr(EvalState & state, const Strings & attrPaths, + bool parseOnly, bool strict, Bindings & autoArgs, + bool evalOnly, bool xmlOutput, bool location, Expr * e) +{ + if (parseOnly) { + std::cout << format("%1%\n") % *e; + return; + } + + Value vRoot; + state.eval(e, vRoot); + + foreach (Strings::const_iterator, i, attrPaths) { + Value & v(*findAlongAttrPath(state, *i, autoArgs, vRoot)); + state.forceValue(v); + + PathSet context; + if (evalOnly) + if (xmlOutput) + printValueAsXML(state, strict, location, v, std::cout, context); + else { + if (strict) state.strictForceValue(v); + std::cout << v << std::endl; + } + else { + DrvInfos drvs; + getDerivations(state, v, "", autoArgs, drvs, false); + foreach (DrvInfos::iterator, i, drvs) { + Path drvPath = i->queryDrvPath(); + + /* What output do we want? */ + string outputName = i->queryOutputName(); + if (outputName == "") + throw Error(format("derivation `%1%' lacks an `outputName' attribute ") % drvPath); + + if (gcRoot == "") + printGCWarning(); + else { + Path rootName = gcRoot; + if (++rootNr > 1) rootName += "-" + int2String(rootNr); + drvPath = addPermRoot(*store, drvPath, rootName, indirectRoot); + } + std::cout << format("%1%%2%\n") % drvPath % (outputName != "out" ? "!" + outputName : ""); + } + } + } +} + + +void run(Strings args) +{ + EvalState state; + Strings files; + bool readStdin = false; + bool findFile = false; + bool evalOnly = false; + bool parseOnly = false; + bool xmlOutput = false; + bool xmlOutputSourceLocation = true; + bool strict = false; + Strings attrPaths; + Bindings autoArgs; + + for (Strings::iterator i = args.begin(); i != args.end(); ) { + string arg = *i++; + + if (arg == "-") + readStdin = true; + else if (arg == "--eval-only") { + settings.readOnlyMode = true; + evalOnly = true; + } + else if (arg == "--parse-only") { + settings.readOnlyMode = true; + parseOnly = evalOnly = true; + } + else if (arg == "--find-file") + findFile = true; + else if (arg == "--attr" || arg == "-A") { + if (i == args.end()) + throw UsageError("`--attr' requires an argument"); + attrPaths.push_back(*i++); + } + else if (parseOptionArg(arg, i, args.end(), state, autoArgs)) + ; + else if (parseSearchPathArg(arg, i, args.end(), state)) + ; + else if (arg == "--add-root") { + if (i == args.end()) + throw UsageError("`--add-root' requires an argument"); + gcRoot = absPath(*i++); + } + else if (arg == "--indirect") + indirectRoot = true; + else if (arg == "--xml") + xmlOutput = true; + else if (arg == "--no-location") + xmlOutputSourceLocation = false; + else if (arg == "--strict") + strict = true; + else if (arg == "--repair") + state.repair = true; + else if (arg[0] == '-') + throw UsageError(format("unknown flag `%1%'") % arg); + else + files.push_back(arg); + } + + if (attrPaths.empty()) attrPaths.push_back(""); + + if (findFile) { + foreach (Strings::iterator, i, files) { + Path p = state.findFile(*i); + if (p == "") throw Error(format("unable to find `%1%'") % *i); + std::cout << p << std::endl; + } + return; + } + + store = openStore(); + + if (readStdin) { + Expr * e = parseStdin(state); + processExpr(state, attrPaths, parseOnly, strict, autoArgs, + evalOnly, xmlOutput, xmlOutputSourceLocation, e); + } else if (files.empty()) + files.push_back("./default.nix"); + + foreach (Strings::iterator, i, files) { + Expr * e = state.parseExprFromFile(resolveExprPath(lookupFileArg(state, *i))); + processExpr(state, attrPaths, parseOnly, strict, autoArgs, + evalOnly, xmlOutput, xmlOutputSourceLocation, e); + } + + state.printStats(); +} + + +string programId = "nix-instantiate"; diff --git a/src/nix-log2xml/local.mk b/src/nix-log2xml/local.mk new file mode 100644 index 000000000000..46eb2e02ca64 --- /dev/null +++ b/src/nix-log2xml/local.mk @@ -0,0 +1,8 @@ +programs += nix-log2xml + +nix-log2xml_DIR := $(d) + +nix-log2xml_SOURCES := $(d)/log2xml.cc + +$(foreach file, mark-errors.xsl log2html.xsl treebits.js, \ + $(eval $(call install-data-in, $(d)/$(file), $(datadir)/nix/log2html))) diff --git a/src/nix-log2xml/log2html.xsl b/src/nix-log2xml/log2html.xsl new file mode 100644 index 000000000000..209399470640 --- /dev/null +++ b/src/nix-log2xml/log2html.xsl @@ -0,0 +1,83 @@ +<?xml version="1.0"?> + +<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform"> + + <xsl:output method='html' encoding="UTF-8" + doctype-public="-//W3C//DTD HTML 4.01//EN" + doctype-system="http://www.w3.org/TR/html4/strict.dtd" /> + + <xsl:template match="logfile"> + <html> + <head> + <script type="text/javascript" src="treebits.js" /> + <link rel="stylesheet" href="logfile.css" type="text/css" /> + <title>Log File</title> + </head> + <body> + <ul class='toplevel'> + <xsl:for-each select='line|nest'> + <li> + <xsl:apply-templates select='.'/> + </li> + </xsl:for-each> + </ul> + </body> + </html> + </xsl:template> + + + <xsl:template match="nest"> + + <!-- The tree should be collapsed by default if all children are + unimportant or if the header is unimportant. --> +<!-- <xsl:variable name="collapsed" + select="count(.//line[not(@priority = 3)]) = 0 or ./head[@priority = 3]" /> --> + <xsl:variable name="collapsed" select="count(.//*[@error]) = 0"/> + + <xsl:variable name="style"><xsl:if test="$collapsed">display: none;</xsl:if></xsl:variable> + <xsl:variable name="arg"><xsl:choose><xsl:when test="$collapsed">true</xsl:when><xsl:otherwise>false</xsl:otherwise></xsl:choose></xsl:variable> + + <script type='text/javascript'>showTreeToggle(<xsl:value-of select="$collapsed"/>)</script> + <xsl:apply-templates select='head'/> + + <!-- Be careful to only generate <ul>s if there are <li>s, otherwise it’s malformed. --> + <xsl:if test="line|nest"> + + <ul class='nesting' style="{$style}"> + <xsl:for-each select='line|nest'> + + <!-- Is this the last line? If so, mark it as such so that it + can be rendered differently. --> + <xsl:variable name="class"><xsl:choose><xsl:when test="position() != last()">line</xsl:when><xsl:otherwise>lastline</xsl:otherwise></xsl:choose></xsl:variable> + + <li class='{$class}'> + <span class='lineconn' /> + <span class='linebody'> + <xsl:apply-templates select='.'/> + </span> + </li> + </xsl:for-each> + </ul> + </xsl:if> + + </xsl:template> + + + <xsl:template match="head|line"> + <code> + <xsl:if test="@error"> + <xsl:attribute name="class">error</xsl:attribute> + </xsl:if> + <xsl:apply-templates/> + </code> + </xsl:template> + + + <xsl:template match="storeref"> + <em class='storeref'> + <span class='popup'><xsl:apply-templates/></span> + <span class='elided'>/...</span><xsl:apply-templates select='name'/><xsl:apply-templates select='path'/> + </em> + </xsl:template> + +</xsl:stylesheet> \ No newline at end of file diff --git a/src/nix-log2xml/log2xml.cc b/src/nix-log2xml/log2xml.cc new file mode 100644 index 000000000000..6645dc500fed --- /dev/null +++ b/src/nix-log2xml/log2xml.cc @@ -0,0 +1,200 @@ +#include <vector> +#include <iostream> +#include <cstdio> +#include <string> +#include <cstring> + +using namespace std; + + +struct Decoder +{ + enum { stTop, stEscape, stCSI } state; + string line; + bool inHeader; + int level; + vector<int> args; + bool newNumber; + int priority; + bool ignoreLF; + int lineNo, charNo; + bool warning; + bool error; + + Decoder() + { + state = stTop; + line = ""; + inHeader = false; + level = 0; + priority = 1; + ignoreLF = false; + lineNo = 1; + charNo = 0; + warning = false; + error = false; + } + + void pushChar(char c); + + void finishLine(); + + void decodeFile(istream & st); +}; + + +void Decoder::pushChar(char c) +{ + if (c == '\n') { + lineNo++; + charNo = 0; + } else charNo++; + + switch (state) { + + case stTop: + if (c == '\e') { + state = stEscape; + } else if (c == '\n' && !ignoreLF) { + finishLine(); + } else line += c; + break; + + case stEscape: + if (c == '[') { + state = stCSI; + args.clear(); + newNumber = true; + } else + state = stTop; /* !!! wrong */ + break; + + case stCSI: + if (c >= 0x40 && c != 0x7e) { + state = stTop; + switch (c) { + case 'p': + if (line.size()) finishLine(); + level++; + inHeader = true; + cout << "<nest>" << endl; + priority = args.size() >= 1 ? args[0] : 1; + break; + case 'q': + if (line.size()) finishLine(); + if (level > 0) { + level--; + cout << "</nest>" << endl; + } else + cerr << "not enough nesting levels at line " + << lineNo << ", character " << charNo << endl; + break; + case 's': + if (line.size()) finishLine(); + priority = args.size() >= 1 ? args[0] : 1; + break; + case 'a': + ignoreLF = true; + break; + case 'b': + ignoreLF = false; + break; + case 'e': + error = true; + break; + case 'w': + warning = true; + break; + } + } else if (c >= '0' && c <= '9') { + int n = 0; + if (!newNumber) { + n = args.back() * 10; + args.pop_back(); + } + n += c - '0'; + args.push_back(n); + } + break; + + } +} + + +void Decoder::finishLine() +{ + string storeDir = "/nix/store/"; + int sz = storeDir.size(); + string tag = inHeader ? "head" : "line"; + cout << "<" << tag; + if (priority != 1) cout << " priority='" << priority << "'"; + if (warning) cout << " warning='1'"; + if (error) cout << " error='1'"; + cout << ">"; + + for (unsigned int i = 0; i < line.size(); i++) { + + if (line[i] == '<') cout << "<"; + else if (line[i] == '&') cout << "&"; + else if (line[i] == '\r') ; /* ignore carriage return */ + else if (line[i] < 32 && line[i] != 9) cout << "�"; + else if (i + sz + 33 < line.size() && + string(line, i, sz) == storeDir && + line[i + sz + 32] == '-') + { + int j = i + sz + 32; + /* skip name */ + while (!strchr("/\n\r\t ()[]:;?<>", line[j])) j++; + int k = j; + while (!strchr("\n\r\t ()[]:;?<>", line[k])) k++; + // !!! escaping + cout << "<storeref>" + << "<storedir>" + << string(line, i, sz) + << "</storedir>" + << "<hash>" + << string(line, i + sz, 32) + << "</hash>" + << "<name>" + << string(line, i + sz + 32, j - (i + sz + 32)) + << "</name>" + << "<path>" + << string(line, j, k - j) + << "</path>" + << "</storeref>"; + i = k - 1; + } else cout << line[i]; + } + + cout << "</" << tag << ">" << endl; + line = ""; + inHeader = false; + priority = 1; + warning = false; + error = false; +} + + +void Decoder::decodeFile(istream & st) +{ + int c; + + cout << "<logfile>" << endl; + + while ((c = st.get()) != EOF) { + pushChar(c); + } + + if (line.size()) finishLine(); + + while (level--) cout << "</nest>" << endl; + + cout << "</logfile>" << endl; +} + + +int main(int argc, char * * argv) +{ + Decoder dec; + dec.decodeFile(cin); +} diff --git a/src/nix-log2xml/logfile.css b/src/nix-log2xml/logfile.css new file mode 100644 index 000000000000..ed390d64a9ef --- /dev/null +++ b/src/nix-log2xml/logfile.css @@ -0,0 +1,86 @@ +body { + font-family: sans-serif; + background: white; +} + + +ul.nesting, ul.toplevel { + padding: 0; + margin: 0; +} + +ul.toplevel { + list-style-type: none; +} + +ul.nesting li.line, ul.nesting li.lastline { + position: relative; + list-style-type: none; +} + +ul.nesting li.line { + padding-left: 1.1em; +} + +ul.nesting li.lastline { + padding-left: 1.2em; // for the 0.1em border-left in .lastline > .lineconn +} + +li.line { + border-left: 0.1em solid #6185a0; +} + +li.line > span.lineconn, li.lastline > span.lineconn { + position: absolute; + height: 0.65em; + left: 0em; + width: 1em; + border-bottom: 0.1em solid #6185a0; +} + +li.lastline > span.lineconn { + border-left: 0.1em solid #6185a0; +} + + +em.storeref { + color: #500000; + position: relative; + width: 100%; +} + +em.storeref:hover { + background-color: #eeeeee; +} + +*.popup { + display: none; +/* background: url('http://losser.st-lab.cs.uu.nl/~mbravenb/menuback.png') repeat; */ + background: #ffffcd; + border: solid #555555 1px; + position: absolute; + top: 0em; + left: 0em; + margin: 0; + padding: 0; + z-index: 100; +} + +em.storeref:hover span.popup { + display: inline; +} + + +.toggle { + text-decoration: none; +} + +.showTree, .hideTree { + font-family: monospace; + font-size: larger; +} + +.error { + color: #ff0000; + font-weight: bold; +} \ No newline at end of file diff --git a/src/nix-log2xml/mark-errors.xsl b/src/nix-log2xml/mark-errors.xsl new file mode 100644 index 000000000000..4e91913e570a --- /dev/null +++ b/src/nix-log2xml/mark-errors.xsl @@ -0,0 +1,24 @@ +<?xml version="1.0"?> + +<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform"> + + <xsl:template match="@*|node()"> + <xsl:copy> + <xsl:apply-templates select="@*|node()"/> + </xsl:copy> + </xsl:template> + + <xsl:template match="line"> + <line> + <xsl:if test="contains(text(), ' *** ') or + contains(text(), 'LaTeX Error') or + contains(text(), 'FAIL:') or + contains(text(), ' error: ') or + true"> + <xsl:attribute name="error"></xsl:attribute> + </xsl:if> + <xsl:apply-templates select="@*|node()"/> + </line> + </xsl:template> + +</xsl:stylesheet> \ No newline at end of file diff --git a/src/nix-log2xml/treebits.js b/src/nix-log2xml/treebits.js new file mode 100644 index 000000000000..3011b391d88b --- /dev/null +++ b/src/nix-log2xml/treebits.js @@ -0,0 +1,50 @@ +/* Acknowledgement: this is based on the Wikipedia table-of-contents + * toggle. */ + + +var idCounter = 0; + + +function showTreeToggle(isHidden) { + if (document.getElementById) { + var id = "toggle_" + idCounter; + document.writeln( + '<a href="javascript:toggleTree(\'' + id + '\')" class="toggle" id="' + id + '">' + + '<span class="showTree" ' + (isHidden ? '' : 'style="display: none;"') + '>+</span>' + + '<span class="hideTree" ' + (isHidden ? 'style="display: none;"' : '') + '>-</span>' + + '</a>'); + idCounter = idCounter + 1; + } +} + + +function toggleTree(id) { + + var href = document.getElementById(id); + + var node = href; + var tree = null; + while (node != null) { + if (node.className == "nesting") tree = node; + node = node.nextSibling; + } + + node = href.firstChild; + var hideTree = null; + var showTree = null; + while (node != null) { + if (node.className == "showTree") showTree = node; + else if (node.className == "hideTree") hideTree = node; + node = node.nextSibling; + } + + if (tree.style.display == 'none') { + tree.style.display = ''; + hideTree.style.display = ''; + showTree.style.display = 'none'; + } else { + tree.style.display = 'none'; + hideTree.style.display = 'none'; + showTree.style.display = ''; + } +} diff --git a/src/nix-store/dotgraph.cc b/src/nix-store/dotgraph.cc new file mode 100644 index 000000000000..72146eb68efc --- /dev/null +++ b/src/nix-store/dotgraph.cc @@ -0,0 +1,162 @@ +#include "dotgraph.hh" +#include "util.hh" +#include "store-api.hh" + +#include <iostream> + + +using std::cout; + +namespace nix { + + +static string dotQuote(const string & s) +{ + return "\"" + s + "\""; +} + + +static string nextColour() +{ + static int n = 0; + static string colours[] = + { "black", "red", "green", "blue" + , "magenta", "burlywood" }; + return colours[n++ % (sizeof(colours) / sizeof(string))]; +} + + +static string makeEdge(const string & src, const string & dst) +{ + format f = format("%1% -> %2% [color = %3%];\n") + % dotQuote(src) % dotQuote(dst) % dotQuote(nextColour()); + return f.str(); +} + + +static string makeNode(const string & id, const string & label, + const string & colour) +{ + format f = format("%1% [label = %2%, shape = box, " + "style = filled, fillcolor = %3%];\n") + % dotQuote(id) % dotQuote(label) % dotQuote(colour); + return f.str(); +} + + +static string symbolicName(const string & path) +{ + string p = baseNameOf(path); + int dash = p.find('-'); + return string(p, dash + 1); +} + + +#if 0 +string pathLabel(const Path & nePath, const string & elemPath) +{ + return (string) nePath + "-" + elemPath; +} + + +void printClosure(const Path & nePath, const StoreExpr & fs) +{ + PathSet workList(fs.closure.roots); + PathSet doneSet; + + for (PathSet::iterator i = workList.begin(); i != workList.end(); ++i) { + cout << makeEdge(pathLabel(nePath, *i), nePath); + } + + while (!workList.empty()) { + Path path = *(workList.begin()); + workList.erase(path); + + if (doneSet.find(path) == doneSet.end()) { + doneSet.insert(path); + + ClosureElems::const_iterator elem = fs.closure.elems.find(path); + if (elem == fs.closure.elems.end()) + throw Error(format("bad closure, missing path `%1%'") % path); + + for (StringSet::const_iterator i = elem->second.refs.begin(); + i != elem->second.refs.end(); ++i) + { + workList.insert(*i); + cout << makeEdge(pathLabel(nePath, *i), pathLabel(nePath, path)); + } + + cout << makeNode(pathLabel(nePath, path), + symbolicName(path), "#ff0000"); + } + } +} +#endif + + +void printDotGraph(const PathSet & roots) +{ + PathSet workList(roots); + PathSet doneSet; + + cout << "digraph G {\n"; + + while (!workList.empty()) { + Path path = *(workList.begin()); + workList.erase(path); + + if (doneSet.find(path) != doneSet.end()) continue; + doneSet.insert(path); + + cout << makeNode(path, symbolicName(path), "#ff0000"); + + PathSet references; + store->queryReferences(path, references); + + for (PathSet::iterator i = references.begin(); + i != references.end(); ++i) + { + if (*i != path) { + workList.insert(*i); + cout << makeEdge(*i, path); + } + } + + +#if 0 + StoreExpr ne = storeExprFromPath(path); + + string label, colour; + + if (ne.type == StoreExpr::neDerivation) { + for (PathSet::iterator i = ne.derivation.inputs.begin(); + i != ne.derivation.inputs.end(); ++i) + { + workList.insert(*i); + cout << makeEdge(*i, path); + } + + label = "derivation"; + colour = "#00ff00"; + for (StringPairs::iterator i = ne.derivation.env.begin(); + i != ne.derivation.env.end(); ++i) + if (i->first == "name") label = i->second; + } + + else if (ne.type == StoreExpr::neClosure) { + label = "<closure>"; + colour = "#00ffff"; + printClosure(path, ne); + } + + else abort(); + + cout << makeNode(path, label, colour); +#endif + } + + cout << "}\n"; +} + + +} diff --git a/src/nix-store/dotgraph.hh b/src/nix-store/dotgraph.hh new file mode 100644 index 000000000000..68410d84156d --- /dev/null +++ b/src/nix-store/dotgraph.hh @@ -0,0 +1,9 @@ +#pragma once + +#include "types.hh" + +namespace nix { + +void printDotGraph(const PathSet & roots); + +} diff --git a/src/nix-store/local.mk b/src/nix-store/local.mk new file mode 100644 index 000000000000..7f93e4c19196 --- /dev/null +++ b/src/nix-store/local.mk @@ -0,0 +1,9 @@ +programs += nix-store + +nix-store_DIR := $(d) + +nix-store_SOURCES := $(wildcard $(d)/*.cc) + +nix-store_LIBS = libmain libstore libutil libformat + +nix-store_LDFLAGS = -lbz2 diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc new file mode 100644 index 000000000000..69a98fe4726d --- /dev/null +++ b/src/nix-store/nix-store.cc @@ -0,0 +1,929 @@ +#include "globals.hh" +#include "misc.hh" +#include "archive.hh" +#include "shared.hh" +#include "dotgraph.hh" +#include "xmlgraph.hh" +#include "local-store.hh" +#include "util.hh" + +#include <iostream> +#include <algorithm> +#include <cstdio> + +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> + +#include <bzlib.h> + + +using namespace nix; +using std::cin; +using std::cout; + + +typedef void (* Operation) (Strings opFlags, Strings opArgs); + + +void printHelp() +{ + showManPage("nix-store"); +} + + +static Path gcRoot; +static int rootNr = 0; +static bool indirectRoot = false; +static bool noOutput = false; + + +LocalStore & ensureLocalStore() +{ + LocalStore * store2(dynamic_cast<LocalStore *>(store.get())); + if (!store2) throw Error("you don't have sufficient rights to use this command"); + return *store2; +} + + +static Path useDeriver(Path path) +{ + if (!isDerivation(path)) { + path = store->queryDeriver(path); + if (path == "") + throw Error(format("deriver of path `%1%' is not known") % path); + } + return path; +} + + +/* Realise the given path. For a derivation that means build it; for + other paths it means ensure their validity. */ +static PathSet realisePath(const Path & path, bool build = true) +{ + DrvPathWithOutputs p = parseDrvPathWithOutputs(path); + + if (isDerivation(p.first)) { + if (build) store->buildPaths(singleton<PathSet>(path)); + Derivation drv = derivationFromPath(*store, p.first); + rootNr++; + + if (p.second.empty()) + foreach (DerivationOutputs::iterator, i, drv.outputs) p.second.insert(i->first); + + PathSet outputs; + foreach (StringSet::iterator, j, p.second) { + DerivationOutputs::iterator i = drv.outputs.find(*j); + if (i == drv.outputs.end()) + throw Error(format("derivation `%1%' does not have an output named `%2%'") % p.first % *j); + Path outPath = i->second.path; + if (gcRoot == "") + printGCWarning(); + else { + Path rootName = gcRoot; + if (rootNr > 1) rootName += "-" + int2String(rootNr); + if (i->first != "out") rootName += "-" + i->first; + outPath = addPermRoot(*store, outPath, rootName, indirectRoot); + } + outputs.insert(outPath); + } + return outputs; + } + + else { + if (build) store->ensurePath(path); + else if (!store->isValidPath(path)) throw Error(format("path `%1%' does not exist and cannot be created") % path); + return singleton<PathSet>(path); + } +} + + +/* Realise the given paths. */ +static void opRealise(Strings opFlags, Strings opArgs) +{ + bool dryRun = false; + bool repair = false; + bool ignoreUnknown = false; + + foreach (Strings::iterator, i, opFlags) + if (*i == "--dry-run") dryRun = true; + else if (*i == "--repair") repair = true; + else if (*i == "--ignore-unknown") ignoreUnknown = true; + else throw UsageError(format("unknown flag `%1%'") % *i); + + Paths paths; + foreach (Strings::iterator, i, opArgs) { + DrvPathWithOutputs p = parseDrvPathWithOutputs(*i); + paths.push_back(makeDrvPathWithOutputs(followLinksToStorePath(p.first), p.second)); + } + + unsigned long long downloadSize, narSize; + PathSet willBuild, willSubstitute, unknown; + queryMissing(*store, PathSet(paths.begin(), paths.end()), + willBuild, willSubstitute, unknown, downloadSize, narSize); + + if (ignoreUnknown) { + Paths paths2; + foreach (Paths::iterator, i, paths) + if (unknown.find(*i) == unknown.end()) paths2.push_back(*i); + paths = paths2; + unknown = PathSet(); + } + + printMissing(willBuild, willSubstitute, unknown, downloadSize, narSize); + + if (dryRun) return; + + /* Build all paths at the same time to exploit parallelism. */ + store->buildPaths(PathSet(paths.begin(), paths.end()), repair); + + if (!ignoreUnknown) + foreach (Paths::iterator, i, paths) { + PathSet paths = realisePath(*i, false); + if (!noOutput) + foreach (PathSet::iterator, j, paths) + cout << format("%1%\n") % *j; + } +} + + +/* Add files to the Nix store and print the resulting paths. */ +static void opAdd(Strings opFlags, Strings opArgs) +{ + if (!opFlags.empty()) throw UsageError("unknown flag"); + + for (Strings::iterator i = opArgs.begin(); i != opArgs.end(); ++i) + cout << format("%1%\n") % store->addToStore(*i); +} + + +/* Preload the output of a fixed-output derivation into the Nix + store. */ +static void opAddFixed(Strings opFlags, Strings opArgs) +{ + bool recursive = false; + + for (Strings::iterator i = opFlags.begin(); + i != opFlags.end(); ++i) + if (*i == "--recursive") recursive = true; + else throw UsageError(format("unknown flag `%1%'") % *i); + + if (opArgs.empty()) + throw UsageError("first argument must be hash algorithm"); + + HashType hashAlgo = parseHashType(opArgs.front()); + opArgs.pop_front(); + + for (Strings::iterator i = opArgs.begin(); i != opArgs.end(); ++i) + cout << format("%1%\n") % store->addToStore(*i, recursive, hashAlgo); +} + + +/* Hack to support caching in `nix-prefetch-url'. */ +static void opPrintFixedPath(Strings opFlags, Strings opArgs) +{ + bool recursive = false; + + for (Strings::iterator i = opFlags.begin(); + i != opFlags.end(); ++i) + if (*i == "--recursive") recursive = true; + else throw UsageError(format("unknown flag `%1%'") % *i); + + if (opArgs.size() != 3) + throw UsageError(format("`--print-fixed-path' requires three arguments")); + + Strings::iterator i = opArgs.begin(); + HashType hashAlgo = parseHashType(*i++); + string hash = *i++; + string name = *i++; + + cout << format("%1%\n") % + makeFixedOutputPath(recursive, hashAlgo, + parseHash16or32(hashAlgo, hash), name); +} + + +static PathSet maybeUseOutputs(const Path & storePath, bool useOutput, bool forceRealise) +{ + if (forceRealise) realisePath(storePath); + if (useOutput && isDerivation(storePath)) { + Derivation drv = derivationFromPath(*store, storePath); + PathSet outputs; + foreach (DerivationOutputs::iterator, i, drv.outputs) + outputs.insert(i->second.path); + return outputs; + } + else return singleton<PathSet>(storePath); +} + + +/* Some code to print a tree representation of a derivation dependency + graph. Topological sorting is used to keep the tree relatively + flat. */ + +const string treeConn = "+---"; +const string treeLine = "| "; +const string treeNull = " "; + + +static void printTree(const Path & path, + const string & firstPad, const string & tailPad, PathSet & done) +{ + if (done.find(path) != done.end()) { + cout << format("%1%%2% [...]\n") % firstPad % path; + return; + } + done.insert(path); + + cout << format("%1%%2%\n") % firstPad % path; + + PathSet references; + store->queryReferences(path, references); + + /* Topologically sort under the relation A < B iff A \in + closure(B). That is, if derivation A is an (possibly indirect) + input of B, then A is printed first. This has the effect of + flattening the tree, preventing deeply nested structures. */ + Paths sorted = topoSortPaths(*store, references); + reverse(sorted.begin(), sorted.end()); + + foreach (Paths::iterator, i, sorted) { + Paths::iterator j = i; ++j; + printTree(*i, tailPad + treeConn, + j == sorted.end() ? tailPad + treeNull : tailPad + treeLine, + done); + } +} + + +/* Perform various sorts of queries. */ +static void opQuery(Strings opFlags, Strings opArgs) +{ + enum { qOutputs, qRequisites, qReferences, qReferrers + , qReferrersClosure, qDeriver, qBinding, qHash, qSize + , qTree, qGraph, qXml, qResolve, qRoots } query = qOutputs; + bool useOutput = false; + bool includeOutputs = false; + bool forceRealise = false; + string bindingName; + + foreach (Strings::iterator, i, opFlags) + if (*i == "--outputs") query = qOutputs; + else if (*i == "--requisites" || *i == "-R") query = qRequisites; + else if (*i == "--references") query = qReferences; + else if (*i == "--referrers" || *i == "--referers") query = qReferrers; + else if (*i == "--referrers-closure" || *i == "--referers-closure") query = qReferrersClosure; + else if (*i == "--deriver" || *i == "-d") query = qDeriver; + else if (*i == "--binding" || *i == "-b") { + if (opArgs.size() == 0) + throw UsageError("expected binding name"); + bindingName = opArgs.front(); + opArgs.pop_front(); + query = qBinding; + } + else if (*i == "--hash") query = qHash; + else if (*i == "--size") query = qSize; + else if (*i == "--tree") query = qTree; + else if (*i == "--graph") query = qGraph; + else if (*i == "--xml") query = qXml; + else if (*i == "--resolve") query = qResolve; + else if (*i == "--roots") query = qRoots; + else if (*i == "--use-output" || *i == "-u") useOutput = true; + else if (*i == "--force-realise" || *i == "--force-realize" || *i == "-f") forceRealise = true; + else if (*i == "--include-outputs") includeOutputs = true; + else throw UsageError(format("unknown flag `%1%'") % *i); + + switch (query) { + + case qOutputs: { + foreach (Strings::iterator, i, opArgs) { + *i = followLinksToStorePath(*i); + if (forceRealise) realisePath(*i); + Derivation drv = derivationFromPath(*store, *i); + foreach (DerivationOutputs::iterator, j, drv.outputs) + cout << format("%1%\n") % j->second.path; + } + break; + } + + case qRequisites: + case qReferences: + case qReferrers: + case qReferrersClosure: { + PathSet paths; + foreach (Strings::iterator, i, opArgs) { + PathSet ps = maybeUseOutputs(followLinksToStorePath(*i), useOutput, forceRealise); + foreach (PathSet::iterator, j, ps) { + if (query == qRequisites) computeFSClosure(*store, *j, paths, false, includeOutputs); + else if (query == qReferences) store->queryReferences(*j, paths); + else if (query == qReferrers) store->queryReferrers(*j, paths); + else if (query == qReferrersClosure) computeFSClosure(*store, *j, paths, true); + } + } + Paths sorted = topoSortPaths(*store, paths); + for (Paths::reverse_iterator i = sorted.rbegin(); + i != sorted.rend(); ++i) + cout << format("%s\n") % *i; + break; + } + + case qDeriver: + foreach (Strings::iterator, i, opArgs) { + Path deriver = store->queryDeriver(followLinksToStorePath(*i)); + cout << format("%1%\n") % + (deriver == "" ? "unknown-deriver" : deriver); + } + break; + + case qBinding: + foreach (Strings::iterator, i, opArgs) { + Path path = useDeriver(followLinksToStorePath(*i)); + Derivation drv = derivationFromPath(*store, path); + StringPairs::iterator j = drv.env.find(bindingName); + if (j == drv.env.end()) + throw Error(format("derivation `%1%' has no environment binding named `%2%'") + % path % bindingName); + cout << format("%1%\n") % j->second; + } + break; + + case qHash: + case qSize: + foreach (Strings::iterator, i, opArgs) { + PathSet paths = maybeUseOutputs(followLinksToStorePath(*i), useOutput, forceRealise); + foreach (PathSet::iterator, j, paths) { + ValidPathInfo info = store->queryPathInfo(*j); + if (query == qHash) { + assert(info.hash.type == htSHA256); + cout << format("sha256:%1%\n") % printHash32(info.hash); + } else if (query == qSize) + cout << format("%1%\n") % info.narSize; + } + } + break; + + case qTree: { + PathSet done; + foreach (Strings::iterator, i, opArgs) + printTree(followLinksToStorePath(*i), "", "", done); + break; + } + + case qGraph: { + PathSet roots; + foreach (Strings::iterator, i, opArgs) { + PathSet paths = maybeUseOutputs(followLinksToStorePath(*i), useOutput, forceRealise); + roots.insert(paths.begin(), paths.end()); + } + printDotGraph(roots); + break; + } + + case qXml: { + PathSet roots; + foreach (Strings::iterator, i, opArgs) { + PathSet paths = maybeUseOutputs(followLinksToStorePath(*i), useOutput, forceRealise); + roots.insert(paths.begin(), paths.end()); + } + printXmlGraph(roots); + break; + } + + case qResolve: { + foreach (Strings::iterator, i, opArgs) + cout << format("%1%\n") % followLinksToStorePath(*i); + break; + } + + case qRoots: { + PathSet referrers; + foreach (Strings::iterator, i, opArgs) { + PathSet paths = maybeUseOutputs(followLinksToStorePath(*i), useOutput, forceRealise); + foreach (PathSet::iterator, j, paths) + computeFSClosure(*store, *j, referrers, true, + settings.gcKeepOutputs, settings.gcKeepDerivations); + } + Roots roots = store->findRoots(); + foreach (Roots::iterator, i, roots) + if (referrers.find(i->second) != referrers.end()) + cout << format("%1%\n") % i->first; + break; + } + + default: + abort(); + } +} + + +static string shellEscape(const string & s) +{ + string r; + foreach (string::const_iterator, i, s) + if (*i == '\'') r += "'\\''"; else r += *i; + return r; +} + + +static void opPrintEnv(Strings opFlags, Strings opArgs) +{ + if (!opFlags.empty()) throw UsageError("unknown flag"); + if (opArgs.size() != 1) throw UsageError("`--print-env' requires one derivation store path"); + + Path drvPath = opArgs.front(); + Derivation drv = derivationFromPath(*store, drvPath); + + /* Print each environment variable in the derivation in a format + that can be sourced by the shell. */ + foreach (StringPairs::iterator, i, drv.env) + cout << format("export %1%; %1%='%2%'\n") % i->first % shellEscape(i->second); + + /* Also output the arguments. This doesn't preserve whitespace in + arguments. */ + cout << "export _args; _args='"; + foreach (Strings::iterator, i, drv.args) { + if (i != drv.args.begin()) cout << ' '; + cout << shellEscape(*i); + } + cout << "'\n"; +} + + +static void opReadLog(Strings opFlags, Strings opArgs) +{ + if (!opFlags.empty()) throw UsageError("unknown flag"); + + foreach (Strings::iterator, i, opArgs) { + Path path = useDeriver(followLinksToStorePath(*i)); + + for (int j = 0; j <= 2; j++) { + if (j == 2) throw Error(format("build log of derivation `%1%' is not available") % path); + + string baseName = baseNameOf(path); + Path logPath = + j == 0 + ? (format("%1%/%2%/%3%/%4%") % settings.nixLogDir % drvsLogDir % string(baseName, 0, 2) % string(baseName, 2)).str() + : (format("%1%/%2%/%3%") % settings.nixLogDir % drvsLogDir % baseName).str(); + Path logBz2Path = logPath + ".bz2"; + + if (pathExists(logPath)) { + /* !!! Make this run in O(1) memory. */ + string log = readFile(logPath); + writeFull(STDOUT_FILENO, (const unsigned char *) log.data(), log.size()); + break; + } + + else if (pathExists(logBz2Path)) { + AutoCloseFD fd = open(logBz2Path.c_str(), O_RDONLY); + FILE * f = 0; + if (fd == -1 || (f = fdopen(fd.borrow(), "r")) == 0) + throw SysError(format("opening file `%1%'") % logBz2Path); + int err; + BZFILE * bz = BZ2_bzReadOpen(&err, f, 0, 0, 0, 0); + if (!bz) throw Error(format("cannot open bzip2 file `%1%'") % logBz2Path); + unsigned char buf[128 * 1024]; + do { + int n = BZ2_bzRead(&err, bz, buf, sizeof(buf)); + if (err != BZ_OK && err != BZ_STREAM_END) + throw Error(format("error reading bzip2 file `%1%'") % logBz2Path); + writeFull(STDOUT_FILENO, buf, n); + } while (err != BZ_STREAM_END); + BZ2_bzReadClose(&err, bz); + break; + } + } + } +} + + +static void opDumpDB(Strings opFlags, Strings opArgs) +{ + if (!opFlags.empty()) throw UsageError("unknown flag"); + if (!opArgs.empty()) + throw UsageError("no arguments expected"); + PathSet validPaths = store->queryAllValidPaths(); + foreach (PathSet::iterator, i, validPaths) + cout << store->makeValidityRegistration(singleton<PathSet>(*i), true, true); +} + + +static void registerValidity(bool reregister, bool hashGiven, bool canonicalise) +{ + ValidPathInfos infos; + + while (1) { + ValidPathInfo info = decodeValidPathInfo(cin, hashGiven); + if (info.path == "") break; + if (!store->isValidPath(info.path) || reregister) { + /* !!! races */ + if (canonicalise) + canonicalisePathMetaData(info.path, -1); + if (!hashGiven) { + HashResult hash = hashPath(htSHA256, info.path); + info.hash = hash.first; + info.narSize = hash.second; + } + infos.push_back(info); + } + } + + ensureLocalStore().registerValidPaths(infos); +} + + +static void opLoadDB(Strings opFlags, Strings opArgs) +{ + if (!opFlags.empty()) throw UsageError("unknown flag"); + if (!opArgs.empty()) + throw UsageError("no arguments expected"); + registerValidity(true, true, false); +} + + +static void opRegisterValidity(Strings opFlags, Strings opArgs) +{ + bool reregister = false; // !!! maybe this should be the default + bool hashGiven = false; + + for (Strings::iterator i = opFlags.begin(); + i != opFlags.end(); ++i) + if (*i == "--reregister") reregister = true; + else if (*i == "--hash-given") hashGiven = true; + else throw UsageError(format("unknown flag `%1%'") % *i); + + if (!opArgs.empty()) throw UsageError("no arguments expected"); + + registerValidity(reregister, hashGiven, true); +} + + +static void opCheckValidity(Strings opFlags, Strings opArgs) +{ + bool printInvalid = false; + + for (Strings::iterator i = opFlags.begin(); + i != opFlags.end(); ++i) + if (*i == "--print-invalid") printInvalid = true; + else throw UsageError(format("unknown flag `%1%'") % *i); + + for (Strings::iterator i = opArgs.begin(); + i != opArgs.end(); ++i) + { + Path path = followLinksToStorePath(*i); + if (!store->isValidPath(path)) { + if (printInvalid) + cout << format("%1%\n") % path; + else + throw Error(format("path `%1%' is not valid") % path); + } + } +} + + +static string showBytes(unsigned long long bytes) +{ + return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str(); +} + + +struct PrintFreed +{ + bool show; + const GCResults & results; + PrintFreed(bool show, const GCResults & results) + : show(show), results(results) { } + ~PrintFreed() + { + if (show) + cout << format("%1% store paths deleted, %2% freed\n") + % results.paths.size() + % showBytes(results.bytesFreed); + } +}; + + +static void opGC(Strings opFlags, Strings opArgs) +{ + bool printRoots = false; + GCOptions options; + options.action = GCOptions::gcDeleteDead; + + GCResults results; + + /* Do what? */ + foreach (Strings::iterator, i, opFlags) + if (*i == "--print-roots") printRoots = true; + else if (*i == "--print-live") options.action = GCOptions::gcReturnLive; + else if (*i == "--print-dead") options.action = GCOptions::gcReturnDead; + else if (*i == "--delete") options.action = GCOptions::gcDeleteDead; + else if (*i == "--max-freed") { + long long maxFreed = getIntArg<long long>(*i, i, opFlags.end()); + options.maxFreed = maxFreed >= 0 ? maxFreed : 0; + } + else throw UsageError(format("bad sub-operation `%1%' in GC") % *i); + + if (!opArgs.empty()) throw UsageError("no arguments expected"); + + if (printRoots) { + Roots roots = store->findRoots(); + foreach (Roots::iterator, i, roots) + cout << i->first << " -> " << i->second << std::endl; + } + + else { + PrintFreed freed(options.action == GCOptions::gcDeleteDead, results); + store->collectGarbage(options, results); + + if (options.action != GCOptions::gcDeleteDead) + foreach (PathSet::iterator, i, results.paths) + cout << *i << std::endl; + } +} + + +/* Remove paths from the Nix store if possible (i.e., if they do not + have any remaining referrers and are not reachable from any GC + roots). */ +static void opDelete(Strings opFlags, Strings opArgs) +{ + GCOptions options; + options.action = GCOptions::gcDeleteSpecific; + + foreach (Strings::iterator, i, opFlags) + if (*i == "--ignore-liveness") options.ignoreLiveness = true; + else throw UsageError(format("unknown flag `%1%'") % *i); + + foreach (Strings::iterator, i, opArgs) + options.pathsToDelete.insert(followLinksToStorePath(*i)); + + GCResults results; + PrintFreed freed(true, results); + store->collectGarbage(options, results); +} + + +/* Dump a path as a Nix archive. The archive is written to standard + output. */ +static void opDump(Strings opFlags, Strings opArgs) +{ + if (!opFlags.empty()) throw UsageError("unknown flag"); + if (opArgs.size() != 1) throw UsageError("only one argument allowed"); + + FdSink sink(STDOUT_FILENO); + string path = *opArgs.begin(); + dumpPath(path, sink); +} + + +/* Restore a value from a Nix archive. The archive is read from + standard input. */ +static void opRestore(Strings opFlags, Strings opArgs) +{ + if (!opFlags.empty()) throw UsageError("unknown flag"); + if (opArgs.size() != 1) throw UsageError("only one argument allowed"); + + FdSource source(STDIN_FILENO); + restorePath(*opArgs.begin(), source); +} + + +static void opExport(Strings opFlags, Strings opArgs) +{ + bool sign = false; + for (Strings::iterator i = opFlags.begin(); + i != opFlags.end(); ++i) + if (*i == "--sign") sign = true; + else throw UsageError(format("unknown flag `%1%'") % *i); + + FdSink sink(STDOUT_FILENO); + Paths sorted = topoSortPaths(*store, PathSet(opArgs.begin(), opArgs.end())); + reverse(sorted.begin(), sorted.end()); + exportPaths(*store, sorted, sign, sink); +} + + +static void opImport(Strings opFlags, Strings opArgs) +{ + bool requireSignature = false; + foreach (Strings::iterator, i, opFlags) + if (*i == "--require-signature") requireSignature = true; + else throw UsageError(format("unknown flag `%1%'") % *i); + + if (!opArgs.empty()) throw UsageError("no arguments expected"); + + FdSource source(STDIN_FILENO); + Paths paths = store->importPaths(requireSignature, source); + + foreach (Paths::iterator, i, paths) + cout << format("%1%\n") % *i << std::flush; +} + + +/* Initialise the Nix databases. */ +static void opInit(Strings opFlags, Strings opArgs) +{ + if (!opFlags.empty()) throw UsageError("unknown flag"); + if (!opArgs.empty()) + throw UsageError("no arguments expected"); + /* Doesn't do anything right now; database tables are initialised + automatically. */ +} + + +/* Verify the consistency of the Nix environment. */ +static void opVerify(Strings opFlags, Strings opArgs) +{ + if (!opArgs.empty()) + throw UsageError("no arguments expected"); + + bool checkContents = false; + bool repair = false; + + for (Strings::iterator i = opFlags.begin(); + i != opFlags.end(); ++i) + if (*i == "--check-contents") checkContents = true; + else if (*i == "--repair") repair = true; + else throw UsageError(format("unknown flag `%1%'") % *i); + + if (ensureLocalStore().verifyStore(checkContents, repair)) { + printMsg(lvlError, "warning: not all errors were fixed"); + exitCode = 1; + } +} + + +/* Verify whether the contents of the given store path have not changed. */ +static void opVerifyPath(Strings opFlags, Strings opArgs) +{ + if (!opFlags.empty()) + throw UsageError("no flags expected"); + + foreach (Strings::iterator, i, opArgs) { + Path path = followLinksToStorePath(*i); + printMsg(lvlTalkative, format("checking path `%1%'...") % path); + ValidPathInfo info = store->queryPathInfo(path); + HashResult current = hashPath(info.hash.type, path); + if (current.first != info.hash) { + printMsg(lvlError, + format("path `%1%' was modified! expected hash `%2%', got `%3%'") + % path % printHash(info.hash) % printHash(current.first)); + exitCode = 1; + } + } +} + + +/* Repair the contents of the given path by redownloading it using a + substituter (if available). */ +static void opRepairPath(Strings opFlags, Strings opArgs) +{ + if (!opFlags.empty()) + throw UsageError("no flags expected"); + + foreach (Strings::iterator, i, opArgs) { + Path path = followLinksToStorePath(*i); + ensureLocalStore().repairPath(path); + } +} + + +static void showOptimiseStats(OptimiseStats & stats) +{ + printMsg(lvlError, + format("%1% freed by hard-linking %2% files; there are %3% files with equal contents out of %4% files in total") + % showBytes(stats.bytesFreed) + % stats.filesLinked + % stats.sameContents + % stats.totalFiles); +} + + +/* Optimise the disk space usage of the Nix store by hard-linking + files with the same contents. */ +static void opOptimise(Strings opFlags, Strings opArgs) +{ + if (!opArgs.empty() || !opFlags.empty()) + throw UsageError("no arguments expected"); + + OptimiseStats stats; + try { + ensureLocalStore().optimiseStore(stats); + } catch (...) { + showOptimiseStats(stats); + throw; + } + showOptimiseStats(stats); +} + + +static void opQueryFailedPaths(Strings opFlags, Strings opArgs) +{ + if (!opArgs.empty() || !opFlags.empty()) + throw UsageError("no arguments expected"); + PathSet failed = store->queryFailedPaths(); + foreach (PathSet::iterator, i, failed) + cout << format("%1%\n") % *i; +} + + +static void opClearFailedPaths(Strings opFlags, Strings opArgs) +{ + if (!opFlags.empty()) + throw UsageError("no flags expected"); + store->clearFailedPaths(PathSet(opArgs.begin(), opArgs.end())); +} + + +/* Scan the arguments; find the operation, set global flags, put all + other flags in a list, and put all other arguments in another + list. */ +void run(Strings args) +{ + Strings opFlags, opArgs; + Operation op = 0; + + for (Strings::iterator i = args.begin(); i != args.end(); ) { + string arg = *i++; + + Operation oldOp = op; + + if (arg == "--realise" || arg == "--realize" || arg == "-r") + op = opRealise; + else if (arg == "--add" || arg == "-A") + op = opAdd; + else if (arg == "--add-fixed") + op = opAddFixed; + else if (arg == "--print-fixed-path") + op = opPrintFixedPath; + else if (arg == "--delete") + op = opDelete; + else if (arg == "--query" || arg == "-q") + op = opQuery; + else if (arg == "--print-env") + op = opPrintEnv; + else if (arg == "--read-log" || arg == "-l") + op = opReadLog; + else if (arg == "--dump-db") + op = opDumpDB; + else if (arg == "--load-db") + op = opLoadDB; + else if (arg == "--register-validity") + op = opRegisterValidity; + else if (arg == "--check-validity") + op = opCheckValidity; + else if (arg == "--gc") + op = opGC; + else if (arg == "--dump") + op = opDump; + else if (arg == "--restore") + op = opRestore; + else if (arg == "--export") + op = opExport; + else if (arg == "--import") + op = opImport; + else if (arg == "--init") + op = opInit; + else if (arg == "--verify") + op = opVerify; + else if (arg == "--verify-path") + op = opVerifyPath; + else if (arg == "--repair-path") + op = opRepairPath; + else if (arg == "--optimise" || arg == "--optimize") + op = opOptimise; + else if (arg == "--query-failed-paths") + op = opQueryFailedPaths; + else if (arg == "--clear-failed-paths") + op = opClearFailedPaths; + else if (arg == "--add-root") { + if (i == args.end()) + throw UsageError("`--add-root requires an argument"); + gcRoot = absPath(*i++); + } + else if (arg == "--indirect") + indirectRoot = true; + else if (arg == "--no-output") + noOutput = true; + else if (arg[0] == '-') { + opFlags.push_back(arg); + if (arg == "--max-freed" || arg == "--max-links" || arg == "--max-atime") { /* !!! hack */ + if (i != args.end()) opFlags.push_back(*i++); + } + } + else + opArgs.push_back(arg); + + if (oldOp && oldOp != op) + throw UsageError("only one operation may be specified"); + } + + if (!op) throw UsageError("no operation specified"); + + if (op != opDump && op != opRestore) /* !!! hack */ + store = openStore(op != opGC); + + op(opFlags, opArgs); +} + + +string programId = "nix-store"; diff --git a/src/nix-store/xmlgraph.cc b/src/nix-store/xmlgraph.cc new file mode 100644 index 000000000000..1b3ad3d28ad4 --- /dev/null +++ b/src/nix-store/xmlgraph.cc @@ -0,0 +1,71 @@ +#include "xmlgraph.hh" +#include "util.hh" +#include "store-api.hh" + +#include <iostream> + + +using std::cout; + +namespace nix { + + +static inline const string & xmlQuote(const string & s) +{ + // Luckily, store paths shouldn't contain any character that needs to be + // quoted. + return s; +} + + +static string makeEdge(const string & src, const string & dst) +{ + format f = format(" <edge src=\"%1%\" dst=\"%2%\"/>\n") + % xmlQuote(src) % xmlQuote(dst); + return f.str(); +} + + +static string makeNode(const string & id) +{ + format f = format(" <node name=\"%1%\"/>\n") % xmlQuote(id); + return f.str(); +} + + +void printXmlGraph(const PathSet & roots) +{ + PathSet workList(roots); + PathSet doneSet; + + cout << "<?xml version='1.0' encoding='utf-8'?>\n" + << "<nix>\n"; + + while (!workList.empty()) { + Path path = *(workList.begin()); + workList.erase(path); + + if (doneSet.find(path) != doneSet.end()) continue; + doneSet.insert(path); + + cout << makeNode(path); + + PathSet references; + store->queryReferences(path, references); + + for (PathSet::iterator i = references.begin(); + i != references.end(); ++i) + { + if (*i != path) { + workList.insert(*i); + cout << makeEdge(*i, path); + } + } + + } + + cout << "</nix>\n"; +} + + +} diff --git a/src/nix-store/xmlgraph.hh b/src/nix-store/xmlgraph.hh new file mode 100644 index 000000000000..c2216c5a4627 --- /dev/null +++ b/src/nix-store/xmlgraph.hh @@ -0,0 +1,9 @@ +#pragma once + +#include "types.hh" + +namespace nix { + +void printXmlGraph(const PathSet & roots); + +} |