/*********************************************************************/
// dar - disk archive - a backup/restoration program
// Copyright (C) 2002-2022 Denis Corbin
//
// This program is free software; you can redistribute it and/or
// modify it under the terms of the GNU General Public License
// as published by the Free Software Foundation; either version 2
// of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
//
// to contact the author, see the AUTHOR file
/*********************************************************************/

#include "../my_config.h"

extern "C"
{
#if HAVE_STDLIB_H
#include <stdlib.h>
#endif

#if HAVE_GCRYPT_H
#ifndef GCRYPT_NO_DEPRECATED
#define GCRYPT_NO_DEPRECATED
#endif
#include <gcrypt.h>
#endif
} // end extern "C"

#include "macro_tools.hpp"
#include "terminateur.hpp"
#include "user_interaction.hpp"
#include "zapette.hpp"
#include "sar.hpp"
#include "elastic.hpp"
#include "tronc.hpp"
#include "trontextual.hpp"
#include "thread_cancellation.hpp"
#include "deci.hpp"
#include "escape_catalogue.hpp"
#include "tronc.hpp"
#include "cache.hpp"
#include "null_file.hpp"
#include "secu_memory_file.hpp"
#include "generic_to_global_file.hpp"
#include "tlv.hpp"
#include "crypto_sym.hpp"
#include "tronconneuse.hpp"
#include "crypto_asym.hpp"
#include "cat_all_entrees.hpp"
#include "crc.hpp"
#include "entrepot_libcurl.hpp"
#include "scrambler.hpp"
#include "hash_fichier.hpp"
#include "tools.hpp"
#include "compressor.hpp"
#include "compressor_zstd.hpp"
#include "lz4_module.hpp"
#include "gzip_module.hpp"
#include "bzip2_module.hpp"
#include "xz_module.hpp"
#include "lzo_module.hpp"
#include "zstd_module.hpp"
#include "block_compressor.hpp"

#ifdef LIBTHREADAR_AVAILABLE
#include "parallel_block_compressor.hpp"
#include "parallel_tronconneuse.hpp"
#endif

#define PRE_2_7_0_LZO_BLOCK_SIZE 246660

using namespace std;

namespace libdar
{

    const string LIBDAR_STACK_LABEL_CACHE_PIPE = "CACHE_PIPE";
    const string LIBDAR_STACK_LABEL_UNCOMPRESSED = "UNCOMPRESSED";
    const string LIBDAR_STACK_LABEL_CLEAR =  "CLEAR";
    const string LIBDAR_STACK_LABEL_UNCYPHERED = "UNCYPHERED";
    const string LIBDAR_STACK_LABEL_LEVEL1 = "LEVEL1";


	/// this is the archive version format generated by the application
	/// this is also the highest version of format that can be read
    const archive_version macro_tools_supported_version = archive_version(11,1);


    static void version_check(user_interaction & dialog, const header_version & ver);

	/// append an elastic buffer of given size to the file

	/// \param[in,out] f file to append elastic buffer to
	/// \param[in] max_size size of the elastic buffer to add
	/// \param[in] modulo defines the size to choose (see note)
	/// \param[in] offset defines the offset to apply (see note)
	/// \note the size of the elastic buffer should not exceed max_size but
	/// should be chosen in order to reach a size which is zero modulo "modulo"
	/// assuming the offset we add the elastic buffer at is "offset". If modulo is zero
	/// this the elastic buffer is randomly chosen from 1 to max_size without any
	/// concern about being congruent to a given modulo.
	/// Example if module is 5 and offset is 2, the elastic buffer possible size
	/// can be 3 (2+3 is congruent to 0 modulo 5), 8 (2+8 is congruent to modulo 5), 12, etc.
	/// but not exceed max_size+modulo-1
	/// \note this is to accomodate the case when encrypted data is followed by clear data
	/// at the end of an archive. There is no way to known when we read clear data, but we
	/// know the clear data size is very inferior to crypted block size, thus when reading
	/// a uncompleted block of data we can be sure we have reached and of file and that
	/// the data is clear without any encrypted part because else we would have read an entire
	/// block of data.
    static void macro_tools_add_elastic_buffer(generic_file & f,
					       U_32 max_size,
					       U_32 modulo,
					       U_32 offset);

	/// create a compress_module based on the provided arguments
    static unique_ptr<compress_module> make_compress_module_ptr(compression algo, U_I compression_level = 9);

    catalogue *macro_tools_get_catalogue_from(const shared_ptr<user_interaction> & dialog,
					      pile & stack,
					      const header_version & ver,
					      bool info_details,
					      infinint &cat_size,
					      const infinint & second_terminateur_offset,
					      list<signator> & signatories,
					      bool lax_mode)
    {
	return macro_tools_get_derivated_catalogue_from(dialog,
							stack,
							stack,
							ver,
							info_details,
							cat_size,
							second_terminateur_offset,
							signatories,
							lax_mode);
    }

    catalogue *macro_tools_get_derivated_catalogue_from(const shared_ptr<user_interaction> & dialog,
							pile & data_stack,
							pile & cata_stack,
							const header_version & ver,
							bool info_details,
							infinint &cat_size,
							const infinint & second_terminateur_offset,
							list<signator> & signatories,
							bool lax_mode)
    {
        terminateur term;
        catalogue *ret = nullptr;
	pile_descriptor data_pdesc(&data_stack);
	pile_descriptor cata_pdesc(&cata_stack);
	generic_file *crypto = cata_stack.get_by_label(LIBDAR_STACK_LABEL_UNCYPHERED);
	contextual *data_ctxt = nullptr;
	contextual *cata_ctxt = nullptr;

      	if(!dialog)
	    throw SRC_BUG; // dialog points to nothing

	signatories.clear();
	data_stack.find_first_from_top(data_ctxt);
	if(data_ctxt == nullptr)
	    throw SRC_BUG;

	cata_stack.find_first_from_top(cata_ctxt);
	if(cata_ctxt == nullptr)
	    throw SRC_BUG;

        if(info_details)
            dialog->message(gettext("Locating archive contents..."));

	if(ver.get_edition() > 3)
	    term.read_catalogue(*crypto, ver.is_ciphered(), ver.get_edition(), 0);
	    // terminator is encrypted since format "04"
	    // elastic buffer present when encryption is used
	else
	    term.read_catalogue(*crypto, false, ver.get_edition());
	    // elastic buffer did not exist before format "04"
	cata_stack.flush_read_above(crypto);

        if(info_details)
            dialog->message(gettext("Reading archive contents..."));

        if(cata_stack.skip(term.get_catalogue_start()))
        {
	    if(term.get_catalogue_start() > term.get_terminateur_start())
		throw SRC_BUG;
	    cat_size = term.get_terminateur_start() - term.get_catalogue_start();

	    ret = macro_tools_read_catalogue(dialog,
					     ver,
					     cata_pdesc,
					     cat_size,
					     signatories,
					     lax_mode,
					     label_zero,
					     false); // only_detruit

	    if(ret == nullptr)
		throw Ememory("get_catalogue_from");

		// the following is necessary in multi-threaded context
		// where the data_ctxt is a tuyau object:
		// both read data and orders (to change the context value)
		// use the same channel
		// and having this thread changing the contextual
		// info without first stopping subthreads would
		// conflict the thread that may still be reading ahead
		// data from the archive
	    data_stack.flush_read_above(dynamic_cast<generic_file*>(data_ctxt));
	    cata_stack.flush_read_above(dynamic_cast<generic_file*>(cata_ctxt));

	    try
	    {
		data_ctxt->set_info_status(CONTEXT_OP);
		cata_ctxt->set_info_status(CONTEXT_OP);

		if(&cata_stack != &data_stack)
		    ret->change_location(data_pdesc);
	    }
	    catch(...)
	    {
		if(ret != nullptr)
		    delete ret;
		throw;
	    }
	}
	else
	    throw Erange("get_catalogue_from", gettext("Missing catalogue in file."));


        return ret;
    }

    catalogue *macro_tools_read_catalogue(const shared_ptr<user_interaction> & dialog,
					  const header_version & ver,
					  const pile_descriptor & cata_pdesc,
					  const infinint & cat_size,
					  list<signator> & signatories,
					  bool lax_mode,
					  const label & lax_layer1_data_name,
					  bool only_detruits)
    {
        catalogue *ret = nullptr;
	memory_file hash_to_compare;
	hash_fichier *hasher = nullptr;
	signatories.clear();

	cata_pdesc.check(false);

	try // release hasher in case of exception
	{
	    if(ver.is_signed())
	    {
		generic_to_global_file *global_hash_to_compare = nullptr;
		generic_to_global_file *global_cata_top_stack = nullptr;

		try
		{
		    global_hash_to_compare = new (nothrow) generic_to_global_file(dialog, &hash_to_compare, gf_write_only);
		    if(global_hash_to_compare == nullptr)
			throw Ememory("macro_tools_get_derivated_catalogue_from");
		    global_cata_top_stack = new (nothrow) generic_to_global_file(dialog, cata_pdesc.stack->top(), gf_read_only);
		    if(global_cata_top_stack == nullptr)
			throw Ememory("macro_tools_get_derivated_catalogue_from");

		    hasher = new (nothrow) hash_fichier(dialog,
							global_cata_top_stack,
							"x",
							global_hash_to_compare,
							hash_algo::sha512);
		    if(hasher == nullptr)
			throw Ememory("macro_tools_get_derivated_catalogue_from");

			// at this stage, hasher is created
			// and manages the objects global_cata_stack and global_hash_to_compare
		}
		catch(...)
		{
		    if(global_hash_to_compare != nullptr)
			delete global_hash_to_compare;
		    if(global_cata_top_stack != nullptr)
			delete global_cata_top_stack;
		    throw;
		}

		cata_pdesc.stack->push(hasher);
	    }

	    try // trap cast and rethrow exceptions
	    {
		if(!cat_size.is_zero())
		    cata_pdesc.stack->read_ahead(cat_size);
		ret = new (nothrow) catalogue(dialog,
					      cata_pdesc,
					      ver.get_edition(),
					      ver.get_compression_algo(),
					      lax_mode,
					      lax_layer1_data_name,
					      only_detruits);
		if(ret == nullptr)
		    throw Ememory("macro_tools_read_catalogue");
		try
		{

		    if(hasher != nullptr)
		    {
			hasher->terminate();
			if(cata_pdesc.stack->top() != hasher)
			    throw SRC_BUG;
			if(cata_pdesc.stack->pop() != hasher)
			    throw SRC_BUG;
		    }

		    if(ver.is_signed())
		    {
			tlv hash_to_decrypt(*cata_pdesc.stack); // read the encrypted hash following the catalogue
			memory_file clear_read_hash;
			crypto_asym engine(dialog);
			crc *tmp = nullptr;

			hash_to_decrypt.skip(0);
			engine.decrypt(hash_to_decrypt, clear_read_hash);
			signatories = engine.verify();
			if(clear_read_hash.diff(hash_to_compare, 0, 0, 1, tmp)) // difference!
			{
			    if(lax_mode)
				dialog->message(gettext("LAX MODE: catalogue computed hash does not match the signed hash of the archive, ignoring"));
			    else
				throw Edata(gettext("Catalogue computed hash does not match the signed hash of the archive, archive has been modified since it was signed!"));
			}
			else
			{
			    if(tmp != nullptr)
				delete tmp;
				// else no difference,
				// the caller has the signatories and will compare those with the list contained in the archive header
			}
		    }
		}
		catch(...)
		{
		    if(ret != nullptr)
			delete ret;
		    throw;
		}

	    }
	    catch(Ebug & e)
	    {
		throw;
	    }
	    catch(Ethread_cancel & e)
	    {
		throw;
	    }
	    catch(Egeneric & e)
	    {
		throw Erange("get_catalogue_from", string(gettext("Cannot open catalogue: ")) + e.get_message());
	    }
	}
	catch(...)
	{
	    if(cata_pdesc.stack->top() == hasher)
	    {
		if(cata_pdesc.stack->pop() != hasher)
		    throw SRC_BUG;
	    }
	    if(hasher != nullptr)
		delete hasher;
	    throw;
	}

	if(hasher != nullptr)
	    delete hasher;

	return ret;
    }


    void macro_tools_open_archive(const shared_ptr<user_interaction> & dialog,
				  const shared_ptr<entrepot> & where,
                                  const string &basename,
				  const infinint & min_digits,
                                  const string &extension,
				  crypto_algo crypto,
                                  const secu_string & pass,
				  U_32 crypto_size,
				  pile & stack,
                                  header_version &ver,
                                  const string &input_pipe,
                                  const string &output_pipe,
                                  const string & execute,
				  infinint & second_terminateur_offset,
				  bool lax,
				  bool has_external_cat,
				  bool sequential_read,
				  bool info_details,
				  list<signator> & gnupg_signed,
				  slice_layout & sl,
				  U_I multi_threaded_crypto,
				  U_I multi_threaded_compress,
				  bool header_only)
    {
	secu_string real_pass = pass;
	generic_file *tmp = nullptr;
	contextual *tmp_ctxt = nullptr;
	cache *tmp_cache = nullptr;
#ifdef LIBCURL_AVAILABLE
	bool libcurl_repo = dynamic_cast<const entrepot_libcurl *>(where.get()) != nullptr;
#else
	bool libcurl_repo = false;
#endif
	string salt;

	if(!dialog)
	    throw SRC_BUG; // dialog points to nothing

	stack.clear();
#ifdef LIBTHREADAR_AVAILABLE
	if(multi_threaded_crypto < 2 && multi_threaded_compress < 2 && !libcurl_repo)
	    stack.ignore_read_ahead(true);
	else
	    stack.ignore_read_ahead(false);
#else
	stack.ignore_read_ahead(true);
#endif
	sl.first_size = 0;
	sl.other_size = 0; // we will change that only if sar object is used

	try
	{

		// ****** Building the sar/tuyau/null layer aka level 1 ******* //

	    if(basename == "-")
	    {
		if(sequential_read)
		{
		    if(input_pipe == "")
		    {
			if(info_details)
			    dialog->message(gettext("Opening standard input to read the archive..."));
			tmp = new (nothrow) trivial_sar(dialog,
							basename,
							lax);
		    }
		    else
		    {
			if(info_details)
			    dialog->printf(gettext("Opening named pipe %S as input to read the archive..."), &input_pipe);
			tmp = new (nothrow) trivial_sar(dialog,
							input_pipe,
							lax);
		    }
		}
		else
		{
		    tuyau *in = nullptr;
		    tuyau *out = nullptr;

		    try
		    {
			dialog->printf(gettext("Opening a pair of pipes to read the archive, expecting dar_slave at the other ends..."));
			macro_tools_open_pipes(dialog, input_pipe, output_pipe, in, out);
			tmp = new (nothrow) zapette(dialog, in, out, true);
			if(tmp == nullptr)
			{
			    delete in;
			    in = nullptr;
			    delete out;
			    out = nullptr;
			}
			else
			{
			    in = out = nullptr; // now managed by the zapette
			    tmp->skip_to_eof(); // not sequential reading mode we must skip at eof
			}
		    }
		    catch(...)
		    {
			if(in != nullptr)
			    delete in;
			if(out != nullptr)
			    delete out;
			throw;
		    }
		}
	    }
	    else
	    {
		sar *tmp_sar = nullptr;
		if(info_details)
		    dialog->message(gettext("Opening the archive using the multi-slice abstraction layer..."));
		tmp = tmp_sar = new (nothrow) sar(dialog,
						  basename,
						  extension,
						  where,
						  !sequential_read, // not openned by the end in sequential read mode
						  min_digits,
						  sequential_read,
						  lax,
						  execute);
		if(tmp_sar != nullptr)
		    sl = tmp_sar->get_slicing();
	    }

	    if(tmp == nullptr)
		throw Ememory("open_archive");
	    else
	    {
		    // we always ignore read_ahead as no slave thread will exist for LEVEL1 layer
		    // except for libcurl which can leverage it
		tmp->ignore_read_ahead(!libcurl_repo);
		stack.push(tmp, LIBDAR_STACK_LABEL_LEVEL1);
		tmp = nullptr;
	    }


		// ****** Reading the header version ************** //

	    stack.find_first_from_top(tmp_ctxt);
	    if(tmp_ctxt == nullptr)
		throw SRC_BUG;
	    second_terminateur_offset = 0;

	    if(info_details)
	    {
		if(sequential_read || (tmp_ctxt != nullptr && tmp_ctxt->is_an_old_start_end_archive()))
		    dialog->message(gettext("Reading the archive header..."));
		else
		    dialog->message(gettext("Reading the archive trailer..."));
	    }

	    if(sequential_read
	       || tmp_ctxt->is_an_old_start_end_archive()
	       || stack.get_position().is_zero()) //< sar layer failed openning the last slice and fallen back openning the first slice
		stack.skip(0);
	    else
	    {
		terminateur term;

		try
		{
		    term.read_catalogue(stack, false, macro_tools_supported_version);
		    stack.skip(term.get_catalogue_start());
		    second_terminateur_offset = term.get_catalogue_start();
		}
		catch(Erange & e)
		{
		    dialog->printf(gettext("Error while reading archive's header, this may be because this archive is an old encrypted archive or that data corruption took place, Assuming it is an old archive, we have to read the header at the beginning of the first slice..."));
		    stack.skip(0);
		}
	    }

	    ver.read(stack, *dialog, lax);

	    if(second_terminateur_offset.is_zero() && !sequential_read && ver.get_edition() > 7)
		if(!has_external_cat)
		{
		    if(!lax)
			throw Erange("macro_tools_open_archive",gettext("Found a correct archive header at the beginning of the archive, which does not stands to be an old archive, the end of the archive is corrupted and thus the catalogue is not readable, aborting. Either retry providing in addition an isolated catalogue of that archive to perform the operation, or try reading the archive in sequential mode or try in lax mode or, last chance, try both lax and sequential read mode at the same time"));
		    else
			dialog->pause(gettext("Found a correct archive header at the beginning of the archive, which does not stands to be an old archive, the end of the archive is thus corrupted. Without external catalogue provided and as we do not read the archive in sequential mode, there is very little chance to retreive something from this corrupted archive. Do we continue anyway ?"));
		}

	    if(header_only)
		return;

		// *************  adding a tronc to hide last terminator and trailer_version ******* //

	    if(!second_terminateur_offset.is_zero())
	    {
		if(info_details)
		    dialog->printf(gettext("Opening construction layer..."));
		tmp = new (nothrow) tronc(stack.top(), 0, second_terminateur_offset, false);
		if(tmp == nullptr)
		    throw Ememory("macro_tools_open_archive");
		else
		{
		    tmp->ignore_read_ahead(!libcurl_repo);
			// no slave thread used below in the stack
		    stack.clear_label(LIBDAR_STACK_LABEL_LEVEL1);
		    stack.push(tmp, LIBDAR_STACK_LABEL_LEVEL1);
		    tmp = nullptr;
		}
	    }

		// *************  building the encryption layer if necessary ************** //

	    if(info_details)
		dialog->message(gettext("Considering cyphering layer..."));

	    if(ver.is_ciphered() && ver.get_edition() >= 9 && crypto == crypto_algo::none && ver.get_sym_crypto_algo() != crypto_algo::none)
	    {
		try
		{
		    if(lax)
			dialog->pause("LAX MODE: Do we use the encryption algorithm stored in the archive header/trailer?");
		    crypto = ver.get_sym_crypto_algo(); // using the crypto algorithm recorded in the archive
		}
		catch(Euser_abort &)
		{
		    crypto = crypto_algo::none;
		}
	    }

	    if(ver.is_ciphered() && crypto == crypto_algo::none)
	    {
		if(lax)
		{
		    dialog->message(gettext("LAX MODE: Archive is flagged as being ciphered, assuming data corruption occurred and considering the archive is not ciphered"));
		}
		else
		    throw Erange("macro_tools_open_archive", tools_printf(gettext("The archive %S is encrypted and no encryption cipher has been given, cannot open archive."), &basename));
	    }

	    if(ver.get_crypted_key() != nullptr) // we will find the passphrase from the header's encrypted key
	    {

		    // detemining the size of the unencrypted key

		infinint i_size = ver.get_crypted_key()->size();
		U_I size = 0;
		i_size.unstack(size);
		if(!i_size.is_zero())
		    throw SRC_BUG;

		    // unciphering the encrypted key using GnuPG user's keyring, asking for passphrase if necessary

		secu_memory_file clear_key = secu_memory_file(size);
		crypto_asym engine(dialog);
		ver.get_crypted_key()->skip(0);
		clear_key.skip(0);
		engine.decrypt(*(ver.get_crypted_key()), clear_key);
		gnupg_signed = engine.verify();

		    // substitution of the pass by the clear_key if decrypt succeeded (else it throws an exception)

		real_pass = clear_key.get_contents();
	    }

	    if(crypto != crypto_algo::none && real_pass == "")
	    {
		if(!secu_string::is_string_secured())
		    dialog->message(gettext("WARNING: support for secure memory was not available at compilation time, in case of heavy memory load, this may lead the password you are about to provide to be wrote to disk (swap space) in clear. You have been warned!"));
		real_pass = dialog->get_secu_string(tools_printf(gettext("Archive %S requires a password: "), &basename), false);
	    }

	    switch(crypto)
	    {
	    case crypto_algo::none:
		if(!ver.get_tape_marks())
		{
		    if(info_details)
			dialog->message(gettext("No cyphering layer opened, adding cache layer for better performance"));

			// adding the cache layer only if no escape layer will tape place
			// over. escape layer act a bit like a cache, making caching here useless
		    tmp = tmp_cache = new (nothrow) cache (*(stack.top()), false);
		    if(tmp == nullptr)
			dialog->message(gettext("Failed opening the cache layer, lack of memory, archive read performances will not be optimized"));
		}
		else
		{
		    if(info_details)
			dialog->message(gettext("No cyphering layer opened"));
		}
		break;
	    case crypto_algo::blowfish:
	    case crypto_algo::aes256:
	    case crypto_algo::twofish256:
	    case crypto_algo::serpent256:
	    case crypto_algo::camellia256:
		if(info_details)
		    dialog->message(gettext("Opening cyphering layer..."));
#ifdef LIBDAR_NO_OPTIMIZATION
		tools_secu_string_show(*dialog, string("Used clear key: "), real_pass);
#endif

		try
		{
		    proto_tronco *tmp_ptr = nullptr;
		    unique_ptr<crypto_module> ptr;
		    try
		    {
			ptr = make_unique<crypto_sym>(real_pass,
						      ver.get_edition(),
						      crypto,
						      ver.get_salt(),
						      ver.get_iteration_count(),
						      ver.get_kdf_hash(),
						      ver.get_crypted_key() == nullptr);
		    }
		    catch(bad_alloc &)
		    {
			throw Ememory("macro_tools_open_archive");
		    }

		    if(!second_terminateur_offset.is_zero()
		       || tmp_ctxt->is_an_old_start_end_archive()) // we have openned the archive by the end
		    {
			if(multi_threaded_crypto > 1)
			{
#if LIBTHREADAR_AVAILABLE
			    tmp = tmp_ptr = new (nothrow) parallel_tronconneuse(multi_threaded_crypto,
										crypto_size,
										*(stack.top()),
										ver.get_edition(),
										ptr);
			    if(info_details)
				dialog->message(tools_printf(gettext("multi-threaded cyphering layer open, with %d worker thread(s)"), multi_threaded_crypto));

#else
			    throw Ecompilation(gettext("libthreadar is required at compilation time in order to use more than one thread for cryptography"));
#endif
			}
			else
			{
			    tmp = tmp_ptr = new (nothrow) tronconneuse(crypto_size,
								       *(stack.top()),
								       ver.get_edition(),
								       ptr);
			    if(info_details)
				dialog->message(tools_printf(gettext("single-threaded cyphering layer open")));
			}

			if(tmp_ptr != nullptr)
			    tmp_ptr->set_initial_shift(ver.get_initial_offset());
		    }
		    else // archive openned by the beginning
		    {
			if(multi_threaded_crypto > 1)
			{
#if LIBTHREADAR_AVAILABLE
			    tmp = tmp_ptr = new (nothrow) parallel_tronconneuse(multi_threaded_crypto,
										crypto_size,
										*(stack.top()),
										ver.get_edition(),
										ptr);
			    if(info_details)
				dialog->message(tools_printf(gettext("multi-threaded cyphering layer open, with %d worker thread(s)"), multi_threaded_crypto));

#else
			    throw Ecompilation(gettext("libthreadar is required at compilation time in order to use more than one thread for cryptography"));
#endif
			}
			else
			{
			    tmp = tmp_ptr = new (nothrow) tronconneuse(crypto_size,
								       *(stack.top()),
								       ver.get_edition(),
								       ptr);
			    if(info_details)
				dialog->message(tools_printf(gettext("single-threaded cyphering layer open")));
			}

			if(tmp_ptr != nullptr)
			{
			    tmp_ptr->set_callback_trailing_clear_data(&macro_tools_get_terminator_start);
			    tmp_ptr->set_initial_shift((stack.top())->get_position());

			    if(sequential_read)
				elastic(*tmp_ptr, elastic_forward, ver.get_edition()); // this is line creates a temporary anonymous object and destroys it just afterward
				// this is necessary to skip the reading of the initial elastic buffer
				// nothing prevents the elastic buffer from carrying what could
				// be considered an escape mark.
			}
		    }
		}
		catch(std::bad_alloc &)
		{
		    throw Ememory("macro_tools_open_archive");
		}
		break;
	    case crypto_algo::scrambling:
		if(info_details)
		    dialog->message(gettext("Opening cyphering layer..."));
#ifdef LIBDAR_NO_OPTIMIZATION
		tools_secu_string_show(*dialog, string("Used clear key: "), real_pass);
#endif
		tmp = new (nothrow) scrambler(real_pass, *(stack.top()));
		break;
	    default:
		throw Erange("macro_tools_open_archive", gettext("Unknown encryption algorithm"));
	    }

	    if(tmp == nullptr)
	    {
		if(crypto != crypto_algo::none || !ver.get_tape_marks())
		    throw Ememory("open_archive");
	    }
	    else
	    {
		    // we always ignore read ahead as encryption layer above sar/zapette/triial_sar has no slave thread below
		tmp->ignore_read_ahead(!libcurl_repo);
		stack.push(tmp);
		tmp = nullptr;
	    }

	    stack.add_label(LIBDAR_STACK_LABEL_UNCYPHERED);

		// *************** building the escape layer if necessary ************ //

	    try
	    {
		if(ver.get_tape_marks())
		{
		    if(info_details)
			dialog->message(gettext("Opening escape sequence abstraction layer..."));
		    if(lax)
		    {
			if(!sequential_read)
			{
			    dialog->pause(gettext("LAX MODE: Archive is flagged as having escape sequence (which is normal in recent archive versions). However if this is not expected, shall I assume a data corruption occurred in this field and that this flag should be ignored? (If unsure, refuse)"));
			    ver.set_tape_marks(false);
			}
			else
			    throw Euser_abort("this exception triggers the creation of the escape layer");
			    // else in lax & sequential_read, escape sequences are mandatory, we do not propose to ignore them
		    }
		    else
			throw Euser_abort("this exception triggers the creation of the escape layer");
		}
		else // no escape layer in the archive
		{
		    if(!lax)
		    {
			if(sequential_read)
			    throw Erange("macro_tools_open_archive", gettext("Sequential read asked, but this archive is flagged to not have the necessary embedded escape sequences for that operation, aborting"));
		    }
		    else // lax mode
			if(sequential_read)
			{
			    dialog->message(gettext("LAX MODE: the requested sequential read mode relies on escape sequence which seem to be absent from this archive. Assuming data corruption occurred. However, if no data corruption occurred and thus no escape sequence are present in this archive, do not use sequential reading mode to explore this archive else you will just get nothing usable from it"));
			    ver.set_tape_marks(true);
			    throw Euser_abort("this exception triggers the creation of the escape layer");
			}
			else // normal mode
			    if(ver.get_edition() >= 8) // most usually escape mark are present, thus we must warn
				dialog->pause(gettext("LAX MODE: Archive is flagged to not have escape sequence which is not the case by default since archive format 8 (release 2.4.x). If corruption occurred and an escape sequence is present, this may lead data restoration to fail, answering no at this question will let me consider that an escape sequence layer has to be added in spite of the archive flags. Do you want to continue as suggested by the archive flag, thus without escape sequence layer?"));
		}
	    }
	    catch(Euser_abort & e)
	    {
		set<escape::sequence_type> unjump;

		unjump.insert(escape::seqt_catalogue);
		tmp = new (nothrow) escape(stack.top(), unjump);
		if(tmp == nullptr)
		    throw Ememory("open_archive");
		tmp->ignore_read_ahead(!libcurl_repo);
		stack.push(tmp);
		tmp = nullptr;
	    }

	    stack.add_label(LIBDAR_STACK_LABEL_CLEAR);


		// *********** building the compression layer ************* //

	    if(info_details)
	    {
		if(ver.get_compression_algo() == compression::none)
		    dialog->message(gettext("Opening the compression abstraction layer (compression algorithm used is none)..."));
		else
		    dialog->message(gettext("Opening the compression layer..."));
	    }

	    version_check(*dialog, ver);

	    if(ver.get_compression_block_size().is_zero())
	    {
		tmp = macro_tools_build_streaming_compressor(ver.get_compression_algo(),
							     *(stack.top()),
							     9, // not used for decompression
							     multi_threaded_compress);

		if(info_details)
		    dialog->message(tools_printf(gettext("streamed compression layer open (single threaded)")));
	    }
	    else
	    {
		U_I compr_bs = 0;
		infinint ibs = ver.get_compression_block_size();

		ibs.unstack(compr_bs);
		if(!ibs.is_zero())
		    throw Erange("macro_tools_open_layers", gettext("compression block size used in the archive exceed integer capacity of the current system"));

		tmp = macro_tools_build_block_compressor(ver.get_compression_algo(),
							 *(stack.top()),
							 9, // not used for decompression
							 multi_threaded_compress,
							 compr_bs);

		if(info_details)
		    dialog->message(tools_printf(gettext("block compression layer open with %d worker thread(s)"), multi_threaded_compress));
	    }

	    if(tmp == nullptr)
		throw Ememory("open_archive");
	    else
	    {
#ifdef LIBTHREADAR_AVAILABLE
		if(multi_threaded_crypto > 1)
		    tmp->ignore_read_ahead(!libcurl_repo);
#else
		tmp->ignore_read_ahead(!libcurl_repo);
#endif
		stack.push(tmp, LIBDAR_STACK_LABEL_UNCOMPRESSED);
		tmp = nullptr;
	    }

		// ************* warning info ************************ //

	    if(info_details)
		dialog->message(gettext("All layers have been created successfully"));

	    if(ver.is_ciphered())
		dialog->printf(gettext("Warning, the archive %S has been encrypted. A wrong key is not possible to detect, it would cause DAR to report the archive as corrupted"),  &basename);
	}
	catch(...)
	{
	    if(tmp != nullptr)
		delete tmp;
	    stack.clear();
	    throw;
	}
    }

    catalogue *macro_tools_lax_search_catalogue(const shared_ptr<user_interaction> & dialog,
						pile & stack,
						const archive_version & edition,
						compression compr_algo,
						bool info_details,
						bool even_partial_catalogue,
						const label & layer1_data_name)
    {
	catalogue *ret = nullptr;
	thread_cancellation thr_cancel;
	pile_descriptor pdesc(&stack);
	bool ok = false;
	U_I check_abort_every = 10000;
	U_I check_abort_count = check_abort_every;
	infinint offset;
	infinint max_offset;
	infinint min_offset;
	infinint amplitude;
	entree_stats stats;
	infinint fraction = 101;

	if(!dialog)
	    throw SRC_BUG; // dialog points to nothing

	    // obtaining from the user the fraction of the archive to inspect

	do
	{
	    string answ = dialog->get_string(gettext("LAX MODE: The catalogue (table of contents) usually takes a few percents of the archive at its end, which percentage do you want me to scan (answer by an *integer* number between 0 and 100)? "), true);
	    try
	    {
		deci num = answ;
		fraction = num.computer();
		if(fraction > 100)
		    dialog->printf(gettext("LAX MODE: %i is not a valid percent value"), &fraction);
	    }
	    catch(Edeci & e)
	    {
		dialog->printf(gettext("%S is not a valid number"), & answ);
	    }
	}
	while(fraction > 100 || fraction.is_zero());

	if(info_details)
	    dialog->printf(gettext("LAX MODE: Beginning search of the catalogue (from the end toward the beginning of the archive, on %i %% of its length), this may take a while..."), &fraction);


	    // finding the upper limit of the search

	if(stack.skip_to_eof())
	    max_offset = stack.get_position();
	else
	{
	    dialog->message(gettext("LAX MODE: Cannot skip at the end of the archive! Using current position to start the catalogue search"));
	    max_offset = stack.get_position();
	}

	if(max_offset.is_zero())
	    throw Erange("macro_tools_lax_search_catalogue", gettext("LAX MODE: Failed to read the catalogue (no data to inspect)"));

	if(fraction.is_zero())
    	    throw Erange("macro_tools_lax_search_catalogue", gettext("LAX MODE: Failed to read the catalogue (0 bytes of the archive length asked to look for the catalogue)"));


	    // calculating starting offset and offset range

	min_offset = ((infinint)100 - fraction)*max_offset/100;
	amplitude = max_offset - min_offset;


	if(pdesc.esc != nullptr)
	{
	    dialog->message(gettext("LAX MODE: Escape sequence seems present in this archive. I have thus two different methods, either I look for the escape sequence indicating the start of the catalogue or I try each position in turn in the hope it will not be data that look like a catalogue"));
	    try
	    {
		dialog->pause(gettext("LAX MODE: Trying to locate the escape sequence (safer choice) ?"));
		if(!pdesc.esc->skip(min_offset))
		    throw SRC_BUG;
		if(pdesc.esc->skip_to_next_mark(escape::seqt_catalogue, true))
		{
		    dialog->message(gettext("LAX MODE: Good point! I could find the escape sequence marking the beginning of the catalogue, now trying to read it..."));
		    pdesc.stack->flush_read_above(pdesc.esc);
		    if(pdesc.stack->get_position() != pdesc.esc->get_position())
			throw SRC_BUG;
		    offset = pdesc.stack->get_position();
		    min_offset = offset; // no need to read before this position, we cannot fetch the catalogue there
		}
		else
		{
		    dialog->message(gettext("LAX MODE: Escape sequence could not be found, it may have been corrupted or out of the scanned portion of the archive, trying to find the catalogue the other way"));
		    throw Euser_abort("THIS MESSAGE SHOULD NEVER APPEAR, It branches the execution to the second way of looking for the catalogue");
		}
	    }
	    catch(Euser_abort & e)
	    {
		offset = min_offset;
		pdesc.stack->skip(offset);
	    }
	}

	while(!ok)
	{

		// checking whether thread cancellation has been asked

	    if(++check_abort_count > check_abort_every)
	    {
		thr_cancel.check_self_cancellation();
		check_abort_count = 0;
		if(info_details)
		{
		    infinint ratio = (offset - min_offset)*100/amplitude;
		    dialog->message(tools_printf(gettext("LAX MODE: %i %% remaining"), & ratio));
		}
	    }

		// trying to read a catalogue at the "offset" position

	    try
	    {
		ret = new (nothrow) catalogue(dialog, pdesc, edition, compr_algo, even_partial_catalogue, layer1_data_name);
		if(ret == nullptr)
		    throw Ememory("macro_tools_lax_search_catalogue");
		stats = ret->get_stats();
		dialog->printf(gettext("Could read a catalogue data structure at offset %i, it contains the following:"), &offset);
		stats.listing(*dialog);
		dialog->pause(gettext("Do you want to use it for the operation?"));
		ok = true;
	    }
	    catch(Ebug & e)
	    {
		throw;
	    }
	    catch(Ethread_cancel & e)
	    {
		throw;
	    }
	    catch(Efeature & e)
	    {
		throw;
	    }
	    catch(Ehardware & e)
	    {
		throw;
	    }
	    catch(Escript & e)
	    {
		throw;
	    }
	    catch(Ecompilation & e)
	    {
		throw;
	    }
	    catch(Egeneric & e)
	    {
		if(offset >= max_offset)
		{
		    if(info_details)
			dialog->message(gettext("LAX MODE: Reached the end of the area to scan, FAILED to find any catalogue"));
		    ret = nullptr;
		    ok = true;
		}
		else
		    stack.skip(++offset);
	    }
	}

	if(ret == nullptr)
	    throw Erange("macro_tools_lax_search_catalogue", gettext("LAX MODE: Failed to read the catalogue"));
	else
	    return ret;
    }

    infinint macro_tools_get_terminator_start(generic_file & f, const archive_version & reading_ver)
    {
	terminateur term;

        f.skip_to_eof();
	term.read_catalogue(f, false, reading_ver);
        return term.get_catalogue_start();
    }

    void macro_tools_create_layers(const shared_ptr<user_interaction> & dialog,
				   pile & layers,
				   header_version & ver,
				   slice_layout & slicing,
				   const slice_layout *ref_slicing,
				   const shared_ptr<entrepot> & sauv_path_t,
				   const string & filename,
				   const string & extension,
				   bool allow_over,
				   bool warn_over,
				   bool info_details,
				   const infinint & pause,
				   compression algo,
				   U_I compression_level,
				   U_I compression_block_size,
				   const infinint & file_size,
				   const infinint & first_file_size,
				   const string & execute,
				   crypto_algo crypto,
				   const secu_string & pass,
				   U_32 crypto_size,
				   const vector<string> & gnupg_recipients,
				   const vector<string> & gnupg_signatories,
				   bool empty,
				   const string & slice_permission,
				   bool add_marks_for_sequential_reading,
				   const string & user_comment,
				   hash_algo hash,
				   const infinint & slice_min_digits,
				   const label & internal_name,
				   const label & data_name,
				   const infinint & iteration_count,
				   hash_algo kdf_hash,
				   U_I multi_threaded_crypto,
				   U_I multi_threaded_compress)
    {
#if GPGME_SUPPORT
	U_I gnupg_key_size;
#endif

	try
	{
	    generic_file *tmp = nullptr;
	    escape *esc = nullptr;
	    generic_file *level1 = nullptr;
	    bool force_permission = (slice_permission != "");
	    U_I permission = force_permission ? tools_octal2int(slice_permission) : 0; // 0 or anything else, this does not matter
	    gf_mode open_mode = gf_read_write; // by default first layer is read-write except in case of hashing or encryption
		// read-write mode is used when skipping back due to file change, the escape layer needs to read the few
		// bytes before the backward position to take care of tape marks
	    slicing.other_size = file_size;
	    slicing.first_size = first_file_size;

	    layers.clear();

	    secu_string real_pass = pass;

	    if(hash != hash_algo::none || crypto != crypto_algo::none)
		open_mode = gf_write_only;

	    try
	    {
		bool writing_to_pipe = false;

		    // note:
		    // the escape object if present in the stack needs read access from below it when skipping backward
		    // is requested (bad compression ration, changed filed at the time of backup, etc.)

		    // **********  building the level 1 generic_file layer *********** //

		if(empty)
		{
		    if(info_details)
			dialog->message(gettext("Creating low layer: Writing archive into a black hole object (equivalent to /dev/null)..."));

		    tmp = new (nothrow) null_file(open_mode);
		}
		else
		    if(file_size.is_zero()) // one SLICE
			if(filename == "-") // output to stdout
			{
			    if(info_details)
				dialog->message(gettext("Creating low layer: Writing archive into standard output object..."));
			    writing_to_pipe = true;
			    tmp = macro_tools_open_archive_tuyau(dialog,
								 1,
								 gf_write_only, // always write only
								 internal_name,
								 data_name,
								 false,
								 execute); //archive goes to stdout
			}
			else
			{
			    if(info_details)
				dialog->message(gettext("Creating low layer: Writing archive into a plain file object..."));
			    tmp = new (nothrow) trivial_sar(dialog,
							    open_mode,
							    filename,
							    extension,
							    *sauv_path_t, // entrepot !!
							    internal_name,
							    data_name,
							    execute,
							    allow_over,
							    warn_over,
							    force_permission,
							    permission,
							    hash,
							    slice_min_digits,
							    false);
			}
		    else
		    {
			sar *tmp_sar = nullptr;
			if(info_details)
			    dialog->message(gettext("Creating low layer: Writing archive into a sar object (Segmentation and Reassembly) for slicing..."));
			tmp = tmp_sar = new (nothrow) sar(dialog,
							  open_mode,
							  filename,
							  extension,
							  file_size,
							  first_file_size,
							  warn_over,
							  allow_over,
							  pause,
							  sauv_path_t, // shared_ptr<entrepot> !!
							  internal_name,
							  data_name,
							  force_permission,
							  permission,
							  hash,
							  slice_min_digits,
							  false,
							  execute);

			if(tmp_sar != nullptr)
			    slicing = tmp_sar->get_slicing();
		    }


		if(tmp == nullptr)
		    throw Ememory("op_create_in_sub");
		else
		{
		    layers.push(tmp);
		    level1 = tmp; // recorded to know where to drop the archive header while the encryption layer will already be added on top
		    tmp = nullptr;
		}

		    // ******** adding cache layer if writing to pipe in order to provide limited read/write mode ***** //

		if(writing_to_pipe)
		{
		    if(info_details)
			dialog->message(gettext("Adding cache layer over pipe to provide limited skippability..."));

		    cache *c_tmp = new (nothrow) cache(*(layers.top()), true);
		    if(c_tmp == nullptr)
			throw Ememory("op_create_in_sub");
		    else
		    {
			tmp = c_tmp; // to handle the object destruction in case of exception
			if(open_mode == gf_read_write)
			    c_tmp->change_to_read_write();
			layers.push(c_tmp, LIBDAR_STACK_LABEL_CACHE_PIPE, true);
			level1 = tmp; // we will use the cache as layer where to write down the archive header
			tmp = nullptr;
		    }
		}

		    // ***** preparing the keys for encryption ********************* //

		if(crypto != crypto_algo::none || !gnupg_recipients.empty())
		{
		    if(!secu_string::is_string_secured())
			dialog->message(gettext("WARNING: support for secure memory was not available at compilation time, in case of heavy memory load, this may lead the password/passphrase provided to be wrote to disk (swap space) in clear. You have been warned!"));
		}

		if(!gnupg_recipients.empty())
		{
#if GPGME_SUPPORT
		    memory_file *key = new (nothrow) memory_file();
		    if(key == nullptr)
			throw Ememory("macro_tools_create_layers");

		    try
		    {
			    // generating random key for symmetric encryption

			if(info_details)
			    dialog->message(gettext("Generating random key for symmetric encryption..."));

			switch(crypto)
			{
			case crypto_algo::none:
			    throw SRC_BUG;
			case crypto_algo::scrambling:
			    throw Erange("macro_tools_create_layers",
					 gettext("Scrambling is a very weak encryption algorithm, this is a non-sens to use with asymmetric encryption"));
			    break;
			case crypto_algo::blowfish:
			case crypto_algo::aes256:
			case crypto_algo::twofish256:
			case crypto_algo::serpent256:
			case crypto_algo::camellia256:
			    gnupg_key_size = tools_max(crypto_sym::max_key_len(crypto),
						       crypto_sym::max_key_len_libdar(crypto));
			    break;
			default:
			    throw SRC_BUG;
			}

			if(gnupg_key_size == 0)
			    throw SRC_BUG;

			secu_memory_file clear(gnupg_key_size);
			clear.randomize(gnupg_key_size);
			U_I iter = 0;
			U_I next = 1;

			    // checking the strength of the random key

			switch(crypto)
			{
			case crypto_algo::none:
			    throw SRC_BUG;
			case crypto_algo::scrambling:
			    break;
			case crypto_algo::blowfish:
			case crypto_algo::aes256:
			case crypto_algo::twofish256:
			case crypto_algo::serpent256:
			case crypto_algo::camellia256:
			    while(!crypto_sym::is_a_strong_password(crypto, clear.get_contents()))
			    {
				clear.randomize(gnupg_key_size);
				++iter;
				if(iter % next == 0)
				    dialog->message(tools_printf(gettext("For your information, this is the iteration %d for which the randomly generated key is reported to be weak by libgcrypt, continuing generating another random key... patience"), iter));
				next *= 10;
			    }
			    if(iter > 1)
				dialog->message(tools_printf(gettext("... A strong randomly generated key could be found after %d iteration(s)"), iter));
			    break;
			default:
			    throw SRC_BUG;
			}

			if(info_details)
			    dialog->message(gettext("Key generated"));

#ifdef LIBDAR_NO_OPTIMIZATION
			tools_secu_string_show(*dialog, string("Generated key: "), clear.get_contents());
#endif

			    // encrypting the symmetric key with asymetric algorithm

			crypto_asym engine(dialog);
			if(!gnupg_signatories.empty())
			{
			    if(gnupg_recipients.size() > 1)
				dialog->message(gettext("WARNING: When signing an archive with more than one recipient, there is a theorical risk that a recipient forges a slightly modified version of the generated archive without breaking the signature validation. It is advised in that context to sign the archive externally by calling \"gpg --detach-sign\" once dar has completed the archive"));
			    engine.set_signatories(gnupg_signatories);
			}
			clear.skip(0);
			key->skip(0);
			if(clear.get_size().is_zero())
			    throw SRC_BUG;
			engine.encrypt(gnupg_recipients, clear, *key);

			    // using the generated key for the crypto layer that will be setup below
			real_pass = clear.get_contents();
			if(real_pass.get_size() == 0)
			    throw SRC_BUG;

			if(crypto == crypto_algo::none)
			    crypto = crypto_algo::blowfish;

			    // updating the hader version with the symmetric key encrypted with the asymmetrical algorithm
			ver.set_crypted_key(key);
			key = nullptr; // now the pointed to object is under the responsibility of ver object
		    }
		    catch(...)
		    {
			if(key != nullptr)
			    delete key;
			throw;
		    }
#else
		    throw Ecompilation("Strong Encryption support");
#endif
		}

		if(crypto != crypto_algo::none)
		{
			// optaining a password on-fly if necessary

		    if(real_pass.get_size() == 0)
		    {
			secu_string t1 = dialog->get_secu_string(tools_printf(gettext("Archive %S requires a password: "), &filename), false);
			secu_string t2 = dialog->get_secu_string(gettext("Please confirm your password: "), false);
			if(t1 == t2)
			    real_pass = t1;
			else
			    throw Erange("op_create_in_sub", gettext("The two passwords are not identical. Aborting"));
		    }
		}

		    // ************ building the encryption layer if required ****** //
		    // the encryption layer need to be setup now, for the salt to be determined,
		    // it will be recorded in the archive header which has to be wrote down at the
		    // beginning of the archive.
		    // Once the archvie header has been wrote the initial_shift (offset of the first
		    // encrypted data) will be possible to be given to the encryption layer
		    //
		    // this info is necessary to determine the encryption block boundaries.

		proto_tronco *tmp_tronco = nullptr;
		bool use_pkcs5 = gnupg_recipients.empty();

		switch(crypto)
		{
		case crypto_algo::scrambling:
		    if(info_details)
			dialog->message(gettext("Adding a new layer on top: scrambler object..."));
		    tmp = new (nothrow) scrambler(real_pass, *(layers.top()));
#ifdef LIBDAR_NO_OPTIMIZATION
		    tools_secu_string_show(*dialog, string("real_pass used: "), real_pass);
#endif
		    break;
		case crypto_algo::blowfish:
		case crypto_algo::aes256:
		case crypto_algo::twofish256:
		case crypto_algo::serpent256:
		case crypto_algo::camellia256:
		    if(info_details)
			dialog->message(gettext("Adding a new layer on top: Strong encryption object..."));
		    try
		    {
			unique_ptr<crypto_module> ptr = make_unique<crypto_sym>(real_pass,
										macro_tools_supported_version,
										crypto,
										"",   // the salt, will be generated by the crypto_module
										iteration_count,
										kdf_hash,
										use_pkcs5);

			if(!ptr)
			    throw Ememory("macro_tools_create_layers");

			    // updating the archive_header with the KDF parameters
			crypto_sym* sym_ptr = dynamic_cast<crypto_sym*>(ptr.get());
			if(sym_ptr != nullptr)
			{
			    if(use_pkcs5)
			    {
				ver.set_salt(sym_ptr->get_salt());
				ver.set_iteration_count(iteration_count);
				ver.set_kdf_hash(kdf_hash);
			    }
			}
			else
			    throw SRC_BUG;
			    // today only crypto_sym is used, the
			    // dynamic_cast should thus always succeed


			if(multi_threaded_crypto > 1)
			{
#if LIBTHREADAR_AVAILABLE
			    tmp = tmp_tronco = new (nothrow) parallel_tronconneuse(multi_threaded_crypto,
										   crypto_size,
										   *(layers.top()),
										   macro_tools_supported_version,
										   ptr);

			    if(info_details)
				dialog->message(tools_printf(gettext("multi-threaded cyphering layer open, with %d worker thread(s)"), multi_threaded_crypto));
#else
			    throw Ecompilation(gettext("libthreadar is required at compilation time in order to use more than one thread for cryptography"));
#endif
			}
			else
			{
			    tmp = tmp_tronco = new (nothrow) tronconneuse(crypto_size,
									  *(layers.top()),
									  macro_tools_supported_version,
									  ptr);
			    if(info_details)
				dialog->message(tools_printf(gettext("single-threaded cyphering layer open")));
			}
		    }
		    catch(std::bad_alloc &)
		    {
			throw Ememory("macro_tools_create_layers");
		    }

#ifdef LIBDAR_NO_OPTIMIZATION
		    tools_secu_string_show(*dialog, string("real_pass used: "), real_pass);
#endif
		    break;
		case crypto_algo::none:
		    if(!writing_to_pipe)
		    {
			if(info_details)
			    dialog->message(gettext("Adding a new layer on top: Caching layer for better performances..."));
			tmp = new (nothrow) cache(*(layers.top()), false);
			if(tmp != nullptr)
			    level1 = tmp; // new level where to write down the archive header, this is not an encryption layer
		    }
		    else
			tmp = nullptr; // a cache is already present just below
		    break;
		default:
		    throw SRC_BUG; // cryto value should have been checked before
		}

				    // ****** we can now add the crypto layer to the stack ****** //

		if(!writing_to_pipe || crypto != crypto_algo::none)
		{
		    if(tmp == nullptr)
			throw Ememory("op_create_in_sub");
		    else
		    {
			layers.push(tmp);
			tmp = nullptr;
		    }
		}
		else
		{
		    if(tmp != nullptr)
			throw SRC_BUG;
		}


		    // ******** finishing the creation of the archive header and writing it down (in clear!) at the beginning of the archive *** //

		ver.set_edition(macro_tools_supported_version);
		ver.set_compression_algo(algo);
		if(algo == compression::lzo1x_1_15 || algo == compression::lzo1x_1)
		    ver.set_compression_algo(compression::lzo);
		ver.set_command_line(user_comment);
		ver.set_sym_crypto_algo(crypto);
		ver.set_tape_marks(add_marks_for_sequential_reading);
		ver.set_signed(!gnupg_signatories.empty());
		ver.set_compression_block_size(compression_block_size);


		if(ref_slicing != nullptr)
		{
		    slice_layout *tmp = new (nothrow) slice_layout(*ref_slicing);
		    if(tmp == nullptr)
			throw Ememory("macro_tools_create_layers");
		    try
		    {
			ver.set_slice_layout(tmp);
			tmp = nullptr; // now tmp is managed by ver
		    }
		    catch(...)
		    {
			if(tmp != nullptr)
			    delete tmp;
			throw;
		    }
		}
		else
		    ver.clear_slice_layout();

		    // we drop the header at the beginning of the archive in any case (to be able to
		    // know whether sequential reading is possible or not, and if sequential reading
		    // is asked, be able to get the required parameter to read the archive.
		    // It also serves as a backup copy for normal reading if the end of the archive
		    // is corrupted.

		if(info_details)
		    dialog->message(gettext("Writing down the archive header..."));
		if(level1 != nullptr)
		    ver.write(*level1); // we write to the level1 which is just below the encryption layer, thus the header is wrote unciphered
		else
		    throw SRC_BUG;

		    // now we can add the initial offset in the archive_header datastructure, which will be written
		    // a second time, but at the end of the archive. If we start reading the archive from the end
		    // we must know where ended the initial archive header.

		ver.set_initial_offset(level1->get_position());


		    // ********** now that the archive header has been wrote down, we can set the initial_shift for the encryption layer ***** //

		if(tmp_tronco != nullptr)
		    tmp_tronco->set_initial_shift(ver.get_initial_offset());

		    // ***** adding a elastic buffer as first ciphered data after the clear archive header ****** //

		if(crypto != crypto_algo::none)
		{
		    if(info_details)
			dialog->message(gettext("Writing down the initial elastic buffer through the encryption layer..."));
		    macro_tools_add_elastic_buffer(layers, GLOBAL_ELASTIC_BUFFER_SIZE, 0, 0);
		}

		    // ********** if required building the escape layer  ***** //

		if(add_marks_for_sequential_reading)
		{
		    set<escape::sequence_type> unjump;

		    if(info_details)
			dialog->message(gettext("Adding a new layer on top: Escape layer to allow sequential reading..."));
		    unjump.insert(escape::seqt_catalogue);
		    tmp = esc = new (nothrow) escape(layers.top(), unjump);
		    if(tmp == nullptr)
			throw Ememory("op_create_in_sub");
		    else
		    {
			layers.push(tmp);
			tmp = nullptr;
		    }
		}

		    // ********** building the level2 layer (compression) ************************ //

		if(info_details && algo != compression::none)
		    dialog->message(gettext("Adding a new layer on top: compression..."));
		if(compression_block_size == 0)
		{
		    tmp = macro_tools_build_streaming_compressor(algo, *(layers.top()), compression_level, multi_threaded_compress);
		    if(info_details)
			dialog->message(tools_printf(gettext("Adding a streamed compression layer")));
		}
		else
		{
		    tmp = macro_tools_build_block_compressor(algo,
							     *(layers.top()),
							     compression_level,
							     multi_threaded_compress,
							     compression_block_size);
		    if(info_details)
			dialog->message(tools_printf(gettext("Adding block compression layer, with %d worker thread(s)"), multi_threaded_compress));
		}

		if(tmp == nullptr)
		    throw Ememory("op_create_in_sub");
		else
		{
		    layers.push(tmp);
		    tmp = nullptr;
		}

		if(info_details)
		    dialog->message(gettext("All layers have been created successfully"));
	    }
	    catch(...)
	    {
		if(tmp != nullptr)
		{
		    delete tmp;
		    tmp = nullptr;
		}
		throw;
	    }
	}
	catch(Erange &e)
	{
	    string msg = string(gettext("Error creating archive layers: ")) + e.get_message();
	    throw Edata(msg);
	}
    }


    void macro_tools_close_layers(const shared_ptr<user_interaction> & dialog,
				  pile & layers,
				  const header_version & ver,
				  const catalogue & cat,
				  bool info_details,
				  crypto_algo crypto,
				  compression algo,
				  const vector<string> & gnupg_recipients,
				  const vector<string> & gnupg_signatories,
				  bool empty)
    {
	terminateur coord;
	pile_descriptor pdesc(&layers);
	proto_tronco *tronco_ptr = nullptr;
	scrambler *scram_ptr = nullptr;
	memory_file *hash_to_sign = nullptr;
	tlv *signed_hash = nullptr;
	hash_fichier *hasher = nullptr;

	    // ***************** sanity check ************************************** //

	if(!gnupg_signatories.empty() && gnupg_recipients.empty())
	    throw SRC_BUG; // cannot sign without encryption in the current implementation

	pdesc.check(false);

	    // *********** flushing pending writes and reseting compressor  ******** //

	if(pdesc.esc != nullptr)
	{
	    layers.sync_write_above(pdesc.esc); // esc is now up to date
	    pdesc.esc->add_mark_at_current_position(escape::seqt_catalogue);
	}
	else
		// still need to reset the compressor
		// in order to be able to read compressed data starting at that position
	    pdesc.compr->sync_write();


	    // ********* if archive is signed adding a hash layer to get a hash of  *//
	    // *** the catalogue. This hash will then be signed and encrypted  and  *//
	    // ** dropped as is just after the end of the catalogue                 */

	try
	{

	    if(!gnupg_signatories.empty())
	    {
		generic_to_global_file *global_hash_to_sign = nullptr;
		generic_to_global_file *global_layers = nullptr;

		hash_to_sign = new (nothrow) memory_file();
		if(hash_to_sign == nullptr)
		    throw Ememory("macro_tools_close_layers");

		signed_hash = new (nothrow) tlv();
		if(signed_hash == nullptr)
		    throw Ememory("macro_tools_close_layers");

		try
		{
		    global_hash_to_sign = new (nothrow) generic_to_global_file(dialog, hash_to_sign, gf_write_only);
		    global_layers = new (nothrow) generic_to_global_file(dialog, layers.top(), gf_write_only);

		    if(global_hash_to_sign == nullptr || global_layers == nullptr)
			throw Ememory("macro_tools_close_layers");

		    hasher = new (nothrow) hash_fichier(dialog,
							global_layers,
							string("x"),
							global_hash_to_sign,
							hash_algo::sha512);
		    if(hasher == nullptr)
			throw Ememory("macro_tools_close_layers");

			// at this stage, hasher has been built and now owns and
			// will destroy when no more needed both global_hash_to_sign
			// and global_layers

		}
		catch(...)
		{
		    if(global_hash_to_sign != nullptr)
			delete global_hash_to_sign;
		    if(global_layers != nullptr)
			delete global_layers;
		    throw;
		}

		layers.push(hasher);
		pdesc = pile_descriptor(&layers);
	    }


		// *********** writing down the catalogue of the archive *************** //


	    coord.set_catalogue_start(layers.get_position());

	    if(info_details)
		dialog->message(gettext("Writing down archive contents..."));
	    cat.reset_dump();
	    cat.dump(pdesc);


		// removing the hash from the top of the stack now that the catalogue has been dropped //

	    if(hasher != nullptr)
	    {
		hasher->terminate(); // this drops the hash to hash_to_sign
		if(layers.top() != hasher)
		    throw SRC_BUG;
		if(layers.pop() != hasher)
		    throw SRC_BUG;
		pdesc = pile_descriptor(&layers);
	    }

		// if signing is activated ... //

	    if(!gnupg_signatories.empty())
	    {
		crypto_asym engine(dialog);

		if(info_details)
		    dialog->message(gettext("Calculating the signature of the catalogue hash..."));

		    // ciphering and signing the hash of the catalogue //

		if(hash_to_sign == nullptr)
		    throw SRC_BUG;
		else
		    hash_to_sign->skip(0);
		if(signed_hash == nullptr)
		    throw SRC_BUG;
		engine.set_signatories(gnupg_signatories);
		engine.encrypt(gnupg_recipients, *hash_to_sign, *signed_hash);

		    // writing down the size of the hash followed by the hash //

		if(info_details)
		    dialog->message(gettext("Writing down the signed hash of the catalogue..."));

		signed_hash->dump(layers);
	    }
	}
	catch(...)
	{
	    if(hasher != nullptr)
	    {
		if(layers.top() == hasher)
		{
		    if(layers.pop() != hasher)
			throw SRC_BUG;
		    pdesc = pile_descriptor(&layers);
		}
		delete hasher;
		hasher = nullptr;
	    }
	    if(signed_hash != nullptr)
	    {
		delete signed_hash;
		signed_hash = nullptr;
	    }
	    if(hash_to_sign != nullptr)
	    {
		delete hash_to_sign;
		hash_to_sign = nullptr;
	    }
	    throw;
	}

	if(hasher != nullptr)
	{
	    delete hasher;
	    hasher = nullptr;
	}
	if(signed_hash != nullptr)
	{
	    delete signed_hash;
	    signed_hash = nullptr;
	}
	if(hash_to_sign != nullptr)
	{
	    delete hash_to_sign;
	    hash_to_sign = nullptr;
	}

	    // releasing the compression layer

	if(info_details && algo != compression::none)
	    dialog->message(gettext("Closing the compression layer..."));

	if(!layers.pop_and_close_if_type_is(pdesc.compr))
	    throw SRC_BUG;
	else
	    pdesc.compr = nullptr;

	    // releasing the escape layer

	if(pdesc.esc != nullptr)
	{
	    if(info_details)
		dialog->message(gettext("Closing the escape layer..."));
	    if(!layers.pop_and_close_if_type_is(pdesc.esc))
		throw SRC_BUG;
	    pdesc.esc = nullptr;
	}

	    // *********** writing down the first terminator at the end of the archive  *************** //

	tronco_ptr = dynamic_cast<proto_tronco *>(layers.top());
	scram_ptr = dynamic_cast<scrambler *>(layers.top());

	if(info_details)
	    dialog->message(gettext("Writing down the first archive terminator..."));
	coord.dump(layers); // since format "04" the terminateur is encrypted

	if(crypto != crypto_algo::none)
	{
	    if(info_details)
		dialog->message(gettext("writing down the final elastic buffer through the encryption layer..."));

		// obtaining the crypto block size (for clear data)

	    U_32 block_size = 0;

	    if(tronco_ptr != nullptr)
		block_size = tronco_ptr->get_clear_block_size();
	    if(tronco_ptr == nullptr && scram_ptr == nullptr)
		throw SRC_BUG;

		// calculating the current offset modulo block_size

	    U_32 offset = 0;

	    if(block_size > 0)
	    {
		infinint times = 0;
		infinint reste = 0;

		euclide(layers.get_position(), infinint(block_size), times, reste);
		reste.unstack(offset);
		if(!reste.is_zero())
		    throw SRC_BUG;
	    }

	    macro_tools_add_elastic_buffer(layers,
					   GLOBAL_ELASTIC_BUFFER_SIZE,
					   block_size,
					   offset);
		// terminal elastic buffer (after terminateur to protect against
		// plain text attack on the terminator string)
	}

	    // releasing memory by calling destructors and releases file descriptors

	if(tronco_ptr != nullptr || scram_ptr != nullptr)
	{
	    if(info_details)
		dialog->message(gettext("Closing the encryption layer..."));
	}

	if(tronco_ptr != nullptr)
	    tronco_ptr->write_end_of_file();

	if(tronco_ptr != nullptr)
	{
	    if(!layers.pop_and_close_if_type_is(tronco_ptr))
		throw SRC_BUG;
	}

	if(scram_ptr != nullptr)
	{
	    if(!layers.pop_and_close_if_type_is(scram_ptr))
		throw SRC_BUG;
	}


	    // *********** writing down the trailier_version with the second terminateur *************** //

	if(info_details)
	    dialog->message(gettext("Writing down archive trailer..."));
	coord.set_catalogue_start(layers.get_position());
	ver.write(layers);

	if(info_details)
	    dialog->message(gettext("Writing down the second archive terminator..."));
	    // due to the possibility a file get shorter after being resaved because it was modified at the
	    // time it was read for backup, we must be sure the last terminator is well positionned at end of file
	layers.skip_to_eof();

	coord.dump(layers);
	layers.sync_write();

	    // *********** closing the archive ******************** //

	if(info_details)
	    dialog->message(gettext("Closing archive low layer..."));

	    // closing all generic_files remaining in the layers
	try
	{
	    layers.terminate();
		// this way we can propagate exceptions
		// and clear() the layers at the same time
	}
	catch(...)
	{
	    layers.clear();
	    throw;
	}
	layers.clear();

	if(info_details)
	    dialog->message(gettext("Archive is closed."));
    }

    range macro_tools_get_slices(const cat_nomme *obj, slice_layout sl)
    {
	range slices;
	infinint offset; // used temporarily to record data, EA and FSA offsets
	infinint slice_num, slice_offset;
	infinint low;
	const cat_inode *tmp_inode = dynamic_cast<const cat_inode *>(obj);
	const cat_file *tmp_file = dynamic_cast<const cat_file *>(obj);
	const cat_mirage *tmp_mir = dynamic_cast<const cat_mirage *>(obj);

	if(obj == nullptr)
	    throw SRC_BUG;

	if(tmp_mir != nullptr)
	{
	    tmp_inode = tmp_mir->get_inode();
	    tmp_file = dynamic_cast<const cat_file *>(tmp_inode);
	}

	if(tmp_inode != nullptr)
	{
	    if(!sl.first_size.is_zero())
	    {
		if(tmp_inode->ea_get_saved_status() == ea_saved_status::full)
		{
		    if(tmp_inode->ea_get_offset(offset))
		    {
			sl.which_slice(offset,
				       slice_num,
				       slice_offset);
			low = slice_num;

			offset += tmp_inode->ea_get_size();
			sl.which_slice(offset,
				       slice_num,
				       slice_offset);
		 	slices += range(low, slice_num);
		    }
		    else
			throw SRC_BUG; // has EA saved but no offset ?!?
		}

		if(tmp_inode->fsa_get_saved_status() == fsa_saved_status::full)
		{
		    if(tmp_inode->fsa_get_offset(offset))
		    {
			sl.which_slice(offset,
				       slice_num,
				       slice_offset);
			low = slice_num;

			offset += tmp_inode->fsa_get_size();
			sl.which_slice(offset,
				       slice_num,
				       slice_offset);
			slices += range(low, slice_num);
		    }
		    else
			throw SRC_BUG;
		}
	    }
	}

	if(tmp_file != nullptr)
	{
	    if(tmp_file->get_saved_status() == saved_status::saved)
	    {
		offset = tmp_file->get_offset();
		sl.which_slice(offset,
			       slice_num,
			       slice_offset);
		low = slice_num;

		offset += tmp_file->get_storage_size();
		sl.which_slice(offset,
			       slice_num,
			       slice_offset);
		slices += range(low, slice_num);
	    }
	}

	return slices;
    }

    trivial_sar *macro_tools_open_archive_tuyau(const shared_ptr<user_interaction> & dialog,
						S_I fd,
						gf_mode mode,
						const label & internal_name,
						const label & data_name,
						bool slice_header_format_07,
						const std::string & execute)
    {
        generic_file *tmp = nullptr;
        trivial_sar *ret = nullptr;

        try
        {
            tmp = new (nothrow) tuyau(dialog, fd, mode);
            if(tmp == nullptr)
                throw Ememory("macro_tools_open_archive_tuyau");
            ret = new (nothrow) trivial_sar(dialog,
					    tmp,
					    internal_name,
					    data_name,
					    slice_header_format_07,
					    execute);
            if(ret == nullptr)
                throw Ememory("macro_tools_open_archive_tuyau");
	    else
		tmp = nullptr;
        }
        catch(...)
        {
            if(ret != nullptr)
                delete ret;
	    if(tmp != nullptr)
		delete tmp;
            throw;
        }

        return ret;
    }

    void macro_tools_open_pipes(const shared_ptr<user_interaction> & dialog,
				const string &input,
				const string & output,
				tuyau *&in,
				tuyau *&out)
    {
        in = out = nullptr;
        try
        {
            if(input != "")
                in = new (nothrow) tuyau(dialog, input, gf_read_only);
            else
                in = new (nothrow) tuyau(dialog, 0, gf_read_only); // stdin by default
            if(in == nullptr)
                throw Ememory("tools_open_pipes");

            if(output != "")
                out = new (nothrow) tuyau(dialog, output, gf_write_only);
            else
                out = new (nothrow) tuyau(dialog, 1, gf_write_only); // stdout by default
            if(out == nullptr)
                throw Ememory("tools_open_pipes");

        }
        catch(...)
        {
            if(in != nullptr)
                delete in;
            if(out != nullptr)
                delete out;
            throw;
        }
    }

    proto_compressor* macro_tools_build_streaming_compressor(compression algo,
							     generic_file & base,
							     U_I compression_level,
							     U_I num_workers)
    {
	proto_compressor* ret;

	switch(algo)
	{
	case compression::none:
	case compression::gzip:
	case compression::bzip2:
	case compression::xz:
	    ret = new (nothrow) compressor(algo, base, compression_level);
	    break;
	case compression::lzo:
	case compression::lzo1x_1_15:
	case compression::lzo1x_1:
	case compression::lz4:
	    if(num_workers > 1)
	    {
#if LIBTHREADAR_AVAILABLE
		ret = new (nothrow) parallel_block_compressor(num_workers,
							      make_compress_module_ptr(algo, compression_level),
							      base,
							      PRE_2_7_0_LZO_BLOCK_SIZE);
#else
		throw Ecompilation(gettext("libthreadar is required at compilation time in order to use more than one thread for block compression"));
#endif
	    }
	    else
		ret = new (nothrow) block_compressor(make_compress_module_ptr(algo, compression_level),
						     base,
						     PRE_2_7_0_LZO_BLOCK_SIZE);
	    break;
	case compression::zstd:
	    ret = new (nothrow) compressor_zstd(base, compression_level);
	    break;
	default:
	    throw SRC_BUG;
	}

	if(ret == nullptr)
	    throw Ememory("macro_tools_build_streaming_compressor");

	return ret;
    }


    proto_compressor* macro_tools_build_block_compressor(compression algo,
							 generic_file & base,
							 U_I compression_level,
							 U_I num_workers,
							 U_I block_size)
    {
	proto_compressor* ret = nullptr;

	if(num_workers > 1)
	{
#if LIBTHREADAR_AVAILABLE
	    ret = new (nothrow) parallel_block_compressor(num_workers,
							  make_compress_module_ptr(algo, compression_level),
							  base,
							  block_size);
#else
	    throw Ecompilation(gettext("libthreadar is required at compilation time in order to use more than one thread for block compression"));
#endif
	}
	else
	{
	    ret = new (nothrow) block_compressor(make_compress_module_ptr(algo, compression_level), // compression level is not used here
						 base,
						 block_size);
	}

	if(ret == nullptr)
	    throw Ememory("macro_tools_build_block_compressor");

	return ret;
    }


	//////////// STATIC FUNCTIONS //////////////////////////////////////////


    static void version_check(user_interaction & dialog, const header_version & ver)
    {
        if(ver.get_edition() > macro_tools_supported_version)
            dialog.pause(gettext("The format version of the archive is too high for that software version, try reading anyway?"));
    }

    static void macro_tools_add_elastic_buffer(generic_file & f,
					       U_32 max_size,
					       U_32 modulo,
					       U_32 offset)
    {
	U_32 size = tools_pseudo_random(max_size-1) + 1; // range from 1 to max_size;

	if(modulo > 0)
	{
	    U_32 shift = modulo - (offset % modulo);
	    size = (size/modulo)*modulo + shift;
	}

        elastic tic = size;
        char *buffer = new (nothrow) char[tic.get_size()];

        if(buffer == nullptr)
            throw Ememory("tools_add_elastic_buffer");
        try
        {
            tic.dump((unsigned char *)buffer, tic.get_size());
            f.write(buffer, tic.get_size());
        }
        catch(...)
        {
            delete [] buffer;
            throw;
        }
        delete [] buffer;
    }

    static unique_ptr<compress_module> make_compress_module_ptr(compression algo, U_I compression_level)
    {
	unique_ptr<compress_module> ret;

	try
	{
	    switch(algo)
	    {
	    case compression::none:
		throw SRC_BUG;
	    case compression::gzip:
		ret = make_unique<gzip_module>(compression_level);
		break;
	    case compression::bzip2:
		ret = make_unique<bzip2_module>(compression_level);
		break;
	    case compression::lzo:
	    case compression::lzo1x_1_15:
	    case compression::lzo1x_1:
		ret = make_unique<lzo_module>(algo, compression_level);
		break;
	    case compression::xz:
		ret = make_unique<xz_module>(compression_level);
		break;
	    case compression::zstd:
		ret = make_unique<zstd_module>(compression_level);
		break;
	    case compression::lz4:
		ret = make_unique<lz4_module>(compression_level);
		break;
	    default:
		throw SRC_BUG;
	    }
	}
	catch(bad_alloc &)
	{
	    throw Ememory("make_compress_module_ptr");
	}

	if(!ret)
	    throw SRC_BUG; // we would return a null pointer!

	return ret;
    }



} // end of namespace
