Home | History | Annotate | Line # | Download | only in dictBuilder 
      1 /*
      2  * Copyright (c) Meta Platforms, Inc. and affiliates.
      3  * All rights reserved.
      4  *
      5  * This source code is licensed under both the BSD-style license (found in the
      6  * LICENSE file in the root directory of this source tree) and the GPLv2 (found
      7  * in the COPYING file in the root directory of this source tree).
      8  * You may select, at your option, one of the above-listed licenses.
      9  */
     10 
     11 #ifndef ZDICT_STATIC_LINKING_ONLY
     12 #  define ZDICT_STATIC_LINKING_ONLY
     13 #endif
     14 
     15 #include "../common/threading.h" /* ZSTD_pthread_mutex_t */
     16 #include "../common/mem.h"   /* U32, BYTE */
     17 #include "../zdict.h"
     18 
     19 /**
     20  * COVER_best_t is used for two purposes:
     21  * 1. Synchronizing threads.
     22  * 2. Saving the best parameters and dictionary.
     23  *
     24  * All of the methods except COVER_best_init() are thread safe if zstd is
     25  * compiled with multithreaded support.
     26  */
     27 typedef struct COVER_best_s {
     28   ZSTD_pthread_mutex_t mutex;
     29   ZSTD_pthread_cond_t cond;
     30   size_t liveJobs;
     31   void *dict;
     32   size_t dictSize;
     33   ZDICT_cover_params_t parameters;
     34   size_t compressedSize;
     35 } COVER_best_t;
     36 
     37 /**
     38  * A segment is a range in the source as well as the score of the segment.
     39  */
     40 typedef struct {
     41   U32 begin;
     42   U32 end;
     43   U32 score;
     44 } COVER_segment_t;
     45 
     46 /**
     47  *Number of epochs and size of each epoch.
     48  */
     49 typedef struct {
     50   U32 num;
     51   U32 size;
     52 } COVER_epoch_info_t;
     53 
     54 /**
     55  * Struct used for the dictionary selection function.
     56  */
     57 typedef struct COVER_dictSelection {
     58   BYTE* dictContent;
     59   size_t dictSize;
     60   size_t totalCompressedSize;
     61 } COVER_dictSelection_t;
     62 
     63 /**
     64  * Computes the number of epochs and the size of each epoch.
     65  * We will make sure that each epoch gets at least 10 * k bytes.
     66  *
     67  * The COVER algorithms divide the data up into epochs of equal size and
     68  * select one segment from each epoch.
     69  *
     70  * @param maxDictSize The maximum allowed dictionary size.
     71  * @param nbDmers     The number of dmers we are training on.
     72  * @param k           The parameter k (segment size).
     73  * @param passes      The target number of passes over the dmer corpus.
     74  *                    More passes means a better dictionary.
     75  */
     76 COVER_epoch_info_t COVER_computeEpochs(U32 maxDictSize, U32 nbDmers,
     77                                        U32 k, U32 passes);
     78 
     79 /**
     80  * Warns the user when their corpus is too small.
     81  */
     82 void COVER_warnOnSmallCorpus(size_t maxDictSize, size_t nbDmers, int displayLevel);
     83 
     84 /**
     85  *  Checks total compressed size of a dictionary
     86  */
     87 size_t COVER_checkTotalCompressedSize(const ZDICT_cover_params_t parameters,
     88                                       const size_t *samplesSizes, const BYTE *samples,
     89                                       size_t *offsets,
     90                                       size_t nbTrainSamples, size_t nbSamples,
     91                                       BYTE *const dict, size_t dictBufferCapacity);
     92 
     93 /**
     94  * Returns the sum of the sample sizes.
     95  */
     96 size_t COVER_sum(const size_t *samplesSizes, unsigned nbSamples) ;
     97 
     98 /**
     99  * Initialize the `COVER_best_t`.
    100  */
    101 void COVER_best_init(COVER_best_t *best);
    102 
    103 /**
    104  * Wait until liveJobs == 0.
    105  */
    106 void COVER_best_wait(COVER_best_t *best);
    107 
    108 /**
    109  * Call COVER_best_wait() and then destroy the COVER_best_t.
    110  */
    111 void COVER_best_destroy(COVER_best_t *best);
    112 
    113 /**
    114  * Called when a thread is about to be launched.
    115  * Increments liveJobs.
    116  */
    117 void COVER_best_start(COVER_best_t *best);
    118 
    119 /**
    120  * Called when a thread finishes executing, both on error or success.
    121  * Decrements liveJobs and signals any waiting threads if liveJobs == 0.
    122  * If this dictionary is the best so far save it and its parameters.
    123  */
    124 void COVER_best_finish(COVER_best_t *best, ZDICT_cover_params_t parameters,
    125                        COVER_dictSelection_t selection);
    126 /**
    127  * Error function for COVER_selectDict function. Checks if the return
    128  * value is an error.
    129  */
    130 unsigned COVER_dictSelectionIsError(COVER_dictSelection_t selection);
    131 
    132  /**
    133   * Error function for COVER_selectDict function. Returns a struct where
    134   * return.totalCompressedSize is a ZSTD error.
    135   */
    136 COVER_dictSelection_t COVER_dictSelectionError(size_t error);
    137 
    138 /**
    139  * Always call after selectDict is called to free up used memory from
    140  * newly created dictionary.
    141  */
    142 void COVER_dictSelectionFree(COVER_dictSelection_t selection);
    143 
    144 /**
    145  * Called to finalize the dictionary and select one based on whether or not
    146  * the shrink-dict flag was enabled. If enabled the dictionary used is the
    147  * smallest dictionary within a specified regression of the compressed size
    148  * from the largest dictionary.
    149  */
    150  COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent, size_t dictBufferCapacity,
    151                        size_t dictContentSize, const BYTE* samplesBuffer, const size_t* samplesSizes, unsigned nbFinalizeSamples,
    152                        size_t nbCheckSamples, size_t nbSamples, ZDICT_cover_params_t params, size_t* offsets, size_t totalCompressedSize);
    153