1 /*- 2 * Copyright (c) 2014 Sebastian Freundt 3 * Author: Sebastian Freundt <devel (at) fresse.org> 4 * 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 #include "archive_platform.h" 29 30 #ifdef HAVE_ERRNO_H 31 #include <errno.h> 32 #endif 33 #include <stdio.h> 34 #ifdef HAVE_STDLIB_H 35 #include <stdlib.h> 36 #endif 37 #ifdef HAVE_STRING_H 38 #include <string.h> 39 #endif 40 #ifdef HAVE_TIME_H 41 #include <time.h> 42 #endif 43 44 #include "archive.h" 45 #include "archive_entry.h" 46 #include "archive_entry_locale.h" 47 #include "archive_private.h" 48 #include "archive_random_private.h" 49 #include "archive_write_private.h" 50 #include "archive_write_set_format_private.h" 51 52 struct warc_s { 53 unsigned int omit_warcinfo:1; 54 55 time_t now; 56 mode_t typ; 57 unsigned int rng; 58 /* populated size */ 59 uint64_t populz; 60 }; 61 62 static const char warcinfo[] = 63 "software: libarchive/" ARCHIVE_VERSION_ONLY_STRING "\r\n" 64 "format: WARC file version 1.0\r\n"; 65 66 typedef enum { 67 WT_NONE, 68 /* warcinfo */ 69 WT_INFO, 70 /* metadata */ 71 WT_META, 72 /* resource */ 73 WT_RSRC, 74 /* request, unsupported */ 75 WT_REQ, 76 /* response, unsupported */ 77 WT_RSP, 78 /* revisit, unsupported */ 79 WT_RVIS, 80 /* conversion, unsupported */ 81 WT_CONV, 82 /* continuation, unsupported at the moment */ 83 WT_CONT, 84 /* invalid type */ 85 LAST_WT 86 } warc_type_t; 87 88 typedef struct { 89 warc_type_t type; 90 const char *tgturi; 91 const char *recid; 92 time_t rtime; 93 time_t mtime; 94 const char *cnttyp; 95 uint64_t cntlen; 96 } warc_essential_hdr_t; 97 98 typedef struct { 99 unsigned int u[4U]; 100 } warc_uuid_t; 101 102 static int _warc_options(struct archive_write*, const char *key, const char *v); 103 static int _warc_header(struct archive_write *a, struct archive_entry *entry); 104 static ssize_t _warc_data(struct archive_write *a, const void *buf, size_t sz); 105 static int _warc_finish_entry(struct archive_write *a); 106 static int _warc_close(struct archive_write *a); 107 static int _warc_free(struct archive_write *a); 108 109 /* private routines */ 110 static ssize_t _popul_ehdr(struct archive_string *t, size_t z, warc_essential_hdr_t); 111 static int _gen_uuid(warc_uuid_t *tgt); 112 113 114 /* 116 * Set output format to ISO 28500 (aka WARC) format. 117 */ 118 int 119 archive_write_set_format_warc(struct archive *_a) 120 { 121 struct archive_write *a = (struct archive_write *)_a; 122 struct warc_s *w; 123 124 archive_check_magic(_a, ARCHIVE_WRITE_MAGIC, 125 ARCHIVE_STATE_NEW, "archive_write_set_format_warc"); 126 127 /* If another format was already registered, unregister it. */ 128 if (a->format_free != NULL) { 129 (a->format_free)(a); 130 } 131 132 w = malloc(sizeof(*w)); 133 if (w == NULL) { 134 archive_set_error(&a->archive, ENOMEM, 135 "Can't allocate warc data"); 136 return (ARCHIVE_FATAL); 137 } 138 /* by default we're emitting a file wide header */ 139 w->omit_warcinfo = 0U; 140 /* obtain current time for date fields */ 141 w->now = time(NULL); 142 /* reset file type info */ 143 w->typ = 0; 144 /* also initialise our rng */ 145 w->rng = (unsigned int)w->now; 146 147 a->format_data = w; 148 a->format_name = "WARC/1.0"; 149 a->format_options = _warc_options; 150 a->format_write_header = _warc_header; 151 a->format_write_data = _warc_data; 152 a->format_close = _warc_close; 153 a->format_free = _warc_free; 154 a->format_finish_entry = _warc_finish_entry; 155 a->archive.archive_format = ARCHIVE_FORMAT_WARC; 156 a->archive.archive_format_name = "WARC/1.0"; 157 return (ARCHIVE_OK); 158 } 159 160 161 /* archive methods */ 163 static int 164 _warc_options(struct archive_write *a, const char *key, const char *val) 165 { 166 struct warc_s *w = a->format_data; 167 168 if (strcmp(key, "omit-warcinfo") == 0) { 169 if (val == NULL || strcmp(val, "true") == 0) { 170 /* great */ 171 w->omit_warcinfo = 1U; 172 return (ARCHIVE_OK); 173 } 174 } 175 176 /* Note: The "warn" return is just to inform the options 177 * supervisor that we didn't handle it. It will generate 178 * a suitable error if no one used this option. */ 179 return (ARCHIVE_WARN); 180 } 181 182 static int 183 _warc_header(struct archive_write *a, struct archive_entry *entry) 184 { 185 struct warc_s *w = a->format_data; 186 struct archive_string hdr; 187 #define MAX_HDR_SIZE 512 188 189 /* check whether warcinfo record needs outputting */ 190 if (!w->omit_warcinfo) { 191 ssize_t r; 192 warc_essential_hdr_t wi = { 193 WT_INFO, 194 /*uri*/NULL, 195 /*urn*/NULL, 196 /*rtm*/0, 197 /*mtm*/0, 198 /*cty*/"application/warc-fields", 199 /*len*/sizeof(warcinfo) - 1U, 200 }; 201 wi.rtime = w->now; 202 wi.mtime = w->now; 203 204 archive_string_init(&hdr); 205 r = _popul_ehdr(&hdr, MAX_HDR_SIZE, wi); 206 if (r >= 0) { 207 /* jackpot! */ 208 /* now also use HDR buffer for the actual warcinfo */ 209 archive_strncat(&hdr, warcinfo, sizeof(warcinfo) -1); 210 211 /* append end-of-record indicator */ 212 archive_strncat(&hdr, "\r\n\r\n", 4); 213 214 /* write to output stream */ 215 __archive_write_output(a, hdr.s, archive_strlen(&hdr)); 216 } 217 /* indicate we're done with file header writing */ 218 w->omit_warcinfo = 1U; 219 archive_string_free(&hdr); 220 } 221 222 if (archive_entry_pathname(entry) == NULL) { 223 archive_set_error(&a->archive, EINVAL, 224 "Invalid filename"); 225 return (ARCHIVE_WARN); 226 } 227 228 w->typ = archive_entry_filetype(entry); 229 w->populz = 0U; 230 if (w->typ == AE_IFREG) { 231 warc_essential_hdr_t rh = { 232 WT_RSRC, 233 /*uri*/NULL, 234 /*urn*/NULL, 235 /*rtm*/0, 236 /*mtm*/0, 237 /*cty*/NULL, 238 /*len*/0, 239 }; 240 ssize_t r; 241 rh.tgturi = archive_entry_pathname(entry); 242 rh.rtime = w->now; 243 rh.mtime = archive_entry_mtime(entry); 244 rh.cntlen = (size_t)archive_entry_size(entry); 245 246 archive_string_init(&hdr); 247 r = _popul_ehdr(&hdr, MAX_HDR_SIZE, rh); 248 if (r < 0) { 249 /* don't bother */ 250 archive_set_error( 251 &a->archive, 252 ARCHIVE_ERRNO_FILE_FORMAT, 253 "cannot archive file"); 254 return (ARCHIVE_WARN); 255 } 256 /* otherwise append to output stream */ 257 __archive_write_output(a, hdr.s, r); 258 /* and let subsequent calls to _data() know about the size */ 259 w->populz = rh.cntlen; 260 archive_string_free(&hdr); 261 return (ARCHIVE_OK); 262 } 263 /* just resort to erroring as per Tim's advice */ 264 __archive_write_entry_filetype_unsupported( 265 &a->archive, entry, "WARC"); 266 return (ARCHIVE_FAILED); 267 } 268 269 static ssize_t 270 _warc_data(struct archive_write *a, const void *buf, size_t len) 271 { 272 struct warc_s *w = a->format_data; 273 274 if (w->typ == AE_IFREG) { 275 int rc; 276 277 /* never write more bytes than announced */ 278 if (len > w->populz) { 279 len = (size_t)w->populz; 280 } 281 282 /* now then, out we put the whole shebang */ 283 rc = __archive_write_output(a, buf, len); 284 if (rc != ARCHIVE_OK) { 285 return rc; 286 } 287 } 288 return len; 289 } 290 291 static int 292 _warc_finish_entry(struct archive_write *a) 293 { 294 static const char _eor[] = "\r\n\r\n"; 295 struct warc_s *w = a->format_data; 296 297 if (w->typ == AE_IFREG) { 298 int rc = __archive_write_output(a, _eor, sizeof(_eor) - 1U); 299 300 if (rc != ARCHIVE_OK) { 301 return rc; 302 } 303 } 304 /* reset type info */ 305 w->typ = 0; 306 return (ARCHIVE_OK); 307 } 308 309 static int 310 _warc_close(struct archive_write *a) 311 { 312 (void)a; /* UNUSED */ 313 return (ARCHIVE_OK); 314 } 315 316 static int 317 _warc_free(struct archive_write *a) 318 { 319 struct warc_s *w = a->format_data; 320 321 free(w); 322 a->format_data = NULL; 323 return (ARCHIVE_OK); 324 } 325 326 327 /* private routines */ 329 static void 330 xstrftime(struct archive_string *as, const char *fmt, time_t t) 331 { 332 /** like strftime(3) but for time_t objects */ 333 struct tm *rt; 334 #if defined(HAVE_GMTIME_R) || defined(HAVE_GMTIME_S) 335 struct tm timeHere; 336 #endif 337 char strtime[100]; 338 size_t len; 339 340 #if defined(HAVE_GMTIME_S) 341 rt = gmtime_s(&timeHere, &t) ? NULL : &timeHere; 342 #elif defined(HAVE_GMTIME_R) 343 rt = gmtime_r(&t, &timeHere); 344 #else 345 rt = gmtime(&t); 346 #endif 347 if (!rt) 348 return; 349 /* leave the hard yacker to our role model strftime() */ 350 len = strftime(strtime, sizeof(strtime)-1, fmt, rt); 351 archive_strncat(as, strtime, len); 352 } 353 354 static ssize_t 355 _popul_ehdr(struct archive_string *tgt, size_t tsz, warc_essential_hdr_t hdr) 356 { 357 static const char _ver[] = "WARC/1.0\r\n"; 358 static const char * const _typ[LAST_WT] = { 359 NULL, "warcinfo", "metadata", "resource", NULL 360 }; 361 char std_uuid[48U]; 362 363 if (hdr.type == WT_NONE || hdr.type > WT_RSRC) { 364 /* brilliant, how exactly did we get here? */ 365 return -1; 366 } 367 368 archive_strcpy(tgt, _ver); 369 370 archive_string_sprintf(tgt, "WARC-Type: %s\r\n", _typ[hdr.type]); 371 372 if (hdr.tgturi != NULL) { 373 /* check if there's a xyz:// */ 374 static const char _uri[] = ""; 375 static const char _fil[] = "file://"; 376 const char *u; 377 char *chk = strchr(hdr.tgturi, ':'); 378 379 if (chk != NULL && chk[1U] == '/' && chk[2U] == '/') { 380 /* yep, it's definitely a URI */ 381 u = _uri; 382 } else { 383 /* hm, best to prepend file:// then */ 384 u = _fil; 385 } 386 archive_string_sprintf(tgt, 387 "WARC-Target-URI: %s%s\r\n", u, hdr.tgturi); 388 } 389 390 /* record time is usually when the http is sent off, 391 * just treat the archive writing as such for a moment */ 392 xstrftime(tgt, "WARC-Date: %Y-%m-%dT%H:%M:%SZ\r\n", hdr.rtime); 393 394 /* while we're at it, record the mtime */ 395 xstrftime(tgt, "Last-Modified: %Y-%m-%dT%H:%M:%SZ\r\n", hdr.mtime); 396 397 if (hdr.recid == NULL) { 398 /* generate one, grrrr */ 399 warc_uuid_t u; 400 401 _gen_uuid(&u); 402 /* Unfortunately, archive_string_sprintf does not 403 * handle the minimum number following '%'. 404 * So we have to use snprintf function here instead 405 * of archive_string_snprintf function. */ 406 #if defined(_WIN32) && !defined(__CYGWIN__) && !( defined(_MSC_VER) && _MSC_VER >= 1900) 407 #define snprintf _snprintf 408 #endif 409 snprintf( 410 std_uuid, sizeof(std_uuid), 411 "<urn:uuid:%08x-%04x-%04x-%04x-%04x%08x>", 412 u.u[0U], 413 u.u[1U] >> 16U, u.u[1U] & 0xffffU, 414 u.u[2U] >> 16U, u.u[2U] & 0xffffU, 415 u.u[3U]); 416 hdr.recid = std_uuid; 417 } 418 419 /* record-id is mandatory, fingers crossed we won't fail */ 420 archive_string_sprintf(tgt, "WARC-Record-ID: %s\r\n", hdr.recid); 421 422 if (hdr.cnttyp != NULL) { 423 archive_string_sprintf(tgt, "Content-Type: %s\r\n", hdr.cnttyp); 424 } 425 426 /* next one is mandatory */ 427 archive_string_sprintf(tgt, "Content-Length: %ju\r\n", (uintmax_t)hdr.cntlen); 428 /**/ 429 archive_strncat(tgt, "\r\n", 2); 430 431 return (archive_strlen(tgt) >= tsz)? -1: (ssize_t)archive_strlen(tgt); 432 } 433 434 static int 435 _gen_uuid(warc_uuid_t *tgt) 436 { 437 archive_random(tgt->u, sizeof(tgt->u)); 438 /* obey uuid version 4 rules */ 439 tgt->u[1U] &= 0xffff0fffU; 440 tgt->u[1U] |= 0x4000U; 441 tgt->u[2U] &= 0x3fffffffU; 442 tgt->u[2U] |= 0x80000000U; 443 return 0; 444 } 445 446 /* archive_write_set_format_warc.c ends here */ 447
Indexes created Mon Oct 05 00:25:34 UTC 2026