Line Flow Count Block(s) Source
1 - /*
2 - * Copyright (C) the libgit2 contributors. All rights reserved.
3 - *
4 - * This file is part of libgit2, distributed under the GNU GPL v2 with
5 - * a Linking Exception. For full terms see the included COPYING file.
6 - */
7 -
8 - #include "pack-objects.h"
9 -
10 - #include "zstream.h"
11 - #include "delta.h"
12 - #include "iterator.h"
13 - #include "netops.h"
14 - #include "pack.h"
15 - #include "thread-utils.h"
16 - #include "tree.h"
17 - #include "util.h"
18 - #include "revwalk.h"
19 - #include "commit_list.h"
20 -
21 - #include "git2/pack.h"
22 - #include "git2/commit.h"
23 - #include "git2/tag.h"
24 - #include "git2/indexer.h"
25 - #include "git2/config.h"
26 -
27 - struct unpacked {
28 - git_pobject *object;
29 - void *data;
30 - struct git_delta_index *index;
31 - size_t depth;
32 - };
33 -
34 - struct tree_walk_context {
35 - git_packbuilder *pb;
36 - git_buf buf;
37 - };
38 -
39 - struct pack_write_context {
40 - git_indexer *indexer;
41 - git_indexer_progress *stats;
42 - };
43 -
44 - struct walk_object {
45 - git_oid id;
46 - unsigned int uninteresting:1,
47 - seen:1;
48 - };
49 -
50 - #ifdef GIT_THREADS
51 -
52 - #define GIT_PACKBUILDER__MUTEX_OP(pb, mtx, op) do { \
53 - int result = git_mutex_##op(&(pb)->mtx); \
54 - assert(!result); \
55 - GIT_UNUSED(result); \
56 - } while (0)
57 -
58 - #else
59 -
60 - #define GIT_PACKBUILDER__MUTEX_OP(pb,mtx,op) GIT_UNUSED(pb)
61 -
62 - #endif /* GIT_THREADS */
63 -
64 - #define git_packbuilder__cache_lock(pb) GIT_PACKBUILDER__MUTEX_OP(pb, cache_mutex, lock)
65 - #define git_packbuilder__cache_unlock(pb) GIT_PACKBUILDER__MUTEX_OP(pb, cache_mutex, unlock)
66 - #define git_packbuilder__progress_lock(pb) GIT_PACKBUILDER__MUTEX_OP(pb, progress_mutex, lock)
67 - #define git_packbuilder__progress_unlock(pb) GIT_PACKBUILDER__MUTEX_OP(pb, progress_mutex, unlock)
68 -
69 - /* The minimal interval between progress updates (in seconds). */
70 - #define MIN_PROGRESS_UPDATE_INTERVAL 0.5
71 -
72 - /* Size of the buffer to feed to zlib */
73 - #define COMPRESS_BUFLEN (1024 * 1024)
74 -
75 2317 2 static unsigned name_hash(const char *name)
76 - {
77 2317 2 unsigned c, hash = 0;
78 -
79 2317 2 if (!name)
80 1532 3 return 0;
81 -
82 - /*
83 - * This effectively just creates a sortable number from the
84 - * last sixteen non-whitespace characters. Last characters
85 - * count "most", so things that end in ".c" sort together.
86 - */
87 10858 4,9 while ((c = *name++) != 0) {
88 10073 5,6 if (git__isspace(c))
89 ##### 7 continue;
90 10073 8 hash = (hash >> 2) + (c << 24);
91 - }
92 785 10 return hash;
93 - }
94 -
95 56 2 static int packbuilder_config(git_packbuilder *pb)
96 - {
97 - git_config *config;
98 56 2 int ret = 0;
99 - int64_t val;
100 -
101 56 2,3 if ((ret = git_repository_config_snapshot(&config, pb->repo)) < 0)
102 ##### 4 return ret;
103 -
104 - #define config_get(KEY,DST,DFLT) do { \
105 - ret = git_config_get_int64(&val, config, KEY); \
106 - if (!ret) { \
107 - if (!git__is_sizet(val)) { \
108 - git_error_set(GIT_ERROR_CONFIG, \
109 - "configuration value '%s' is too large", KEY); \
110 - ret = -1; \
111 - goto out; \
112 - } \
113 - (DST) = (size_t)val; \
114 - } else if (ret == GIT_ENOTFOUND) { \
115 - (DST) = (DFLT); \
116 - ret = 0; \
117 - } else if (ret < 0) goto out; } while (0)
118 -
119 56 5-15 config_get("pack.deltaCacheSize", pb->max_delta_cache_size,
120 - GIT_PACK_DELTA_CACHE_SIZE);
121 56 16-26 config_get("pack.deltaCacheLimit", pb->cache_max_small_delta_size,
122 - GIT_PACK_DELTA_CACHE_LIMIT);
123 56 27-37 config_get("pack.deltaCacheSize", pb->big_file_threshold,
124 - GIT_PACK_BIG_FILE_THRESHOLD);
125 56 38-48 config_get("pack.windowMemory", pb->window_memory_limit, 0);
126 -
127 - #undef config_get
128 -
129 - out:
130 56 49 git_config_free(config);
131 -
132 56 50 return ret;
133 - }
134 -
135 56 2 int git_packbuilder_new(git_packbuilder **out, git_repository *repo)
136 - {
137 - git_packbuilder *pb;
138 -
139 56 2 *out = NULL;
140 -
141 56 2 pb = git__calloc(1, sizeof(*pb));
142 56 3,4 GIT_ERROR_CHECK_ALLOC(pb);
143 -
144 56 5,6,8 if (git_oidmap_new(&pb->object_ix) < 0 ||
145 56 7,10 git_oidmap_new(&pb->walk_objects) < 0 ||
146 56 9 git_pool_init(&pb->object_pool, sizeof(struct walk_object)) < 0)
147 - goto on_error;
148 -
149 56 11 pb->repo = repo;
150 56 11 pb->nr_threads = 1; /* do not spawn any thread by default */
151 -
152 56 11,12,14 if (git_hash_ctx_init(&pb->ctx) < 0 ||
153 56 13,16 git_zstream_init(&pb->zstream, GIT_ZSTREAM_DEFLATE) < 0 ||
154 56 15,18 git_repository_odb(&pb->odb, repo) < 0 ||
155 56 17 packbuilder_config(pb) < 0)
156 - goto on_error;
157 -
158 - #ifdef GIT_THREADS
159 -
160 56 19,20,22 if (git_mutex_init(&pb->cache_mutex) ||
161 56 21,24 git_mutex_init(&pb->progress_mutex) ||
162 56 23 git_cond_init(&pb->progress_cond))
163 - {
164 ##### 25 git_error_set(GIT_ERROR_OS, "failed to initialize packbuilder mutex");
165 ##### 27 goto on_error;
166 - }
167 -
168 - #endif
169 -
170 56 26 *out = pb;
171 56 26 return 0;
172 -
173 - on_error:
174 ##### 28 git_packbuilder_free(pb);
175 ##### 29 return -1;
176 - }
177 -
178 44 2 unsigned int git_packbuilder_set_threads(git_packbuilder *pb, unsigned int n)
179 - {
180 44 2,3 assert(pb);
181 -
182 - #ifdef GIT_THREADS
183 44 4 pb->nr_threads = n;
184 - #else
185 - GIT_UNUSED(n);
186 - assert(1 == pb->nr_threads);
187 - #endif
188 -
189 44 4 return pb->nr_threads;
190 - }
191 -
192 53 2 static int rehash(git_packbuilder *pb)
193 - {
194 - git_pobject *po;
195 - size_t i;
196 -
197 53 2 git_oidmap_clear(pb->object_ix);
198 -
199 53 3,7,8 for (i = 0, po = pb->object_list; i < pb->nr_objects; i++, po++) {
200 ##### 4,5 if (git_oidmap_set(pb->object_ix, &po->id, po) < 0)
201 ##### 6 return -1;
202 - }
203 -
204 53 9 return 0;
205 - }
206 -
207 3974 2 int git_packbuilder_insert(git_packbuilder *pb, const git_oid *oid,
208 - const char *name)
209 - {
210 - git_pobject *po;
211 - size_t newsize;
212 - int ret;
213 -
214 3974 2-4 assert(pb && oid);
215 -
216 - /* If the object already exists in the hash table, then we don't
217 - * have any work to do */
218 3974 5,6 if (git_oidmap_exists(pb->object_ix, oid))
219 1657 7 return 0;
220 -
221 2317 8 if (pb->nr_objects >= pb->nr_alloc) {
222 53 9-15 GIT_ERROR_CHECK_ALLOC_ADD(&newsize, pb->nr_alloc, 1024);
223 53 16-22 GIT_ERROR_CHECK_ALLOC_MULTIPLY(&newsize, newsize / 2, 3);
224 -
225 53 23,24 if (!git__is_uint32(newsize)) {
226 ##### 25 git_error_set(GIT_ERROR_NOMEMORY, "packfile too large to fit in memory.");
227 ##### 26 return -1;
228 - }
229 -
230 53 27 pb->nr_alloc = newsize;
231 -
232 53 27 pb->object_list = git__reallocarray(pb->object_list,
233 - pb->nr_alloc, sizeof(*po));
234 53 28,29 GIT_ERROR_CHECK_ALLOC(pb->object_list);
235 -
236 53 30,31 if (rehash(pb) < 0)
237 ##### 32 return -1;
238 - }
239 -
240 2317 33 po = pb->object_list + pb->nr_objects;
241 2317 33 memset(po, 0x0, sizeof(*po));
242 -
243 2317 33,34 if ((ret = git_odb_read_header(&po->size, &po->type, pb->odb, oid)) < 0)
244 ##### 35 return ret;
245 -
246 2317 36 pb->nr_objects++;
247 2317 36 git_oid_cpy(&po->id, oid);
248 2317 37 po->hash = name_hash(name);
249 -
250 2317 38,39 if (git_oidmap_set(pb->object_ix, &po->id, po) < 0) {
251 ##### 40 git_error_set_oom();
252 ##### 41 return -1;
253 - }
254 -
255 2317 42 pb->done = false;
256 -
257 2317 42 if (pb->progress_cb) {
258 1995 43 double current_time = git__timer();
259 1995 44 double elapsed = current_time - pb->last_progress_report_time;
260 -
261 1995 44 if (elapsed >= MIN_PROGRESS_UPDATE_INTERVAL) {
262 36 45 pb->last_progress_report_time = current_time;
263 -
264 36 45 ret = pb->progress_cb(
265 - GIT_PACKBUILDER_ADDING_OBJECTS,
266 - pb->nr_objects, 0, pb->progress_cb_payload);
267 -
268 36 46 if (ret)
269 ##### 47 return git_error_set_after_callback(ret);
270 - }
271 - }
272 -
273 2317 48 return 0;
274 - }
275 -
276 ##### 2 static int get_delta(void **out, git_odb *odb, git_pobject *po)
277 - {
278 ##### 2 git_odb_object *src = NULL, *trg = NULL;
279 - size_t delta_size;
280 - void *delta_buf;
281 - int error;
282 -
283 ##### 2 *out = NULL;
284 -
285 ##### 2,3,5 if (git_odb_read(&src, odb, &po->delta->id) < 0 ||
286 ##### 4 git_odb_read(&trg, odb, &po->id) < 0)
287 - goto on_error;
288 -
289 ##### 6-10 error = git_delta(&delta_buf, &delta_size,
290 - git_odb_object_data(src), git_odb_object_size(src),
291 - git_odb_object_data(trg), git_odb_object_size(trg),
292 - 0);
293 -
294 ##### 11,12 if (error < 0 && error != GIT_EBUFS)
295 ##### 13 goto on_error;
296 -
297 ##### 14,15 if (error == GIT_EBUFS || delta_size != po->delta_size) {
298 ##### 16 git_error_set(GIT_ERROR_INVALID, "delta size changed");
299 ##### 20 goto on_error;
300 - }
301 -
302 ##### 17 *out = delta_buf;
303 -
304 ##### 17 git_odb_object_free(src);
305 ##### 18 git_odb_object_free(trg);
306 ##### 19 return 0;
307 -
308 - on_error:
309 ##### 21 git_odb_object_free(src);
310 ##### 22 git_odb_object_free(trg);
311 ##### 23 return -1;
312 - }
313 -
314 2222 2 static int write_object(
315 - git_packbuilder *pb,
316 - git_pobject *po,
317 - int (*write_cb)(void *buf, size_t size, void *cb_data),
318 - void *cb_data)
319 - {
320 2222 2 git_odb_object *obj = NULL;
321 - git_object_t type;
322 2222 2 unsigned char hdr[10], *zbuf = NULL;
323 2222 2 void *data = NULL;
324 2222 2 size_t hdr_len, zbuf_len = COMPRESS_BUFLEN, data_len;
325 - int error;
326 -
327 - /*
328 - * If we have a delta base, let's use the delta to save space.
329 - * Otherwise load the whole object. 'data' ends up pointing to
330 - * whatever data we want to put into the packfile.
331 - */
332 2222 2 if (po->delta) {
333 124 3 if (po->delta_data)
334 124 4 data = po->delta_data;
335 ##### 5,6 else if ((error = get_delta(&data, pb->odb, po)) < 0)
336 ##### 7 goto done;
337 -
338 124 8 data_len = po->delta_size;
339 124 8 type = GIT_OBJECT_REF_DELTA;
340 - } else {
341 2098 9,10 if ((error = git_odb_read(&obj, pb->odb, &po->id)) < 0)
342 ##### 11 goto done;
343 -
344 2098 12 data = (void *)git_odb_object_data(obj);
345 2098 13 data_len = git_odb_object_size(obj);
346 2098 14 type = git_odb_object_type(obj);
347 - }
348 -
349 - /* Write header */
350 2222 15 hdr_len = git_packfile__object_header(hdr, data_len, type);
351 -
352 2222 16-19 if ((error = write_cb(hdr, hdr_len, cb_data)) < 0 ||
353 2222 18 (error = git_hash_update(&pb->ctx, hdr, hdr_len)) < 0)
354 - goto done;
355 -
356 2222 20 if (type == GIT_OBJECT_REF_DELTA) {
357 124 21-24 if ((error = write_cb(po->delta->id.id, GIT_OID_RAWSZ, cb_data)) < 0 ||
358 124 23 (error = git_hash_update(&pb->ctx, po->delta->id.id, GIT_OID_RAWSZ)) < 0)
359 - goto done;
360 - }
361 -
362 - /* Write data */
363 2222 25 if (po->z_delta_size) {
364 124 26 data_len = po->z_delta_size;
365 -
366 124 26-29 if ((error = write_cb(data, data_len, cb_data)) < 0 ||
367 124 28 (error = git_hash_update(&pb->ctx, data, data_len)) < 0)
368 - goto done;
369 - } else {
370 2098 30 zbuf = git__malloc(zbuf_len);
371 2098 31,32 GIT_ERROR_CHECK_ALLOC(zbuf);
372 -
373 2098 33 git_zstream_reset(&pb->zstream);
374 -
375 2098 34,35 if ((error = git_zstream_set_input(&pb->zstream, data, data_len)) < 0)
376 ##### 36 goto done;
377 -
378 4196 37,45,46 while (!git_zstream_done(&pb->zstream)) {
379 2098 38-41 if ((error = git_zstream_get_output(zbuf, &zbuf_len, &pb->zstream)) < 0 ||
380 2098 40,42,43 (error = write_cb(zbuf, zbuf_len, cb_data)) < 0 ||
381 2098 42 (error = git_hash_update(&pb->ctx, zbuf, zbuf_len)) < 0)
382 - goto done;
383 -
384 2098 44 zbuf_len = COMPRESS_BUFLEN; /* reuse buffer */
385 - }
386 - }
387 -
388 - /*
389 - * If po->delta is true, data is a delta and it is our
390 - * responsibility to free it (otherwise it's a git_object's
391 - * data). We set po->delta_data to NULL in case we got the
392 - * data from there instead of get_delta(). If we didn't,
393 - * there's no harm.
394 - */
395 2222 47 if (po->delta) {
396 124 48 git__free(data);
397 124 49 po->delta_data = NULL;
398 - }
399 -
400 2222 50 pb->nr_written++;
401 -
402 - done:
403 2222 51 git__free(zbuf);
404 2222 52 git_odb_object_free(obj);
405 2222 53 return error;
406 - }
407 -
408 - enum write_one_status {
409 - WRITE_ONE_SKIP = -1, /* already written */
410 - WRITE_ONE_BREAK = 0, /* writing this will bust the limit; not written */
411 - WRITE_ONE_WRITTEN = 1, /* normal */
412 - WRITE_ONE_RECURSIVE = 2 /* already scheduled to be written */
413 - };
414 -
415 2346 2 static int write_one(
416 - enum write_one_status *status,
417 - git_packbuilder *pb,
418 - git_pobject *po,
419 - int (*write_cb)(void *buf, size_t size, void *cb_data),
420 - void *cb_data)
421 - {
422 - int error;
423 -
424 2346 2 if (po->recursing) {
425 ##### 3 *status = WRITE_ONE_RECURSIVE;
426 ##### 3 return 0;
427 2346 4 } else if (po->written) {
428 124 5 *status = WRITE_ONE_SKIP;
429 124 5 return 0;
430 - }
431 -
432 2222 6 if (po->delta) {
433 124 7 po->recursing = 1;
434 -
435 124 7,8 if ((error = write_one(status, pb, po->delta, write_cb, cb_data)) < 0)
436 ##### 9 return error;
437 -
438 - /* we cannot depend on this one */
439 124 10 if (*status == WRITE_ONE_RECURSIVE)
440 ##### 11 po->delta = NULL;
441 - }
442 -
443 2222 12 *status = WRITE_ONE_WRITTEN;
444 2222 12 po->written = 1;
445 2222 12 po->recursing = 0;
446 -
447 2222 12 return write_object(pb, po, write_cb, cb_data);
448 - }
449 -
450 2475 2 GIT_INLINE(void) add_to_write_order(git_pobject **wo, size_t *endp,
451 - git_pobject *po)
452 - {
453 2475 2 if (po->filled)
454 2475 3,5 return;
455 2297 4 wo[(*endp)++] = po;
456 2297 4 po->filled = 1;
457 - }
458 -
459 522 2 static void add_descendants_to_write_order(git_pobject **wo, size_t *endp,
460 - git_pobject *po)
461 - {
462 522 2 int add_to_order = 1;
463 522 2,20 while (po) {
464 522 3 if (add_to_order) {
465 - git_pobject *s;
466 - /* add this node... */
467 522 4 add_to_write_order(wo, endp, po);
468 - /* all its siblings... */
469 522 5,7,8 for (s = po->delta_sibling; s; s = s->delta_sibling) {
470 ##### 6 add_to_write_order(wo, endp, s);
471 - }
472 - }
473 - /* drop down a level to add left subtree nodes if possible */
474 522 9 if (po->delta_child) {
475 ##### 10 add_to_order = 1;
476 ##### 10 po = po->delta_child;
477 - } else {
478 522 11 add_to_order = 0;
479 - /* our sibling might have some children, it is next */
480 522 11 if (po->delta_sibling) {
481 ##### 12 po = po->delta_sibling;
482 ##### 12 continue;
483 - }
484 - /* go back to our parent node */
485 522 13 po = po->delta;
486 522 13,15,16 while (po && !po->delta_sibling) {
487 - /* we're on the right side of a subtree, keep
488 - * going up until we can go right again */
489 ##### 14 po = po->delta;
490 - }
491 522 17 if (!po) {
492 - /* done- we hit our original root node */
493 522 18,21 return;
494 - }
495 - /* pass it off to sibling at this level */
496 ##### 19 po = po->delta_sibling;
497 - }
498 - };
499 - }
500 -
501 522 2 static void add_family_to_write_order(git_pobject **wo, size_t *endp,
502 - git_pobject *po)
503 - {
504 - git_pobject *root;
505 -
506 522 2-4 for (root = po; root->delta; root = root->delta)
507 - ; /* nothing */
508 522 5 add_descendants_to_write_order(wo, endp, root);
509 522 6 }
510 -
511 350 2 static int cb_tag_foreach(const char *name, git_oid *oid, void *data)
512 - {
513 350 2 git_packbuilder *pb = data;
514 - git_pobject *po;
515 -
516 - GIT_UNUSED(name);
517 -
518 350 2,3 if ((po = git_oidmap_get(pb->object_ix, oid)) == NULL)
519 85 4 return 0;
520 -
521 265 5 po->tagged = 1;
522 -
523 - /* TODO: peel objects */
524 -
525 265 5 return 0;
526 - }
527 -
528 54 2 static git_pobject **compute_write_order(git_packbuilder *pb)
529 - {
530 - size_t i, wo_end, last_untagged;
531 - git_pobject **wo;
532 -
533 54 2,3 if ((wo = git__mallocarray(pb->nr_objects, sizeof(*wo))) == NULL)
534 ##### 4 return NULL;
535 -
536 2351 5-7 for (i = 0; i < pb->nr_objects; i++) {
537 2297 6 git_pobject *po = pb->object_list + i;
538 2297 6 po->tagged = 0;
539 2297 6 po->filled = 0;
540 2297 6 po->delta_child = NULL;
541 2297 6 po->delta_sibling = NULL;
542 - }
543 -
544 - /*
545 - * Fully connect delta_child/delta_sibling network.
546 - * Make sure delta_sibling is sorted in the original
547 - * recency order.
548 - */
549 2351 8,12 for (i = pb->nr_objects; i > 0;) {
550 2297 9 git_pobject *po = &pb->object_list[--i];
551 2297 9 if (!po->delta)
552 2169 10 continue;
553 - /* Mark me as the first child */
554 128 11 po->delta_sibling = po->delta->delta_child;
555 128 11 po->delta->delta_child = po;
556 - }
557 -
558 - /*
559 - * Mark objects that are at the tip of tags.
560 - */
561 54 13,14 if (git_tag_foreach(pb->repo, &cb_tag_foreach, pb) < 0) {
562 ##### 15 git__free(wo);
563 ##### 16 return NULL;
564 - }
565 -
566 - /*
567 - * Give the objects in the original recency order until
568 - * we see a tagged tip.
569 - */
570 270 17,21,22 for (i = wo_end = 0; i < pb->nr_objects; i++) {
571 265 18 git_pobject *po = pb->object_list + i;
572 265 18 if (po->tagged)
573 49 19 break;
574 216 20 add_to_write_order(wo, &wo_end, po);
575 - }
576 54 23 last_untagged = i;
577 -
578 - /*
579 - * Then fill all the tagged tips.
580 - */
581 2135 23,26,27 for (; i < pb->nr_objects; i++) {
582 2081 24 git_pobject *po = pb->object_list + i;
583 2081 24 if (po->tagged)
584 227 25 add_to_write_order(wo, &wo_end, po);
585 - }
586 -
587 - /*
588 - * And then all remaining commits and tags.
589 - */
590 2135 28,33,34 for (i = last_untagged; i < pb->nr_objects; i++) {
591 2081 29 git_pobject *po = pb->object_list + i;
592 2081 29,30 if (po->type != GIT_OBJECT_COMMIT &&
593 1528 30 po->type != GIT_OBJECT_TAG)
594 1351 31 continue;
595 730 32 add_to_write_order(wo, &wo_end, po);
596 - }
597 -
598 - /*
599 - * And then all the trees.
600 - */
601 2135 35,39,40 for (i = last_untagged; i < pb->nr_objects; i++) {
602 2081 36 git_pobject *po = pb->object_list + i;
603 2081 36 if (po->type != GIT_OBJECT_TREE)
604 1301 37 continue;
605 780 38 add_to_write_order(wo, &wo_end, po);
606 - }
607 -
608 - /*
609 - * Finally all the rest in really tight order
610 - */
611 2135 41,44,45 for (i = last_untagged; i < pb->nr_objects; i++) {
612 2081 42 git_pobject *po = pb->object_list + i;
613 2081 42 if (!po->filled)
614 522 43 add_family_to_write_order(wo, &wo_end, po);
615 - }
616 -
617 54 46 if (wo_end != pb->nr_objects) {
618 ##### 47 git__free(wo);
619 ##### 48 git_error_set(GIT_ERROR_INVALID, "invalid write order");
620 ##### 49 return NULL;
621 - }
622 -
623 54 50 return wo;
624 - }
625 -
626 54 2 static int write_pack(git_packbuilder *pb,
627 - int (*write_cb)(void *buf, size_t size, void *cb_data),
628 - void *cb_data)
629 - {
630 - git_pobject **write_order;
631 - git_pobject *po;
632 - enum write_one_status status;
633 - struct git_pack_header ph;
634 - git_oid entry_oid;
635 54 2 size_t i = 0;
636 54 2 int error = 0;
637 -
638 54 2 write_order = compute_write_order(pb);
639 54 3 if (write_order == NULL)
640 ##### 4 return -1;
641 -
642 54 5,6 if (!git__is_uint32(pb->nr_objects)) {
643 ##### 7 git_error_set(GIT_ERROR_INVALID, "too many objects");
644 ##### 8 return -1;
645 - }
646 -
647 - /* Write pack header */
648 54 9 ph.hdr_signature = htonl(PACK_SIGNATURE);
649 54 10 ph.hdr_version = htonl(PACK_VERSION);
650 54 11 ph.hdr_entries = htonl(pb->nr_objects);
651 -
652 54 12-15 if ((error = write_cb(&ph, sizeof(ph), cb_data)) < 0 ||
653 52 14 (error = git_hash_update(&pb->ctx, &ph, sizeof(ph))) < 0)
654 - goto done;
655 -
656 52 16 pb->nr_remaining = pb->nr_objects;
657 - do {
658 52 17 pb->nr_written = 0;
659 2274 17,21,22 for ( ; i < pb->nr_objects; ++i) {
660 2222 18 po = write_order[i];
661 -
662 2222 18,19 if ((error = write_one(&status, pb, po, write_cb, cb_data)) < 0)
663 ##### 20 goto done;
664 - }
665 -
666 52 23 pb->nr_remaining -= pb->nr_written;
667 52 23,24 } while (pb->nr_remaining && i < pb->nr_objects);
668 -
669 52 25,26 if ((error = git_hash_final(&entry_oid, &pb->ctx)) < 0)
670 ##### 27 goto done;
671 -
672 52 28 error = write_cb(entry_oid.id, GIT_OID_RAWSZ, cb_data);
673 -
674 - done:
675 - /* if callback cancelled writing, we must still free delta_data */
676 129 29,33,34 for ( ; i < pb->nr_objects; ++i) {
677 75 30 po = write_order[i];
678 75 30 if (po->delta_data) {
679 4 31 git__free(po->delta_data);
680 4 32 po->delta_data = NULL;
681 - }
682 - }
683 -
684 54 35 git__free(write_order);
685 54 36 return error;
686 - }
687 -
688 ##### 2 static int write_pack_buf(void *buf, size_t size, void *data)
689 - {
690 ##### 2 git_buf *b = (git_buf *)data;
691 ##### 2 return git_buf_put(b, buf, size);
692 - }
693 -
694 7023 2 static int type_size_sort(const void *_a, const void *_b)
695 - {
696 7023 2 const git_pobject *a = (git_pobject *)_a;
697 7023 2 const git_pobject *b = (git_pobject *)_b;
698 -
699 7023 2 if (a->type > b->type)
700 316 3 return -1;
701 6707 4 if (a->type < b->type)
702 3198 5 return 1;
703 3509 6 if (a->hash > b->hash)
704 141 7 return -1;
705 3368 8 if (a->hash < b->hash)
706 ##### 9 return 1;
707 - /*
708 - * TODO
709 - *
710 - if (a->preferred_base > b->preferred_base)
711 - return -1;
712 - if (a->preferred_base < b->preferred_base)
713 - return 1;
714 - */
715 3368 10 if (a->size > b->size)
716 943 11 return -1;
717 2425 12 if (a->size < b->size)
718 1996 13 return 1;
719 429 14 return a < b ? -1 : (a > b); /* newest first */
720 - }
721 -
722 129 2 static int delta_cacheable(
723 - git_packbuilder *pb,
724 - size_t src_size,
725 - size_t trg_size,
726 - size_t delta_size)
727 - {
728 - size_t new_size;
729 -
730 129 2,3 if (git__add_sizet_overflow(&new_size, pb->delta_cache_size, delta_size))
731 ##### 4 return 0;
732 -
733 129 5,6 if (pb->max_delta_cache_size && new_size > pb->max_delta_cache_size)
734 ##### 7 return 0;
735 -
736 129 8 if (delta_size < pb->cache_max_small_delta_size)
737 129 9 return 1;
738 -
739 - /* cache delta, if objects are large enough compared to delta size */
740 ##### 10 if ((src_size >> 20) + (trg_size >> 21) > (delta_size >> 10))
741 ##### 11 return 1;
742 -
743 ##### 12 return 0;
744 - }
745 -
746 7262 2 static int try_delta(git_packbuilder *pb, struct unpacked *trg,
747 - struct unpacked *src, size_t max_depth,
748 - size_t *mem_usage, int *ret)
749 - {
750 7262 2 git_pobject *trg_object = trg->object;
751 7262 2 git_pobject *src_object = src->object;
752 - git_odb_object *obj;
753 - size_t trg_size, src_size, delta_size, sizediff, max_size, sz;
754 - size_t ref_depth;
755 - void *delta_buf;
756 -
757 - /* Don't bother doing diffs between different types */
758 7262 2 if (trg_object->type != src_object->type) {
759 861 3 *ret = -1;
760 861 3 return 0;
761 - }
762 -
763 6401 4 *ret = 0;
764 -
765 - /* TODO: support reuse-delta */
766 -
767 - /* Let's not bust the allowed depth. */
768 6401 4 if (src->depth >= max_depth)
769 ##### 5 return 0;
770 -
771 - /* Now some size filtering heuristics. */
772 6401 6 trg_size = trg_object->size;
773 6401 6 if (!trg_object->delta) {
774 6222 7 max_size = trg_size/2 - 20;
775 6222 7 ref_depth = 1;
776 - } else {
777 179 8 max_size = trg_object->delta_size;
778 179 8 ref_depth = trg->depth;
779 - }
780 -
781 6401 9,9 max_size = (uint64_t)max_size * (max_depth - src->depth) /
782 6401 9 (max_depth - ref_depth + 1);
783 6401 9 if (max_size == 0)
784 ##### 10 return 0;
785 -
786 6401 11 src_size = src_object->size;
787 6401 11-13 sizediff = src_size < trg_size ? trg_size - src_size : 0;
788 6401 14 if (sizediff >= max_size)
789 105 15 return 0;
790 6296 16 if (trg_size < src_size / 32)
791 ##### 17 return 0;
792 -
793 - /* Load data if not already done */
794 6296 18 if (!trg->data) {
795 1181 19,20 if (git_odb_read(&obj, pb->odb, &trg_object->id) < 0)
796 ##### 21 return -1;
797 -
798 1181 22 sz = git_odb_object_size(obj);
799 1181 23 trg->data = git__malloc(sz);
800 1181 24,25 GIT_ERROR_CHECK_ALLOC(trg->data);
801 1181 26 memcpy(trg->data, git_odb_object_data(obj), sz);
802 -
803 1181 27 git_odb_object_free(obj);
804 -
805 1181 28 if (sz != trg_size) {
806 ##### 29 git_error_set(GIT_ERROR_INVALID,
807 - "inconsistent target object length");
808 ##### 30 return -1;
809 - }
810 -
811 1181 31 *mem_usage += sz;
812 - }
813 6296 32 if (!src->data) {
814 - size_t obj_sz;
815 -
816 138 33-35,37 if (git_odb_read(&obj, pb->odb, &src_object->id) < 0 ||
817 138 35,36 !git__is_ulong(obj_sz = git_odb_object_size(obj)))
818 ##### 38 return -1;
819 -
820 138 39 sz = obj_sz;
821 138 39 src->data = git__malloc(sz);
822 138 40,41 GIT_ERROR_CHECK_ALLOC(src->data);
823 138 42 memcpy(src->data, git_odb_object_data(obj), sz);
824 -
825 138 43 git_odb_object_free(obj);
826 -
827 138 44 if (sz != src_size) {
828 ##### 45 git_error_set(GIT_ERROR_INVALID,
829 - "inconsistent source object length");
830 ##### 46 return -1;
831 - }
832 -
833 138 47 *mem_usage += sz;
834 - }
835 6296 48 if (!src->index) {
836 1146 49,50 if (git_delta_index_init(&src->index, src->data, src_size) < 0)
837 ##### 51 return 0; /* suboptimal pack - out of memory */
838 -
839 1146 52,53 *mem_usage += git_delta_index_size(src->index);
840 - }
841 -
842 6296 54,55 if (git_delta_create_from_index(&delta_buf, &delta_size, src->index, trg->data, trg_size,
843 - max_size) < 0)
844 6167 56 return 0;
845 -
846 129 57 if (trg_object->delta) {
847 - /* Prefer only shallower same-sized deltas. */
848 1 58,59 if (delta_size == trg_object->delta_size &&
849 ##### 59 src->depth + 1 >= trg->depth) {
850 ##### 60 git__free(delta_buf);
851 ##### 61 return 0;
852 - }
853 - }
854 -
855 129 62-64 git_packbuilder__cache_lock(pb);
856 129 65 if (trg_object->delta_data) {
857 1 66 git__free(trg_object->delta_data);
858 1 67,68 assert(pb->delta_cache_size >= trg_object->delta_size);
859 1 69 pb->delta_cache_size -= trg_object->delta_size;
860 1 69 trg_object->delta_data = NULL;
861 - }
862 129 70,71 if (delta_cacheable(pb, src_size, trg_size, delta_size)) {
863 129 72 bool overflow = git__add_sizet_overflow(
864 - &pb->delta_cache_size, pb->delta_cache_size, delta_size);
865 -
866 129 73-75 git_packbuilder__cache_unlock(pb);
867 -
868 129 76 if (overflow) {
869 ##### 77 git__free(delta_buf);
870 ##### 78 return -1;
871 - }
872 -
873 129 79 trg_object->delta_data = git__realloc(delta_buf, delta_size);
874 129 80-82 GIT_ERROR_CHECK_ALLOC(trg_object->delta_data);
875 - } else {
876 - /* create delta when writing the pack */
877 ##### 83-85 git_packbuilder__cache_unlock(pb);
878 ##### 86 git__free(delta_buf);
879 - }
880 -
881 129 87 trg_object->delta = src_object;
882 129 87 trg_object->delta_size = delta_size;
883 129 87 trg->depth = src->depth + 1;
884 -
885 129 87 *ret = 1;
886 129 87 return 0;
887 - }
888 -
889 ##### 2 static size_t check_delta_limit(git_pobject *me, size_t n)
890 - {
891 ##### 2 git_pobject *child = me->delta_child;
892 ##### 2 size_t m = n;
893 -
894 ##### 2,7 while (child) {
895 ##### 3 size_t c = check_delta_limit(child, n + 1);
896 ##### 4 if (m < c)
897 ##### 5 m = c;
898 ##### 6 child = child->delta_sibling;
899 - }
900 ##### 8 return m;
901 - }
902 -
903 1355 2 static size_t free_unpacked(struct unpacked *n)
904 - {
905 1355 2 size_t freed_mem = 0;
906 -
907 1355 2 if (n->index) {
908 651 3 freed_mem += git_delta_index_size(n->index);
909 651 4 git_delta_index_free(n->index);
910 - }
911 1355 5 n->index = NULL;
912 -
913 1355 5 if (n->data) {
914 758 6 freed_mem += n->object->size;
915 758 6 git__free(n->data);
916 758 7 n->data = NULL;
917 - }
918 1355 8 n->object = NULL;
919 1355 8 n->depth = 0;
920 1355 8 return freed_mem;
921 - }
922 -
923 1407 2 static int report_delta_progress(
924 - git_packbuilder *pb, uint32_t count, bool force)
925 - {
926 - int ret;
927 -
928 1407 2 if (pb->progress_cb) {
929 1196 3 double current_time = git__timer();
930 1196 4 double elapsed = current_time - pb->last_progress_report_time;
931 -
932 1196 4,5 if (force || elapsed >= MIN_PROGRESS_UPDATE_INTERVAL) {
933 36 6 pb->last_progress_report_time = current_time;
934 -
935 36 6 ret = pb->progress_cb(
936 - GIT_PACKBUILDER_DELTAFICATION,
937 - count, pb->nr_objects, pb->progress_cb_payload);
938 -
939 36 7 if (ret)
940 ##### 8 return git_error_set_after_callback(ret);
941 - }
942 - }
943 -
944 1407 9 return 0;
945 - }
946 -
947 51 2 static int find_deltas(git_packbuilder *pb, git_pobject **list,
948 - size_t *list_size, size_t window, size_t depth)
949 - {
950 - git_pobject *po;
951 51 2 git_buf zbuf = GIT_BUF_INIT;
952 - struct unpacked *array;
953 51 2 size_t idx = 0, count = 0;
954 51 2 size_t mem_usage = 0;
955 - size_t i;
956 51 2 int error = -1;
957 -
958 51 2 array = git__calloc(window, sizeof(struct unpacked));
959 51 3,4 GIT_ERROR_CHECK_ALLOC(array);
960 -
961 - for (;;) {
962 1406 5 struct unpacked *n = array + idx;
963 1406 5 size_t max_depth, j, best_base = SIZE_MAX;
964 -
965 1406 5-7 git_packbuilder__progress_lock(pb);
966 1406 8 if (!*list_size) {
967 51 9-11 git_packbuilder__progress_unlock(pb);
968 51 12 break;
969 - }
970 -
971 1355 13 pb->nr_deltified += 1;
972 1355 13 report_delta_progress(pb, pb->nr_deltified, false);
973 -
974 1355 14 po = *list++;
975 1355 14 (*list_size)--;
976 1355 14-16 git_packbuilder__progress_unlock(pb);
977 -
978 1355 17 mem_usage -= free_unpacked(n);
979 1355 18 n->object = po;
980 -
981 1355 18,21,22 while (pb->window_memory_limit &&
982 ##### 22,23 mem_usage > pb->window_memory_limit &&
983 - count > 1) {
984 ##### 19 size_t tail = (idx + window - count) % window;
985 ##### 19 mem_usage -= free_unpacked(array + tail);
986 ##### 20 count--;
987 - }
988 -
989 - /*
990 - * If the current object is at pack edge, take the depth the
991 - * objects that depend on the current object into account
992 - * otherwise they would become too deep.
993 - */
994 1355 24 max_depth = depth;
995 1355 24 if (po->delta_child) {
996 ##### 25 size_t delta_limit = check_delta_limit(po, 0);
997 -
998 ##### 26 if (delta_limit > max_depth)
999 ##### 27 goto next;
1000 -
1001 ##### 28 max_depth -= delta_limit;
1002 - }
1003 -
1004 1355 29 j = window;
1005 7756 29,43 while (--j > 0) {
1006 - int ret;
1007 7529 30 size_t other_idx = idx + j;
1008 - struct unpacked *m;
1009 -
1010 7529 30 if (other_idx >= window)
1011 4153 31 other_idx -= window;
1012 -
1013 7529 32 m = array + other_idx;
1014 7529 32 if (!m->object)
1015 1128 33,42 break;
1016 -
1017 7262 34,35 if (try_delta(pb, n, m, max_depth, &mem_usage, &ret) < 0)
1018 ##### 36 goto on_error;
1019 7262 37 if (ret < 0)
1020 861 38 break;
1021 6401 39 else if (ret > 0)
1022 6401 40,41 best_base = other_idx;
1023 - }
1024 -
1025 - /*
1026 - * If we decided to cache the delta data, then it is best
1027 - * to compress it right away. First because we have to do
1028 - * it anyway, and doing it here while we're threaded will
1029 - * save a lot of time in the non threaded write phase,
1030 - * as well as allow for caching more deltas within
1031 - * the same cache size limit.
1032 - * ...
1033 - * But only if not writing to stdout, since in that case
1034 - * the network is most likely throttling writes anyway,
1035 - * and therefore it is best to go to the write phase ASAP
1036 - * instead, as we can afford spending more time compressing
1037 - * between writes at that moment.
1038 - */
1039 1355 44 if (po->delta_data) {
1040 128 45,46 if (git_zstream_deflatebuf(&zbuf, po->delta_data, po->delta_size) < 0)
1041 ##### 47 goto on_error;
1042 -
1043 128 48 git__free(po->delta_data);
1044 128 49 po->delta_data = git__malloc(zbuf.size);
1045 128 50,51 GIT_ERROR_CHECK_ALLOC(po->delta_data);
1046 -
1047 128 52 memcpy(po->delta_data, zbuf.ptr, zbuf.size);
1048 128 52 po->z_delta_size = zbuf.size;
1049 128 52 git_buf_clear(&zbuf);
1050 -
1051 128 53-55 git_packbuilder__cache_lock(pb);
1052 128 56 pb->delta_cache_size -= po->delta_size;
1053 128 56 pb->delta_cache_size += po->z_delta_size;
1054 128 56-58 git_packbuilder__cache_unlock(pb);
1055 - }
1056 -
1057 - /*
1058 - * If we made n a delta, and if n is already at max
1059 - * depth, leaving it in the window is pointless. we
1060 - * should evict it first.
1061 - */
1062 1355 59,60 if (po->delta && max_depth <= n->depth)
1063 ##### 61 continue;
1064 -
1065 - /*
1066 - * Move the best delta base up in the window, after the
1067 - * currently deltified object, to keep it longer. It will
1068 - * be the first base object to be attempted next.
1069 - */
1070 1355 62 if (po->delta) {
1071 128 63 struct unpacked swap = array[best_base];
1072 128 63 size_t dist = (window + idx - best_base) % window;
1073 128 63 size_t dst = best_base;
1074 430 63,65 while (dist--) {
1075 302 64 size_t src = (dst + 1) % window;
1076 302 64 array[dst] = array[src];
1077 302 64 dst = src;
1078 - }
1079 128 66 array[dst] = swap;
1080 - }
1081 -
1082 - next:
1083 1355 67 idx++;
1084 1355 67 if (count + 1 < window)
1085 510 68 count++;
1086 1355 69 if (idx >= window)
1087 88 70 idx = 0;
1088 1355 71 }
1089 51 72 error = 0;
1090 -
1091 - on_error:
1092 612 73,76,77 for (i = 0; i < window; ++i) {
1093 561 74 git__free(array[i].index);
1094 561 75 git__free(array[i].data);
1095 - }
1096 51 78 git__free(array);
1097 51 79 git_buf_dispose(&zbuf);
1098 -
1099 51 80 return error;
1100 - }
1101 -
1102 - #ifdef GIT_THREADS
1103 -
1104 - struct thread_params {
1105 - git_thread thread;
1106 - git_packbuilder *pb;
1107 -
1108 - git_pobject **list;
1109 -
1110 - git_cond cond;
1111 - git_mutex mutex;
1112 -
1113 - size_t list_size;
1114 - size_t remaining;
1115 -
1116 - size_t window;
1117 - size_t depth;
1118 - size_t working;
1119 - size_t data_ready;
1120 - };
1121 -
1122 36 2 static void *threaded_find_deltas(void *arg)
1123 - {
1124 36 2 struct thread_params *me = arg;
1125 -
1126 72 2,19 while (me->remaining) {
1127 36 3 if (find_deltas(me->pb, me->list, &me->remaining,
1128 - me->window, me->depth) < 0) {
1129 - ; /* TODO */
1130 - }
1131 -
1132 36 4-6 git_packbuilder__progress_lock(me->pb);
1133 36 7 me->working = 0;
1134 36 7 git_cond_signal(&me->pb->progress_cond);
1135 36 8-10 git_packbuilder__progress_unlock(me->pb);
1136 -
1137 36 11,12 if (git_mutex_lock(&me->mutex)) {
1138 ##### 13 git_error_set(GIT_ERROR_THREAD, "unable to lock packfile condition mutex");
1139 ##### 14 return NULL;
1140 - }
1141 -
1142 72 15,17 while (!me->data_ready)
1143 36 16 git_cond_wait(&me->cond, &me->mutex);
1144 -
1145 - /*
1146 - * We must not set ->data_ready before we wait on the
1147 - * condition because the main thread may have set it to 1
1148 - * before we get here. In order to be sure that new
1149 - * work is available if we see 1 in ->data_ready, it
1150 - * was initialized to 0 before this thread was spawned
1151 - * and we reset it to 0 right away.
1152 - */
1153 36 18 me->data_ready = 0;
1154 36 18 git_mutex_unlock(&me->mutex);
1155 - }
1156 - /* leave ->working 1 so that this doesn't get more work assigned */
1157 36 20 return NULL;
1158 - }
1159 -
1160 51 2 static int ll_find_deltas(git_packbuilder *pb, git_pobject **list,
1161 - size_t list_size, size_t window, size_t depth)
1162 - {
1163 - struct thread_params *p;
1164 - size_t i;
1165 51 2 int ret, active_threads = 0;
1166 -
1167 51 2 if (!pb->nr_threads)
1168 36 3,4 pb->nr_threads = git_online_cpus();
1169 -
1170 51 5 if (pb->nr_threads <= 1) {
1171 15 6 find_deltas(pb, list, &list_size, window, depth);
1172 15 7 return 0;
1173 - }
1174 -
1175 36 8 p = git__mallocarray(pb->nr_threads, sizeof(*p));
1176 36 9,10 GIT_ERROR_CHECK_ALLOC(p);
1177 -
1178 - /* Partition the work among the threads */
1179 612 11,21,22 for (i = 0; i < pb->nr_threads; ++i) {
1180 576 12 size_t sub_size = list_size / (pb->nr_threads - i);
1181 -
1182 - /* don't use too small segments or no deltas will be found */
1183 576 12,13 if (sub_size < 2*window && i+1 < pb->nr_threads)
1184 540 14 sub_size = 0;
1185 -
1186 576 15 p[i].pb = pb;
1187 576 15 p[i].window = window;
1188 576 15 p[i].depth = depth;
1189 576 15 p[i].working = 1;
1190 576 15 p[i].data_ready = 0;
1191 -
1192 - /* try to split chunks on "path" boundaries */
1193 576 15,17-19 while (sub_size && sub_size < list_size &&
1194 ##### 19,20 list[sub_size]->hash &&
1195 ##### 20 list[sub_size]->hash == list[sub_size-1]->hash)
1196 ##### 16 sub_size++;
1197 -
1198 576 21 p[i].list = list;
1199 576 21 p[i].list_size = sub_size;
1200 576 21 p[i].remaining = sub_size;
1201 -
1202 576 21 list += sub_size;
1203 576 21 list_size -= sub_size;
1204 - }
1205 -
1206 - /* Start work threads */
1207 612 23,33,34 for (i = 0; i < pb->nr_threads; ++i) {
1208 576 24 if (!p[i].list_size)
1209 540 25 continue;
1210 -
1211 36 26 git_mutex_init(&p[i].mutex);
1212 36 27 git_cond_init(&p[i].cond);
1213 -
1214 36 28 ret = git_thread_create(&p[i].thread,
1215 - threaded_find_deltas, &p[i]);
1216 36 29 if (ret) {
1217 ##### 30 git_error_set(GIT_ERROR_THREAD, "unable to create thread");
1218 ##### 31 return -1;
1219 - }
1220 36 32 active_threads++;
1221 - }
1222 -
1223 - /*
1224 - * Now let's wait for work completion. Each time a thread is done
1225 - * with its work, we steal half of the remaining work from the
1226 - * thread with the largest number of unprocessed objects and give
1227 - * it to that newly idle thread. This ensure good load balancing
1228 - * until the remaining object list segments are simply too short
1229 - * to be worth splitting anymore.
1230 - */
1231 72 35,80 while (active_threads) {
1232 36 36 struct thread_params *target = NULL;
1233 36 36 struct thread_params *victim = NULL;
1234 36 36 size_t sub_size = 0;
1235 -
1236 - /* Start by locating a thread that has transitioned its
1237 - * 'working' flag from 1 -> 0. This indicates that it is
1238 - * ready to receive more work using our work-stealing
1239 - * algorithm. */
1240 36 36-38 git_packbuilder__progress_lock(pb);
1241 - for (;;) {
1242 1224 40,43-45 for (i = 0; !target && i < pb->nr_threads; i++)
1243 1152 41 if (!p[i].working)
1244 36 42 target = &p[i];
1245 72 46 if (target)
1246 36 47 break;
1247 36 48 git_cond_wait(&pb->progress_cond, &pb->progress_mutex);
1248 36 39 }
1249 -
1250 - /* At this point we hold the progress lock and have located
1251 - * a thread to receive more work. We still need to locate a
1252 - * thread from which to steal work (the victim). */
1253 612 49,54,55 for (i = 0; i < pb->nr_threads; i++)
1254 576 50,51 if (p[i].remaining > 2*window &&
1255 ##### 52 (!victim || victim->remaining < p[i].remaining))
1256 ##### 53 victim = &p[i];
1257 -
1258 36 56 if (victim) {
1259 ##### 57 sub_size = victim->remaining / 2;
1260 ##### 57 list = victim->list + victim->list_size - sub_size;
1261 ##### 57,59-61 while (sub_size && list[0]->hash &&
1262 ##### 61 list[0]->hash == list[-1]->hash) {
1263 ##### 58 list++;
1264 ##### 58 sub_size--;
1265 - }
1266 ##### 62 if (!sub_size) {
1267 - /*
1268 - * It is possible for some "paths" to have
1269 - * so many objects that no hash boundary
1270 - * might be found. Let's just steal the
1271 - * exact half in that case.
1272 - */
1273 ##### 63 sub_size = victim->remaining / 2;
1274 ##### 63 list -= sub_size;
1275 - }
1276 ##### 64 target->list = list;
1277 ##### 64 victim->list_size -= sub_size;
1278 ##### 64 victim->remaining -= sub_size;
1279 - }
1280 36 65 target->list_size = sub_size;
1281 36 65 target->remaining = sub_size;
1282 36 65 target->working = 1;
1283 36 65-67 git_packbuilder__progress_unlock(pb);
1284 -
1285 36 68,69 if (git_mutex_lock(&target->mutex)) {
1286 ##### 70 git_error_set(GIT_ERROR_THREAD, "unable to lock packfile condition mutex");
1287 ##### 71 git__free(p);
1288 ##### 72 return -1;
1289 - }
1290 -
1291 36 73 target->data_ready = 1;
1292 36 73 git_cond_signal(&target->cond);
1293 36 74 git_mutex_unlock(&target->mutex);
1294 -
1295 36 75 if (!sub_size) {
1296 36 76 git_thread_join(&target->thread, NULL);
1297 36 77 git_cond_free(&target->cond);
1298 36 78 git_mutex_free(&target->mutex);
1299 36 79 active_threads--;
1300 - }
1301 - }
1302 -
1303 36 81 git__free(p);
1304 36 82 return 0;
1305 - }
1306 -
1307 - #else
1308 - #define ll_find_deltas(pb, l, ls, w, d) find_deltas(pb, l, &ls, w, d)
1309 - #endif
1310 -
1311 69 2 static int prepare_pack(git_packbuilder *pb)
1312 - {
1313 - git_pobject **delta_list;
1314 69 2 size_t i, n = 0;
1315 -
1316 69 2,3 if (pb->nr_objects == 0 || pb->done)
1317 17 4 return 0; /* nothing to do */
1318 -
1319 - /*
1320 - * Although we do not report progress during deltafication, we
1321 - * at least report that we are in the deltafication stage
1322 - */
1323 52 5 if (pb->progress_cb)
1324 36 6 pb->progress_cb(GIT_PACKBUILDER_DELTAFICATION, 0, pb->nr_objects, pb->progress_cb_payload);
1325 -
1326 52 7 delta_list = git__mallocarray(pb->nr_objects, sizeof(*delta_list));
1327 52 8,9 GIT_ERROR_CHECK_ALLOC(delta_list);
1328 -
1329 2349 10,15,16 for (i = 0; i < pb->nr_objects; ++i) {
1330 2297 11 git_pobject *po = pb->object_list + i;
1331 -
1332 - /* Make sure the item is within our size limits */
1333 2297 11,12 if (po->size < 50 || po->size > pb->big_file_threshold)
1334 941 13 continue;
1335 -
1336 1356 14 delta_list[n++] = po;
1337 - }
1338 -
1339 52 17 if (n > 1) {
1340 51 18 git__tsort((void **)delta_list, n, type_size_sort);
1341 51 19,20 if (ll_find_deltas(pb, delta_list, n,
1342 - GIT_PACK_WINDOW + 1,
1343 - GIT_PACK_DEPTH) < 0) {
1344 ##### 21 git__free(delta_list);
1345 ##### 22 return -1;
1346 - }
1347 - }
1348 -
1349 52 23 report_delta_progress(pb, pb->nr_objects, true);
1350 -
1351 52 24 pb->done = true;
1352 52 24 git__free(delta_list);
1353 52 25 return 0;
1354 - }
1355 -
1356 - #define PREPARE_PACK if (prepare_pack(pb) < 0) { return -1; }
1357 -
1358 54 2 int git_packbuilder_foreach(git_packbuilder *pb, int (*cb)(void *buf, size_t size, void *payload), void *payload)
1359 - {
1360 54 2-4 PREPARE_PACK;
1361 54 5 return write_pack(pb, cb, payload);
1362 - }
1363 -
1364 ##### 2 int git_packbuilder_write_buf(git_buf *buf, git_packbuilder *pb)
1365 - {
1366 ##### 2-4 PREPARE_PACK;
1367 ##### 5 git_buf_sanitize(buf);
1368 ##### 6 return write_pack(pb, &write_pack_buf, buf);
1369 - }
1370 -
1371 526 2 static int write_cb(void *buf, size_t len, void *payload)
1372 - {
1373 526 2 struct pack_write_context *ctx = payload;
1374 526 2 return git_indexer_append(ctx->indexer, buf, len, ctx->stats);
1375 - }
1376 -
1377 15 2 int git_packbuilder_write(
1378 - git_packbuilder *pb,
1379 - const char *path,
1380 - unsigned int mode,
1381 - git_indexer_progress_cb progress_cb,
1382 - void *progress_cb_payload)
1383 - {
1384 15 2 int error = -1;
1385 15 2 git_buf object_path = GIT_BUF_INIT;
1386 15 2 git_indexer_options opts = GIT_INDEXER_OPTIONS_INIT;
1387 15 2 git_indexer *indexer = NULL;
1388 - git_indexer_progress stats;
1389 - struct pack_write_context ctx;
1390 - int t;
1391 -
1392 15 2-4 PREPARE_PACK;
1393 -
1394 15 5 if (path == NULL) {
1395 1 6,7 if ((error = git_repository_item_path(&object_path, pb->repo, GIT_REPOSITORY_ITEM_OBJECTS)) < 0)
1396 ##### 8 goto cleanup;
1397 1 9-11 if ((error = git_buf_joinpath(&object_path, git_buf_cstr(&object_path), "pack")) < 0)
1398 ##### 12 goto cleanup;
1399 1 13 path = git_buf_cstr(&object_path);
1400 - }
1401 -
1402 15 14 opts.progress_cb = progress_cb;
1403 15 14 opts.progress_cb_payload = progress_cb_payload;
1404 -
1405 15 14,15 if ((error = git_indexer_new(&indexer, path, mode, pb->odb, &opts)) < 0)
1406 ##### 16 goto cleanup;
1407 -
1408 15 17-19 if (!git_repository__configmap_lookup(&t, pb->repo, GIT_CONFIGMAP_FSYNCOBJECTFILES) && t)
1409 1 20 git_indexer__set_fsync(indexer, 1);
1410 -
1411 15 21 ctx.indexer = indexer;
1412 15 21 ctx.stats = &stats;
1413 -
1414 15 21,22 if ((error = git_packbuilder_foreach(pb, write_cb, &ctx)) < 0)
1415 ##### 23 goto cleanup;
1416 -
1417 15 24,25 if ((error = git_indexer_commit(indexer, &stats)) < 0)
1418 ##### 26 goto cleanup;
1419 -
1420 15 27,28 git_oid_cpy(&pb->pack_oid, git_indexer_hash(indexer));
1421 -
1422 - cleanup:
1423 15 29 git_indexer_free(indexer);
1424 15 30 git_buf_dispose(&object_path);
1425 15 31 return error;
1426 - }
1427 -
1428 - #undef PREPARE_PACK
1429 -
1430 1 2 const git_oid *git_packbuilder_hash(git_packbuilder *pb)
1431 - {
1432 1 2 return &pb->pack_oid;
1433 - }
1434 -
1435 -
1436 499 2 static int cb_tree_walk(
1437 - const char *root, const git_tree_entry *entry, void *payload)
1438 - {
1439 - int error;
1440 499 2 struct tree_walk_context *ctx = payload;
1441 -
1442 - /* A commit inside a tree represents a submodule commit and should be skipped. */
1443 499 2,3 if (git_tree_entry_type(entry) == GIT_OBJECT_COMMIT)
1444 ##### 4 return 0;
1445 -
1446 499 5,6,8,9 if (!(error = git_buf_sets(&ctx->buf, root)) &&
1447 499 7 !(error = git_buf_puts(&ctx->buf, git_tree_entry_name(entry))))
1448 499 10-12 error = git_packbuilder_insert(
1449 499 10 ctx->pb, git_tree_entry_id(entry), git_buf_cstr(&ctx->buf));
1450 -
1451 499 13 return error;
1452 - }
1453 -
1454 180 2 int git_packbuilder_insert_commit(git_packbuilder *pb, const git_oid *oid)
1455 - {
1456 - git_commit *commit;
1457 -
1458 180 2,3,5 if (git_commit_lookup(&commit, pb->repo, oid) < 0 ||
1459 180 4 git_packbuilder_insert(pb, oid, NULL) < 0)
1460 ##### 6 return -1;
1461 -
1462 180 7-9 if (git_packbuilder_insert_tree(pb, git_commit_tree_id(commit)) < 0)
1463 ##### 10 return -1;
1464 -
1465 180 11 git_commit_free(commit);
1466 180 12 return 0;
1467 - }
1468 -
1469 258 2 int git_packbuilder_insert_tree(git_packbuilder *pb, const git_oid *oid)
1470 - {
1471 - int error;
1472 258 2 git_tree *tree = NULL;
1473 258 2 struct tree_walk_context context = { pb, GIT_BUF_INIT };
1474 -
1475 258 2-5 if (!(error = git_tree_lookup(&tree, pb->repo, oid)) &&
1476 - !(error = git_packbuilder_insert(pb, oid, NULL)))
1477 258 6 error = git_tree_walk(tree, GIT_TREEWALK_PRE, cb_tree_walk, &context);
1478 -
1479 258 7 git_tree_free(tree);
1480 258 8 git_buf_dispose(&context.buf);
1481 258 9 return error;
1482 - }
1483 -
1484 611 2 int git_packbuilder_insert_recur(git_packbuilder *pb, const git_oid *id, const char *name)
1485 - {
1486 - git_object *obj;
1487 - int error;
1488 -
1489 611 2-4 assert(pb && id);
1490 -
1491 611 5,6 if ((error = git_object_lookup(&obj, pb->repo, id, GIT_OBJECT_ANY)) < 0)
1492 ##### 7 return error;
1493 -
1494 611 8,9 switch (git_object_type(obj)) {
1495 - case GIT_OBJECT_BLOB:
1496 141 10 error = git_packbuilder_insert(pb, id, name);
1497 141 11 break;
1498 - case GIT_OBJECT_TREE:
1499 1 12 error = git_packbuilder_insert_tree(pb, id);
1500 1 13 break;
1501 - case GIT_OBJECT_COMMIT:
1502 180 14 error = git_packbuilder_insert_commit(pb, id);
1503 180 15 break;
1504 - case GIT_OBJECT_TAG:
1505 289 16,17 if ((error = git_packbuilder_insert(pb, id, name)) < 0)
1506 ##### 18 goto cleanup;
1507 289 19,20 error = git_packbuilder_insert_recur(pb, git_tag_target_id((git_tag *) obj), NULL);
1508 289 21 break;
1509 -
1510 - default:
1511 ##### 22 git_error_set(GIT_ERROR_INVALID, "unknown object type");
1512 ##### 23 error = -1;
1513 - }
1514 -
1515 - cleanup:
1516 611 24 git_object_free(obj);
1517 611 25 return error;
1518 - }
1519 -
1520 72 2 size_t git_packbuilder_object_count(git_packbuilder *pb)
1521 - {
1522 72 2 return pb->nr_objects;
1523 - }
1524 -
1525 ##### 2 size_t git_packbuilder_written(git_packbuilder *pb)
1526 - {
1527 ##### 2 return pb->nr_written;
1528 - }
1529 -
1530 1920 2 static int lookup_walk_object(struct walk_object **out, git_packbuilder *pb, const git_oid *id)
1531 - {
1532 - struct walk_object *obj;
1533 -
1534 1920 2 obj = git_pool_mallocz(&pb->object_pool, 1);
1535 1920 3 if (!obj) {
1536 ##### 4 git_error_set_oom();
1537 ##### 5 return -1;
1538 - }
1539 -
1540 1920 6 git_oid_cpy(&obj->id, id);
1541 -
1542 1920 7 *out = obj;
1543 1920 7 return 0;
1544 - }
1545 -
1546 2570 2 static int retrieve_object(struct walk_object **out, git_packbuilder *pb, const git_oid *id)
1547 - {
1548 - struct walk_object *obj;
1549 - int error;
1550 -
1551 2570 2,3 if ((obj = git_oidmap_get(pb->walk_objects, id)) == NULL) {
1552 1920 4,5 if ((error = lookup_walk_object(&obj, pb, id)) < 0)
1553 ##### 6 return error;
1554 -
1555 1920 7,8 if ((error = git_oidmap_set(pb->walk_objects, &obj->id, obj)) < 0)
1556 ##### 9 return error;
1557 - }
1558 -
1559 2570 10 *out = obj;
1560 2570 10 return 0;
1561 - }
1562 -
1563 3 2 static int mark_blob_uninteresting(git_packbuilder *pb, const git_oid *id)
1564 - {
1565 - int error;
1566 - struct walk_object *obj;
1567 -
1568 3 2,3 if ((error = retrieve_object(&obj, pb, id)) < 0)
1569 ##### 4 return error;
1570 -
1571 3 5 obj->uninteresting = 1;
1572 -
1573 3 5 return 0;
1574 - }
1575 -
1576 1 2 static int mark_tree_uninteresting(git_packbuilder *pb, const git_oid *id)
1577 - {
1578 - struct walk_object *obj;
1579 - git_tree *tree;
1580 - int error;
1581 - size_t i;
1582 -
1583 1 2,3 if ((error = retrieve_object(&obj, pb, id)) < 0)
1584 ##### 4 return error;
1585 -
1586 1 5 if (obj->uninteresting)
1587 ##### 6 return 0;
1588 -
1589 1 7 obj->uninteresting = 1;
1590 -
1591 1 7,8 if ((error = git_tree_lookup(&tree, pb->repo, id)) < 0)
1592 ##### 9 return error;
1593 -
1594 4 10,23-25 for (i = 0; i < git_tree_entrycount(tree); i++) {
1595 3 11 const git_tree_entry *entry = git_tree_entry_byindex(tree, i);
1596 3 12 const git_oid *entry_id = git_tree_entry_id(entry);
1597 3 13,14 switch (git_tree_entry_type(entry)) {
1598 - case GIT_OBJECT_TREE:
1599 ##### 15,16 if ((error = mark_tree_uninteresting(pb, entry_id)) < 0)
1600 ##### 17 goto cleanup;
1601 ##### 18 break;
1602 - case GIT_OBJECT_BLOB:
1603 3 19,20 if ((error = mark_blob_uninteresting(pb, entry_id)) < 0)
1604 ##### 21 goto cleanup;
1605 3 22 break;
1606 - default:
1607 - /* it's a submodule or something unknown, we don't want it */
1608 - ;
1609 - }
1610 - }
1611 -
1612 - cleanup:
1613 1 26 git_tree_free(tree);
1614 1 27 return error;
1615 - }
1616 -
1617 - /*
1618 - * Mark the edges of the graph uninteresting. Since we start from a
1619 - * git_revwalk, the commits are already uninteresting, but we need to
1620 - * mark the trees and blobs.
1621 - */
1622 44 2 static int mark_edges_uninteresting(git_packbuilder *pb, git_commit_list *commits)
1623 - {
1624 - int error;
1625 - git_commit_list *list;
1626 - git_commit *commit;
1627 -
1628 771 2,13,14 for (list = commits; list; list = list->next) {
1629 727 3 if (!list->item->uninteresting)
1630 726 4 continue;
1631 -
1632 1 5,6 if ((error = git_commit_lookup(&commit, pb->repo, &list->item->oid)) < 0)
1633 ##### 7 return error;
1634 -
1635 1 8,9 error = mark_tree_uninteresting(pb, git_commit_tree_id(commit));
1636 1 10 git_commit_free(commit);
1637 -
1638 1 11 if (error < 0)
1639 ##### 12 return error;
1640 - }
1641 -
1642 44 15 return 0;
1643 - }
1644 -
1645 797 2 static int pack_objects_insert_tree(git_packbuilder *pb, git_tree *tree)
1646 - {
1647 - size_t i;
1648 - int error;
1649 - git_tree *subtree;
1650 - struct walk_object *obj;
1651 - const char *name;
1652 -
1653 797 2-4 if ((error = retrieve_object(&obj, pb, git_tree_id(tree))) < 0)
1654 ##### 5 return error;
1655 -
1656 797 6,7 if (obj->seen || obj->uninteresting)
1657 36 8 return 0;
1658 -
1659 761 9 obj->seen = 1;
1660 -
1661 761 9,10 if ((error = git_packbuilder_insert(pb, &obj->id, NULL)))
1662 ##### 11 return error;
1663 -
1664 2165 12,35-37 for (i = 0; i < git_tree_entrycount(tree); i++) {
1665 1404 13 const git_tree_entry *entry = git_tree_entry_byindex(tree, i);
1666 1404 14 const git_oid *entry_id = git_tree_entry_id(entry);
1667 1404 15,16 switch (git_tree_entry_type(entry)) {
1668 - case GIT_OBJECT_TREE:
1669 216 17,18 if ((error = git_tree_lookup(&subtree, pb->repo, entry_id)) < 0)
1670 ##### 19 return error;
1671 -
1672 216 20 error = pack_objects_insert_tree(pb, subtree);
1673 216 21 git_tree_free(subtree);
1674 -
1675 216 22 if (error < 0)
1676 ##### 23 return error;
1677 -
1678 216 24 break;
1679 - case GIT_OBJECT_BLOB:
1680 1188 25,26 if ((error = retrieve_object(&obj, pb, entry_id)) < 0)
1681 ##### 27 return error;
1682 1188 28 if (obj->uninteresting)
1683 ##### 29 continue;
1684 1188 30 name = git_tree_entry_name(entry);
1685 1188 31,32 if ((error = git_packbuilder_insert(pb, entry_id, name)) < 0)
1686 ##### 33 return error;
1687 1188 34 break;
1688 - default:
1689 - /* it's a submodule or something unknown, we don't want it */
1690 - ;
1691 - }
1692 - }
1693 -
1694 -
1695 761 38 return error;
1696 - }
1697 -
1698 581 2 static int pack_objects_insert_commit(git_packbuilder *pb, struct walk_object *obj)
1699 - {
1700 - int error;
1701 581 2 git_commit *commit = NULL;
1702 581 2 git_tree *tree = NULL;
1703 -
1704 581 2 obj->seen = 1;
1705 -
1706 581 2,3 if ((error = git_packbuilder_insert(pb, &obj->id, NULL)) < 0)
1707 ##### 4 return error;
1708 -
1709 581 5,6 if ((error = git_commit_lookup(&commit, pb->repo, &obj->id)) < 0)
1710 ##### 7 return error;
1711 -
1712 581 8-10 if ((error = git_tree_lookup(&tree, pb->repo, git_commit_tree_id(commit))) < 0)
1713 ##### 11 goto cleanup;
1714 -
1715 581 12,13 if ((error = pack_objects_insert_tree(pb, tree)) < 0)
1716 ##### 14 goto cleanup;
1717 -
1718 - cleanup:
1719 581 15 git_commit_free(commit);
1720 581 16 git_tree_free(tree);
1721 581 17 return error;
1722 - }
1723 -
1724 44 2 int git_packbuilder_insert_walk(git_packbuilder *pb, git_revwalk *walk)
1725 - {
1726 - int error;
1727 - git_oid id;
1728 - struct walk_object *obj;
1729 -
1730 44 2-4 assert(pb && walk);
1731 -
1732 44 5,6 if ((error = mark_edges_uninteresting(pb, walk->user_input)) < 0)
1733 ##### 7 return error;
1734 -
1735 - /*
1736 - * TODO: git marks the parents of the edges
1737 - * uninteresting. This may provide a speed advantage, but does
1738 - * seem to assume the remote does not have a single-commit
1739 - * history on the other end.
1740 - */
1741 -
1742 - /* walk down each tree up to the blobs and insert them, stopping when uninteresting */
1743 625 8,18,19 while ((error = git_revwalk_next(&id, walk)) == 0) {
1744 581 9,10 if ((error = retrieve_object(&obj, pb, &id)) < 0)
1745 ##### 11 return error;
1746 -
1747 581 12,13 if (obj->seen || obj->uninteresting)
1748 ##### 14 continue;
1749 -
1750 581 15,16 if ((error = pack_objects_insert_commit(pb, obj)) < 0)
1751 ##### 17 return error;
1752 - }
1753 -
1754 44 20 if (error == GIT_ITEROVER)
1755 44 21 error = 0;
1756 -
1757 44 22 return error;
1758 - }
1759 -
1760 36 2 int git_packbuilder_set_callbacks(git_packbuilder *pb, git_packbuilder_progress progress_cb, void *progress_cb_payload)
1761 - {
1762 36 2 if (!pb)
1763 ##### 3 return -1;
1764 -
1765 36 4 pb->progress_cb = progress_cb;
1766 36 4 pb->progress_cb_payload = progress_cb_payload;
1767 -
1768 36 4 return 0;
1769 - }
1770 -
1771 57 2 void git_packbuilder_free(git_packbuilder *pb)
1772 - {
1773 57 2 if (pb == NULL)
1774 57 3,18 return;
1775 -
1776 - #ifdef GIT_THREADS
1777 -
1778 56 4 git_mutex_free(&pb->cache_mutex);
1779 56 5 git_mutex_free(&pb->progress_mutex);
1780 56 6 git_cond_free(&pb->progress_cond);
1781 -
1782 - #endif
1783 -
1784 56 7 if (pb->odb)
1785 56 8 git_odb_free(pb->odb);
1786 -
1787 56 9 if (pb->object_ix)
1788 56 10 git_oidmap_free(pb->object_ix);
1789 -
1790 56 11 if (pb->object_list)
1791 53 12 git__free(pb->object_list);
1792 -
1793 56 13 git_oidmap_free(pb->walk_objects);
1794 56 14 git_pool_clear(&pb->object_pool);
1795 -
1796 56 15 git_hash_ctx_cleanup(&pb->ctx);
1797 56 16 git_zstream_free(&pb->zstream);
1798 -
1799 56 17 git__free(pb);
1800 - }