From 45271d06c07ea1d08f916f3f69dfdc3399efd6e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Czarnecki?= Date: Wed, 1 Jun 2022 11:26:56 -0700 Subject: [PATCH 1/2] Get libgit2 working for 8thwall. (#24) Upgrade libgit2 with support for 8thwall cloud editor. --- deps/picosha2/picosha2-c.cc | 17 ++ deps/picosha2/picosha2-c.h | 14 + deps/picosha2/picosha2.h | 370 +++++++++++++++++++++++ include/git2/sys/transport.h | 10 +- src/libgit2/refdb_fs.c | 2 +- src/libgit2/transports/emscriptenhttp.cc | 269 ++++++++++++++++ src/libgit2/transports/http.c | 2 +- src/util/thread.c | 8 + src/util/vector.h | 2 +- 9 files changed, 690 insertions(+), 4 deletions(-) create mode 100644 deps/picosha2/picosha2-c.cc create mode 100644 deps/picosha2/picosha2-c.h create mode 100644 deps/picosha2/picosha2.h create mode 100644 src/libgit2/transports/emscriptenhttp.cc diff --git a/deps/picosha2/picosha2-c.cc b/deps/picosha2/picosha2-c.cc new file mode 100644 index 00000000000..847d1150001 --- /dev/null +++ b/deps/picosha2/picosha2-c.cc @@ -0,0 +1,17 @@ +#include "picosha2-c.h" +#include "picosha2.h" + +#include +#include +#include + +extern "C" { + +void picosha2_256(const char *buffer, int len, char *dest) { + std::array hash; + picosha2::hash256(buffer, buffer + len, std::begin(hash), std::end(hash)); + std::string sha256 = + picosha2::bytes_to_hex_string(std::begin(hash), std::end(hash)); + std::memcpy(dest, sha256.c_str(), sha256.size() + 1); +} +} diff --git a/deps/picosha2/picosha2-c.h b/deps/picosha2/picosha2-c.h new file mode 100644 index 00000000000..3bc1dcb1a9b --- /dev/null +++ b/deps/picosha2/picosha2-c.h @@ -0,0 +1,14 @@ +#ifndef INCLUDE_picosha2_c_h__ +#define INCLUDE_picosha2_c_h__ + +#ifdef __cplusplus +extern "C" { +#endif + +void picosha2_256(const char* buffer, int len, char* dest); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/deps/picosha2/picosha2.h b/deps/picosha2/picosha2.h new file mode 100644 index 00000000000..2fcfea43014 --- /dev/null +++ b/deps/picosha2/picosha2.h @@ -0,0 +1,370 @@ +/* +The MIT License (MIT) +Copyright (C) 2017 okdshin +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ +#pragma once +// picosha2:20140213 + +#ifndef PICOSHA2_BUFFER_SIZE_FOR_INPUT_ITERATOR +#define PICOSHA2_BUFFER_SIZE_FOR_INPUT_ITERATOR \ + 1048576 //=1024*1024: default is 1MB memory +#endif + +#include +#include +#include +#include +#include +#include +namespace picosha2 { +typedef unsigned long word_t; +typedef unsigned char byte_t; + +static const size_t k_digest_size = 32; + +namespace detail { +inline byte_t mask_8bit(byte_t x) { return x & 0xff; } + +inline word_t mask_32bit(word_t x) { return x & 0xffffffff; } + +const word_t add_constant[64] = { + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, + 0x923f82a4, 0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, + 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786, + 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, + 0x06ca6351, 0x14292967, 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, + 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, 0xa2bfe8a1, 0xa81a664b, + 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, + 0x5b9cca4f, 0x682e6ff3, 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, + 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2}; + +const word_t initial_message_digest[8] = {0x6a09e667, 0xbb67ae85, 0x3c6ef372, + 0xa54ff53a, 0x510e527f, 0x9b05688c, + 0x1f83d9ab, 0x5be0cd19}; + +inline word_t ch(word_t x, word_t y, word_t z) { return (x & y) ^ ((~x) & z); } + +inline word_t maj(word_t x, word_t y, word_t z) { + return (x & y) ^ (x & z) ^ (y & z); +} + +inline word_t rotr(word_t x, std::size_t n) { + assert(n < 32); + return mask_32bit((x >> n) | (x << (32 - n))); +} + +inline word_t bsig0(word_t x) { return rotr(x, 2) ^ rotr(x, 13) ^ rotr(x, 22); } + +inline word_t bsig1(word_t x) { return rotr(x, 6) ^ rotr(x, 11) ^ rotr(x, 25); } + +inline word_t shr(word_t x, std::size_t n) { + assert(n < 32); + return x >> n; +} + +inline word_t ssig0(word_t x) { return rotr(x, 7) ^ rotr(x, 18) ^ shr(x, 3); } + +inline word_t ssig1(word_t x) { return rotr(x, 17) ^ rotr(x, 19) ^ shr(x, 10); } + +template +void hash256_block(RaIter1 message_digest, RaIter2 first, RaIter2 last) { + assert(first + 64 == last); + static_cast(last); // for avoiding unused-variable warning + word_t w[64]; + std::fill(w, w + 64, 0); + for (std::size_t i = 0; i < 16; ++i) { + w[i] = (static_cast(mask_8bit(*(first + i * 4))) << 24) | + (static_cast(mask_8bit(*(first + i * 4 + 1))) << 16) | + (static_cast(mask_8bit(*(first + i * 4 + 2))) << 8) | + (static_cast(mask_8bit(*(first + i * 4 + 3)))); + } + for (std::size_t i = 16; i < 64; ++i) { + w[i] = + mask_32bit(ssig1(w[i - 2]) + w[i - 7] + ssig0(w[i - 15]) + w[i - 16]); + } + + word_t a = *message_digest; + word_t b = *(message_digest + 1); + word_t c = *(message_digest + 2); + word_t d = *(message_digest + 3); + word_t e = *(message_digest + 4); + word_t f = *(message_digest + 5); + word_t g = *(message_digest + 6); + word_t h = *(message_digest + 7); + + for (std::size_t i = 0; i < 64; ++i) { + word_t temp1 = h + bsig1(e) + ch(e, f, g) + add_constant[i] + w[i]; + word_t temp2 = bsig0(a) + maj(a, b, c); + h = g; + g = f; + f = e; + e = mask_32bit(d + temp1); + d = c; + c = b; + b = a; + a = mask_32bit(temp1 + temp2); + } + *message_digest += a; + *(message_digest + 1) += b; + *(message_digest + 2) += c; + *(message_digest + 3) += d; + *(message_digest + 4) += e; + *(message_digest + 5) += f; + *(message_digest + 6) += g; + *(message_digest + 7) += h; + for (std::size_t i = 0; i < 8; ++i) { + *(message_digest + i) = mask_32bit(*(message_digest + i)); + } +} + +} // namespace detail + +template +void output_hex(InIter first, InIter last, std::ostream &os) { + os.setf(std::ios::hex, std::ios::basefield); + while (first != last) { + os.width(2); + os.fill('0'); + os << static_cast(*first); + ++first; + } + os.setf(std::ios::dec, std::ios::basefield); +} + +template +void bytes_to_hex_string(InIter first, InIter last, std::string &hex_str) { + std::ostringstream oss; + output_hex(first, last, oss); + hex_str.assign(oss.str()); +} + +template +void bytes_to_hex_string(const InContainer &bytes, std::string &hex_str) { + bytes_to_hex_string(bytes.begin(), bytes.end(), hex_str); +} + +template +std::string bytes_to_hex_string(InIter first, InIter last) { + std::string hex_str; + bytes_to_hex_string(first, last, hex_str); + return hex_str; +} + +template +std::string bytes_to_hex_string(const InContainer &bytes) { + std::string hex_str; + bytes_to_hex_string(bytes, hex_str); + return hex_str; +} + +class hash256_one_by_one { +public: + hash256_one_by_one() { init(); } + + void init() { + buffer_.clear(); + std::fill(data_length_digits_, data_length_digits_ + 4, 0); + std::copy(detail::initial_message_digest, + detail::initial_message_digest + 8, h_); + } + + template void process(RaIter first, RaIter last) { + add_to_data_length(static_cast(std::distance(first, last))); + std::copy(first, last, std::back_inserter(buffer_)); + std::size_t i = 0; + for (; i + 64 <= buffer_.size(); i += 64) { + detail::hash256_block(h_, buffer_.begin() + i, buffer_.begin() + i + 64); + } + buffer_.erase(buffer_.begin(), buffer_.begin() + i); + } + + void finish() { + byte_t temp[64]; + std::fill(temp, temp + 64, 0); + std::size_t remains = buffer_.size(); + std::copy(buffer_.begin(), buffer_.end(), temp); + temp[remains] = 0x80; + + if (remains > 55) { + std::fill(temp + remains + 1, temp + 64, 0); + detail::hash256_block(h_, temp, temp + 64); + std::fill(temp, temp + 64 - 4, 0); + } else { + std::fill(temp + remains + 1, temp + 64 - 4, 0); + } + + write_data_bit_length(&(temp[56])); + detail::hash256_block(h_, temp, temp + 64); + } + + template + void get_hash_bytes(OutIter first, OutIter last) const { + for (const word_t *iter = h_; iter != h_ + 8; ++iter) { + for (std::size_t i = 0; i < 4 && first != last; ++i) { + *(first++) = + detail::mask_8bit(static_cast((*iter >> (24 - 8 * i)))); + } + } + } + +private: + void add_to_data_length(word_t n) { + word_t carry = 0; + data_length_digits_[0] += n; + for (std::size_t i = 0; i < 4; ++i) { + data_length_digits_[i] += carry; + if (data_length_digits_[i] >= 65536u) { + carry = data_length_digits_[i] >> 16; + data_length_digits_[i] &= 65535u; + } else { + break; + } + } + } + void write_data_bit_length(byte_t *begin) { + word_t data_bit_length_digits[4]; + std::copy(data_length_digits_, data_length_digits_ + 4, + data_bit_length_digits); + + // convert byte length to bit length (multiply 8 or shift 3 times left) + word_t carry = 0; + for (std::size_t i = 0; i < 4; ++i) { + word_t before_val = data_bit_length_digits[i]; + data_bit_length_digits[i] <<= 3; + data_bit_length_digits[i] |= carry; + data_bit_length_digits[i] &= 65535u; + carry = (before_val >> (16 - 3)) & 65535u; + } + + // write data_bit_length + for (int i = 3; i >= 0; --i) { + (*begin++) = static_cast(data_bit_length_digits[i] >> 8); + (*begin++) = static_cast(data_bit_length_digits[i]); + } + } + std::vector buffer_; + word_t data_length_digits_[4]; // as 64bit integer (16bit x 4 integer) + word_t h_[8]; +}; + +inline void get_hash_hex_string(const hash256_one_by_one &hasher, + std::string &hex_str) { + byte_t hash[k_digest_size]; + hasher.get_hash_bytes(hash, hash + k_digest_size); + return bytes_to_hex_string(hash, hash + k_digest_size, hex_str); +} + +inline std::string get_hash_hex_string(const hash256_one_by_one &hasher) { + std::string hex_str; + get_hash_hex_string(hasher, hex_str); + return hex_str; +} + +namespace impl { +template +void hash256_impl(RaIter first, RaIter last, OutIter first2, OutIter last2, int, + std::random_access_iterator_tag) { + hash256_one_by_one hasher; + // hasher.init(); + hasher.process(first, last); + hasher.finish(); + hasher.get_hash_bytes(first2, last2); +} + +template +void hash256_impl(InputIter first, InputIter last, OutIter first2, + OutIter last2, int buffer_size, std::input_iterator_tag) { + std::vector buffer(buffer_size); + hash256_one_by_one hasher; + // hasher.init(); + while (first != last) { + int size = buffer_size; + for (int i = 0; i != buffer_size; ++i, ++first) { + if (first == last) { + size = i; + break; + } + buffer[i] = *first; + } + hasher.process(buffer.begin(), buffer.begin() + size); + } + hasher.finish(); + hasher.get_hash_bytes(first2, last2); +} +} // namespace impl + +template +void hash256(InIter first, InIter last, OutIter first2, OutIter last2, + int buffer_size = PICOSHA2_BUFFER_SIZE_FOR_INPUT_ITERATOR) { + picosha2::impl::hash256_impl( + first, last, first2, last2, buffer_size, + typename std::iterator_traits::iterator_category()); +} + +template +void hash256(InIter first, InIter last, OutContainer &dst) { + hash256(first, last, dst.begin(), dst.end()); +} + +template +void hash256(const InContainer &src, OutIter first, OutIter last) { + hash256(src.begin(), src.end(), first, last); +} + +template +void hash256(const InContainer &src, OutContainer &dst) { + hash256(src.begin(), src.end(), dst.begin(), dst.end()); +} + +template +void hash256_hex_string(InIter first, InIter last, std::string &hex_str) { + byte_t hashed[k_digest_size]; + hash256(first, last, hashed, hashed + k_digest_size); + std::ostringstream oss; + output_hex(hashed, hashed + k_digest_size, oss); + hex_str.assign(oss.str()); +} + +template +std::string hash256_hex_string(InIter first, InIter last) { + std::string hex_str; + hash256_hex_string(first, last, hex_str); + return hex_str; +} + +inline void hash256_hex_string(const std::string &src, std::string &hex_str) { + hash256_hex_string(src.begin(), src.end(), hex_str); +} + +template +void hash256_hex_string(const InContainer &src, std::string &hex_str) { + hash256_hex_string(src.begin(), src.end(), hex_str); +} + +template +std::string hash256_hex_string(const InContainer &src) { + return hash256_hex_string(src.begin(), src.end()); +} +template +void hash256(std::ifstream &f, OutIter first, OutIter last) { + hash256(std::istreambuf_iterator(f), std::istreambuf_iterator(), + first, last); +} +} // namespace picosha2 diff --git a/include/git2/sys/transport.h b/include/git2/sys/transport.h index 06ae7079ffd..930c281e23a 100644 --- a/include/git2/sys/transport.h +++ b/include/git2/sys/transport.h @@ -8,6 +8,10 @@ #ifndef INCLUDE_sys_git_transport_h #define INCLUDE_sys_git_transport_h +#ifdef __cplusplus +extern "C" { +#endif + #include "git2/net.h" #include "git2/proxy.h" #include "git2/remote.h" @@ -401,11 +405,11 @@ typedef struct git_smart_subtransport_definition { * @param owner The smart transport to own this subtransport * @return 0 or an error code */ + GIT_EXTERN(int) git_smart_subtransport_http( git_smart_subtransport **out, git_transport *owner, void *param); - /** * Create an instance of the git subtransport. * @@ -432,4 +436,8 @@ GIT_EXTERN(int) git_smart_subtransport_ssh( /** @} */ GIT_END_DECL + +#ifdef __cplusplus +} +#endif #endif diff --git a/src/libgit2/refdb_fs.c b/src/libgit2/refdb_fs.c index 43283b3e472..11ad8694664 100644 --- a/src/libgit2/refdb_fs.c +++ b/src/libgit2/refdb_fs.c @@ -1634,7 +1634,7 @@ static int refdb_fs_backend__prune_refs( error = git_futils_rmdir_r(ref_name + commonlen, git_str_cstr(&base_path), - GIT_RMDIR_EMPTY_PARENTS | GIT_RMDIR_SKIP_ROOT); + GIT_RMDIR_EMPTY_PARENTS | GIT_RMDIR_SKIP_ROOT | GIT_RMDIR_REMOVE_FILES); if (error == GIT_ENOTFOUND) error = 0; diff --git a/src/libgit2/transports/emscriptenhttp.cc b/src/libgit2/transports/emscriptenhttp.cc new file mode 100644 index 00000000000..7f344fa9335 --- /dev/null +++ b/src/libgit2/transports/emscriptenhttp.cc @@ -0,0 +1,269 @@ +#ifdef __EMSCRIPTEN__ + +#include "emscripten.h" +#include "emscripten/fetch.h" + +extern "C" { +#include "common.h" +#include "git2/transport.h" +#include "smart.h" +} + +#include + +#include "deps/picosha2/picosha2.h" + +static const char *upload_pack_ls_service_url = "/info/refs?service=git-upload-pack"; +static const char *upload_pack_service_url = "/git-upload-pack"; +static const char *receive_pack_ls_service_url = "/info/refs?service=git-receive-pack"; +static const char *receive_pack_service_url = "/git-receive-pack"; + +namespace { +struct StreamInternal { + std::string url; + std::vector writeBuffer; + std::vector headers; // [key1, value1, key2, value2, ...] + emscripten_fetch_attr_t attr; + emscripten_fetch_t *fetch{nullptr}; + size_t totalBytesRead{0}; +}; + +uint64_t connectionCount{0}; +std::map connectionMap; + +uint64_t xhrConnect( + const std::string url, const char *method, const std::vector headers) { + auto connectionNumber = connectionCount++; + auto &connection = connectionMap[connectionNumber]; + connection.url = url; + connection.headers = headers; + emscripten_fetch_attr_init(&connection.attr); + strcpy(connection.attr.requestMethod, method); + // NOTE(pawel) EMSCRIPTEN_FETCH_REPLACE is needed for synchronous to work... + // https://github.com/emscripten-core/emscripten/issues/8183 + connection.attr.attributes = + EMSCRIPTEN_FETCH_LOAD_TO_MEMORY | EMSCRIPTEN_FETCH_SYNCHRONOUS | EMSCRIPTEN_FETCH_REPLACE; + if (std::string(method) == "GET") { + auto headersToSend = connection.headers; + headersToSend.push_back(0); + connection.attr.requestHeaders = headersToSend.data(); + connection.fetch = emscripten_fetch(&connection.attr, url.c_str()); + } + return connectionNumber; +} + +// This call buffers the writes. The buffer is sent on the wire when xhrRead() is called. +int xhrWrite(uint64_t connectionNumber, const char *buffer, size_t size) { + if (connectionMap.count(connectionNumber) != 1) { + printf("Attempting to write to connection %l but it is not connected", connectionNumber); + return -1; + } + auto &connection = connectionMap[connectionNumber]; + connection.writeBuffer.insert(end(connection.writeBuffer), buffer, buffer + size); + return 0; +} + +// Sends pending writes and returns response async. The result is buffered so this can be invoked +// until the full length of the buffer is read. +int xhrRead(uint64_t connectionNumber, char *buffer, size_t bufferSize, size_t *bytesRead) { + if (connectionMap.count(connectionNumber) != 1) { + printf("Attempting to read from connection %l but it is not connected\n", connectionNumber); + *bytesRead = 0; + return -1; + } + auto &connection = connectionMap[connectionNumber]; + std::vector headersToSend = connection.headers; + std::string sha256; + if (!connection.writeBuffer.empty()) { + const auto data = connection.writeBuffer.data(); + const auto dataSize = connection.writeBuffer.size(); + connection.attr.requestData = data; + connection.attr.requestDataSize = dataSize; + + std::vector hash(picosha2::k_digest_size); + picosha2::hash256(data, data + dataSize, hash.begin(), hash.end()); + sha256 = picosha2::bytes_to_hex_string(hash.begin(), hash.end()); + headersToSend.push_back("x-amz-content-sha256"); + headersToSend.push_back(sha256.c_str()); // This pointer is only valid in this scope. + } + auto &f = connection.fetch; + if (!f) { + headersToSend.push_back(0); // null terminate the array. + connection.attr.requestHeaders = headersToSend.data(); + + f = emscripten_fetch(&connection.attr, connection.url.c_str()); + if (f->status != 200) { + printf("%d %s %s\n", f->status, f->statusText, f->url); + } + if (f->readyState != 4) { + printf("Connection is not in ready state (%d)\n", f->readyState); + } + } + connection.writeBuffer.clear(); + *bytesRead = min(f->numBytes - connection.totalBytesRead, bufferSize); + std::memcpy(buffer, f->data + connection.totalBytesRead, *bytesRead); + connection.totalBytesRead += *bytesRead; + return 0; +} + +void xhrFree(uint64_t connectionNumber) { + if (connectionMap.count(connectionNumber != 1)) { + printf("Attempting to free unkown connection %l", connectionNumber); + return; + } + auto &connection = connectionMap[connectionNumber]; + emscripten_fetch_close(connection.fetch); + connectionMap.erase(connectionNumber); +} +} // namespace + +namespace { +typedef struct { + git_smart_subtransport_stream parent; + git_str service_url; + uint64_t connectionNo; +} emscriptenhttp_stream; + +typedef struct { + git_smart_subtransport parent; + transport_smart *owner; +} emscriptenhttp_subtransport; + +// Since these types are being sent back to C, we need to ensure they are PODs. +static_assert(std::is_pod()); +static_assert(std::is_pod()); + +static int emscriptenhttp_stream_read( + git_smart_subtransport_stream *stream, char *buffer, size_t buf_size, size_t *bytes_read) { + emscriptenhttp_stream *s = (emscriptenhttp_stream *)stream; + + if (s->connectionNo == -1) { + s->connectionNo = xhrConnect(git_str_cstr(&s->service_url), "GET", {}); + } + return xhrRead(s->connectionNo, buffer, buf_size, bytes_read); +} + +static int emscriptenhttp_stream_write_single( + git_smart_subtransport_stream *stream, const char *buffer, size_t len) { + emscriptenhttp_stream *s = (emscriptenhttp_stream *)stream; + + if (s->connectionNo == -1) { + + auto serviceUrl = git_str_cstr(&s->service_url); + bool uploadPack = strstr(serviceUrl, "git-upload-pack") != 0; + + s->connectionNo = xhrConnect( + serviceUrl, + "POST", + { + "Content-Type", + uploadPack ? "application/x-git-upload-pack-request" + : "application/x-git-receive-pack-request", + "Pragma", + "no-cache", + }); + } + return xhrWrite(s->connectionNo, buffer, len); +} + +static void emscriptenhttp_stream_free(git_smart_subtransport_stream *stream) { + emscriptenhttp_stream *s = (emscriptenhttp_stream *)stream; + if (s->connectionNo != -1) { + xhrFree(s->connectionNo); + } + git_str_dispose(&s->service_url); + git__free(s); +} + +static int emscriptenhttp_stream_alloc( + emscriptenhttp_subtransport *t, emscriptenhttp_stream **stream) { + emscriptenhttp_stream *s; + + if (!stream) + return -1; + + s = reinterpret_cast(git__calloc(1, sizeof(emscriptenhttp_stream))); + GIT_ERROR_CHECK_ALLOC(s); + + s->parent.subtransport = &t->parent; + s->parent.read = emscriptenhttp_stream_read; + s->parent.write = emscriptenhttp_stream_write_single; + s->parent.free = emscriptenhttp_stream_free; + s->connectionNo = -1; + s->service_url = GIT_STR_INIT; + + *stream = s; + + return 0; +} + +static int emscriptenhttp_action( + git_smart_subtransport_stream **stream, + git_smart_subtransport *subtransport, + const char *url, + git_smart_service_t action) { + emscriptenhttp_subtransport *t = (emscriptenhttp_subtransport *)subtransport; + emscriptenhttp_stream *s; + + if (emscriptenhttp_stream_alloc(t, &s) < 0) { + return -1; + } + + switch (action) { + case GIT_SERVICE_UPLOADPACK_LS: + git_str_printf(&s->service_url, "%s%s", url, upload_pack_ls_service_url); + break; + case GIT_SERVICE_UPLOADPACK: + git_str_printf(&s->service_url, "%s%s", url, upload_pack_service_url); + break; + case GIT_SERVICE_RECEIVEPACK_LS: + git_str_printf(&s->service_url, "%s%s", url, receive_pack_ls_service_url); + break; + case GIT_SERVICE_RECEIVEPACK: + git_str_printf(&s->service_url, "%s%s", url, receive_pack_service_url); + break; + } + + if (git_str_oom(&s->service_url)) { + return -1; + } + + *stream = &s->parent; + return 0; +} + +static int emscriptenhttp_close(git_smart_subtransport *subtransport) { return 0; } + +static void emscriptenhttp_free(git_smart_subtransport *subtransport) { + emscriptenhttp_subtransport *t = (emscriptenhttp_subtransport *)subtransport; + emscriptenhttp_close(subtransport); + git__free(t); +} + +} // namespace + +extern "C" { +int git_smart_subtransport_http(git_smart_subtransport **out, git_transport *owner, void *param) { + emscriptenhttp_subtransport *t; + + GIT_UNUSED(param); + + if (!out) + return -1; + + t = reinterpret_cast( + git__calloc(1, sizeof(emscriptenhttp_subtransport))); + GIT_ERROR_CHECK_ALLOC(t); + + t->owner = (transport_smart *)owner; + t->parent.action = emscriptenhttp_action; + t->parent.close = emscriptenhttp_close; + t->parent.free = emscriptenhttp_free; + + *out = (git_smart_subtransport *)t; + + return 0; +} +} + +#endif /* __EMSCRIPTEN__ */ diff --git a/src/libgit2/transports/http.c b/src/libgit2/transports/http.c index 7db5582cab4..d096da03fb4 100644 --- a/src/libgit2/transports/http.c +++ b/src/libgit2/transports/http.c @@ -7,7 +7,7 @@ #include "common.h" -#ifndef GIT_WINHTTP +#if !defined(GIT_WINHTTP) && !defined(__EMSCRIPTEN__) #include "http_parser.h" #include "net.h" diff --git a/src/util/thread.c b/src/util/thread.c index bc7364f8c1a..c3f953cab03 100644 --- a/src/util/thread.c +++ b/src/util/thread.c @@ -67,6 +67,14 @@ int git_tlsdata_dispose(git_tlsdata_key key) if (value && destroy_fn) destroy_fn(value); + // NOTE(pawel) In a single-threaded environment this should be okay but multithreaded + // setups may need some additional verification (my intuition is there is an edge case here). + if (key == (tlsdata_cnt - 1)) { + tlsdata_cnt--; + } else { + printf("warning git_tlsdata_dispose() key (%d) != tlsdata_cnt (%d)", key, tlsdata_cnt); + } + return 0; } diff --git a/src/util/vector.h b/src/util/vector.h index e50cdfefcbd..343d821d83f 100644 --- a/src/util/vector.h +++ b/src/util/vector.h @@ -84,7 +84,7 @@ GIT_INLINE(void *) git_vector_last(const git_vector *v) int git_vector_insert(git_vector *v, void *element); int git_vector_insert_sorted(git_vector *v, void *element, - int (*on_dup)(void **old, void *new)); + int (*on_dup)(void **old, void *new_)); int git_vector_remove(git_vector *v, size_t idx); void git_vector_pop(git_vector *v); void git_vector_uniq(git_vector *v, void (*git_free_cb)(void *)); From 6cc6dc2ab48775aebc3b6ec4d257b574beff63e1 Mon Sep 17 00:00:00 2001 From: Pawel Czarnecki Date: Thu, 20 Oct 2022 09:58:44 -0700 Subject: [PATCH 2/2] [experimental] parallel lfs --- src/libgit2/checkout.c | 71 ++++++++++++++++++++++++++++++++++++----- src/libgit2/parallel.cc | 41 ++++++++++++++++++++++++ src/libgit2/parallel.h | 19 +++++++++++ 3 files changed, 123 insertions(+), 8 deletions(-) create mode 100644 src/libgit2/parallel.cc create mode 100644 src/libgit2/parallel.h diff --git a/src/libgit2/checkout.c b/src/libgit2/checkout.c index 6a4643196b0..3edef6a020a 100644 --- a/src/libgit2/checkout.c +++ b/src/libgit2/checkout.c @@ -6,6 +6,7 @@ */ #include "checkout.h" +#include "parallel.h" #include "git2/repository.h" #include "git2/refs.h" @@ -1509,7 +1510,8 @@ static int blob_content_to_file( git_blob *blob, const char *path, const char *hint_path, - mode_t entry_filemode) + mode_t entry_filemode, + git_mutex *mutex) { int flags = data->opts.file_open_flags; mode_t file_mode = data->opts.file_mode ? @@ -1556,6 +1558,7 @@ static int blob_content_to_file( writer.fd = fd; writer.open = 1; + // This is the thing that needs to get parallelized. error = git_filter_list_stream_blob(fl, blob, &writer.base); GIT_ASSERT(writer.open == 0); @@ -1740,7 +1743,8 @@ static int checkout_write_content( const char *full_path, const char *hint_path, unsigned int mode, - struct stat *st) + struct stat *st, + git_mutex *mutex) { int error = 0; git_blob *blob; @@ -1751,7 +1755,7 @@ static int checkout_write_content( if (S_ISLNK(mode)) error = blob_content_to_link(data, st, blob, full_path); else - error = blob_content_to_file(data, st, blob, full_path, hint_path, mode); + error = blob_content_to_file(data, st, blob, full_path, hint_path, mode, mutex); git_blob_free(blob); @@ -1771,7 +1775,8 @@ static int checkout_write_content( static int checkout_blob( checkout_data *data, - const git_diff_file *file) + const git_diff_file *file, + git_mutex *mutex) { git_str *fullpath; struct stat st; @@ -1789,7 +1794,7 @@ static int checkout_blob( } error = checkout_write_content( - data, &file->id, fullpath->ptr, file->path, file->mode, &st); + data, &file->id, fullpath->ptr, file->path, file->mode, &st, mutex); /* update the index unless prevented */ if (!error && (data->strategy & GIT_CHECKOUT_DONT_UPDATE_INDEX) == 0) @@ -1861,6 +1866,30 @@ static int checkout_remove_the_old( return 0; } + +typedef struct { + checkout_data *data; + const git_diff_file *file; + git_mutex *mutex; +} parallel_params; + +static int checkout__create_the_new_parallel(parallel_params *params) { + int error = 0; + if ((error = git_mutex_lock(params->mutex)) != 0) { + return error; + } + error = checkout_blob(params->data, params->file, params->mutex); + if (error < 0) { + // Clean up + } else { + params->data->completed_steps++; + report_progress(params->data, params->file->path); + } + git_mutex_unlock(params->mutex); + free(params); + return error; +} + static int checkout_create_the_new( unsigned int *actions, checkout_data *data) @@ -1869,18 +1898,44 @@ static int checkout_create_the_new( git_diff_delta *delta; size_t i; +#ifdef C8_PARALLEL_CHECKOUT + + Parallel *parallel = initParallel(16); + git_mutex mutex; + git_mutex_init(&mutex); + git_vector_foreach(&data->diff->deltas, i, delta) { if (actions[i] & CHECKOUT_ACTION__UPDATE_BLOB && !S_ISLNK(delta->new_file.mode)) { - if ((error = checkout_blob(data, &delta->new_file)) < 0) + parallel_params *params = malloc(sizeof(parallel_params)); + params->data = data; + params->file = &delta->new_file, + params->mutex = &mutex; + scheduleParallel(parallel, checkout__create_the_new_parallel, params); + } + } + + error = runParallel(parallel); + git_mutex_free(&mutex); + freeParallel(parallel); + + if (error < 0) { + return error; + } + +#else + git_vector_foreach(&data->diff->deltas, i, delta) { + if (actions[i] & CHECKOUT_ACTION__UPDATE_BLOB && !S_ISLNK(delta->new_file.mode)) { + if ((error = checkout_blob(data, &delta->new_file, NULL)) < 0) return error; data->completed_steps++; report_progress(data, delta->new_file.path); } } +#endif git_vector_foreach(&data->diff->deltas, i, delta) { if (actions[i] & CHECKOUT_ACTION__UPDATE_BLOB && S_ISLNK(delta->new_file.mode)) { - if ((error = checkout_blob(data, &delta->new_file)) < 0) + if ((error = checkout_blob(data, &delta->new_file, NULL)) < 0) return error; data->completed_steps++; report_progress(data, delta->new_file.path); @@ -2008,7 +2063,7 @@ static int checkout_write_entry( if (!S_ISGITLINK(side->mode)) return checkout_write_content(data, - &side->id, fullpath->ptr, hint_path, side->mode, &st); + &side->id, fullpath->ptr, hint_path, side->mode, &st, NULL); return 0; } diff --git a/src/libgit2/parallel.cc b/src/libgit2/parallel.cc new file mode 100644 index 00000000000..ba94caa3e3e --- /dev/null +++ b/src/libgit2/parallel.cc @@ -0,0 +1,41 @@ +#include "parallel.h" + +#include + +#include "c8/task-queue.h" +#include "c8/thread-pool.h" + +struct Parallel { + c8::ThreadPool threadPool; + c8::TaskQueue taskQueue; + std::atomic firstError{0}; + + explicit Parallel(int numThreads) : threadPool(numThreads) {} +}; + +extern "C" { + + Parallel *initParallel(int numThreads) { + return new Parallel(numThreads); + } + + void freeParallel(Parallel *parallel) { + delete parallel; + } + + void scheduleParallel(Parallel *parallel, int (*fn)(void *), void *data) { + parallel->taskQueue.addTask([=] { + int res = fn(data); + int expected = 0; + parallel->firstError.compare_exchange_strong( + expected, + res, + std::memory_order_relaxed); + }); + } + + int runParallel(Parallel *parallel) { + parallel->taskQueue.executeWithThreadPool(¶llel->threadPool); + return parallel->firstError; + } +} diff --git a/src/libgit2/parallel.h b/src/libgit2/parallel.h new file mode 100644 index 00000000000..3a31c2f8d65 --- /dev/null +++ b/src/libgit2/parallel.h @@ -0,0 +1,19 @@ +#pragma once + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct Parallel Parallel; + +Parallel *initParallel(int numThreads); + +void freeParallel(Parallel *); + +void scheduleParallel(Parallel*, int (*fn)(void *), void *data); + +int runParallel(Parallel *); + +#ifdef __cplusplus +} // Closes the extern "C" +#endif