17      CurlMultiPartSetoptError ( CURLcode code ) : _code(code){};
 
   18      CURLcode code() 
const;
 
   23    CURLcode CurlMultiPartSetoptError::code()
 const {
 
   27    class CurlMultInitRangeError : 
public zypp::Exception
 
   30      CurlMultInitRangeError ( std::string what ) : zypp::Exception( std::move(what) ){}
 
   86    std::string chksumtype = ( digest ? digest->name() : std::string() );
 
   91        ._chksumtype = std::move(chksumtype),
 
   92        ._checksum = std::move( expectedChkSum ),
 
   93        ._relevantDigestLen = std::move( digestCompareLen ),
 
   94        ._chksumPad  = std::move( dataBlockPadding ),
 
 
  110      WAR << 
"!!!! Downloading ranges without HTTP might be slow !!!!" << std::endl;
 
 
  118      curl_easy_setopt( 
_easyHandle, CURLOPT_HEADERFUNCTION, 
nullptr );
 
  119      curl_easy_setopt( 
_easyHandle, CURLOPT_HEADERDATA, 
nullptr );
 
  120      curl_easy_setopt( 
_easyHandle, CURLOPT_WRITEFUNCTION, 
nullptr );
 
  121      curl_easy_setopt( 
_easyHandle, CURLOPT_WRITEDATA, 
nullptr );
 
 
  129    if ( size * nmemb == 0)
 
  130      return _receiver.headerfunction( ptr, size * nmemb );
 
  134      std::string_view hdr( ptr, size*nmemb );
 
  136      hdr.remove_prefix( std::min( hdr.find_first_not_of(
" \t\r\n"), hdr.size() ) );
 
  137      const auto lastNonWhitespace = hdr.find_last_not_of(
" \t\r\n");
 
  138      if ( lastNonWhitespace != hdr.npos )
 
  139        hdr.remove_suffix( hdr.size() - (lastNonWhitespace + 1) );
 
  141        hdr = std::string_view();
 
  145        return ( size * nmemb );
 
  148      if ( zypp::strv::hasPrefixCI( hdr, 
"HTTP/" ) ) {
 
  151        (void)curl_easy_getinfo( 
_easyHandle, CURLINFO_RESPONSE_CODE, &statuscode);
 
  153        const auto &doRangeFail = [&](){
 
  154          WAR << 
_easyHandle << 
" " << 
"Range FAIL, trying with a smaller batch" << std::endl;
 
  156          setCode ( Code::RangeFail, 
"Expected range status code 206, but got none." );
 
  160            if ( range._rangeState == Running )
 
  161              range._rangeState = Pending;
 
  167        if ( ( statuscode >= 200 && statuscode <= 299 && statuscode != 206 )
 
  168              || statuscode == 416 ) {
 
  169            return doRangeFail();
 
  172      } 
else if ( zypp::strv::hasPrefixCI( hdr, 
"Content-Range:") ) {
 
  178          setCode( Code::InternalError, 
"Invalid Content-Range header format." );
 
  183          WAR << 
"Setting request filesize to " << fileLen << std::endl;
 
  191      } 
else if ( zypp::strv::hasPrefixCI( hdr, 
"Content-Type:") ) {
 
  201    return _receiver.headerfunction( ptr, size * nmemb );
 
 
  206    const auto max = ( size * nmemb );
 
  210      return _receiver.writefunction( ptr, {}, max );
 
  213    size_t bytesConsumedSoFar = 0;
 
  214    while ( bytesConsumedSoFar != max ) {
 
  216      std::optional<off_t> seekTo;
 
  225          setCode( Code::ServerReturnedError, 
"Invalid data from server, range respone was announced but there was no range definiton." );
 
  243        std::optional<uint> foundRange;
 
  246        auto currDist  = ULONG_MAX;
 
  251          if ( currR._rangeState == 
Finished || currR._rangeState == 
Error )
 
  255          if ( currR._len && currR._len == currR.bytesWritten )
 
  258          const auto currRBegin = currR._start + currR.bytesWritten;
 
  259          if ( !( beginSrvRange <= currRBegin && endSrvRange >= currRBegin ) )
 
  263          const auto newDist   = currRBegin - beginSrvRange;
 
  266          if ( currRBegin == beginSrvRange && currR._len == 
_currentSrvRange->_len ) {
 
  276            if ( newDist < currDist ) {
 
  285          setCode( Code::InternalError, 
"Unable to find a matching range for data returned by the server.");
 
  296        const auto skipBytes = *seekTo - beginSrvRange;
 
  297        bytesConsumedSoFar += skipBytes;
 
  300        std::string errBuf = 
"Receiver cancelled starting the current range.";
 
  302          setCode( Code::InternalError, 
"Receiver cancelled starting the current range.");
 
  313            setCode( Code::InternalError, 
"Received data but no range was marked as requested." );
 
  330        auto availableData = max - bytesConsumedSoFar;
 
  338        const auto bytesToWrite = rng._len > 0 ? std::min( rng._len - rng.bytesWritten, availableData ) : availableData;
 
  340        auto written = 
_receiver.writefunction( ptr + bytesConsumedSoFar, seekTo, bytesToWrite );
 
  344        if ( rng._digest && rng._checksum.size() ) {
 
  345          if ( !rng._digest->update( ptr + bytesConsumedSoFar, written ) )
 
  349        rng.bytesWritten += written;
 
  353        if ( rng._len > 0 && rng.bytesWritten >= rng._len ) {
 
  354          std::string errBuf = 
"Receiver cancelled after finishing the current range.";
 
  357            setCode( Code::InternalError, 
"Receiver cancelled starting the current range.");
 
  371        bytesConsumedSoFar += written;
 
  374      if ( bytesConsumedSoFar == max )
 
  383        std::string_view incoming( ptr + bytesConsumedSoFar, max - bytesConsumedSoFar );
 
  384        auto hdrEnd = incoming.find(
"\r\n\r\n");
 
  385        if ( hdrEnd == incoming.npos ) {
 
  393        bytesConsumedSoFar += ( hdrEnd + 4 ); 
 
  397        if ( sepStrIndex == data.npos ) {
 
  398          setCode( Code::InternalError, 
"Invalid multirange header format, seperator string missing." );
 
  403        std::vector<std::string_view> lines;
 
  404        zypp::strv::split( data.substr( startOfHeader ), 
"\r\n", zypp::strv::Trim::trim, [&]( std::string_view 
strv ) { lines.push_back(strv); } );
 
  405        for ( 
const auto &hdrLine : lines ) {
 
  406          if ( zypp::strv::hasPrefixCI(hdrLine, 
"Content-Range:") ) {
 
  411              setCode( Code::InternalError, 
"Invalid Content-Range header format." );
 
  423          setCode( Code::InternalError, 
"Could not read a server range from the response." );
 
  432    return bytesConsumedSoFar;
 
 
  477          setCode ( Code::RangeFail, 
"No more range batch sizes available", 
true );
 
  485    setCode ( Code::NoError, 
"Request has no more work", 
true );
 
 
  508        if ( r._len > 0 && r.bytesWritten != r._len )
 
  509          setCode( Code::MissingData, (
zypp::str::Format(
"Did not receive all requested data from the server ( off: %1%, req: %2%, recv: %3% ).") % r._start % r._len % r.bytesWritten ) );
 
 
  534      setCode( Code::InternalError, 
"Calling the CurlMultiPartHandler::prepare function without a range to download is not supported.");
 
  538    const auto setCurlOption = [&]( CURLoption opt, 
auto &&data )
 
  540      auto ret = curl_easy_setopt( 
_easyHandle, opt, data );
 
  542        throw CurlMultiPartSetoptError(ret);
 
  548      setCurlOption( CURLOPT_HEADERDATA, 
this );
 
  550      setCurlOption( CURLOPT_WRITEDATA, 
this );
 
  552      std::string rangeDesc;
 
  553      uint rangesAdded = 0;
 
  557      auto addRangeString = [ &rangeDesc, &rangesAdded ]( 
const std::pair<size_t, size_t> &range ) {
 
  558        std::string rangeD = 
zypp::str::form(
"%llu-", 
static_cast<unsigned long long>( range.first ) );
 
  559        if( range.second > 0 )
 
  560          rangeD.append( 
zypp::str::form( 
"%llu", 
static_cast<unsigned long long>( range.second ) ) );
 
  562        if ( rangeDesc.size() )
 
  563          rangeDesc.append(
",").append( rangeD );
 
  565          rangeDesc = std::move( rangeD );
 
  570      std::optional<std::pair<size_t, size_t>> currentZippedRange;
 
  571      bool closedRange = 
true;
 
  574        if ( range._rangeState != 
Pending )
 
  578        range.bytesWritten = 0;
 
  583          throw CurlMultInitRangeError(
"It is not supported to request more ranges after a open range.");
 
  585        const auto rangeEnd = range._len > 0 ? range._start + range._len - 1 : 0;
 
  586        closedRange = (rangeEnd > 0);
 
  592        if ( !currentZippedRange ) {
 
  594          currentZippedRange = std::make_pair( range._start, rangeEnd );
 
  597          if ( currentZippedRange->second + 1 == range._start ) {
 
  599            currentZippedRange->second = rangeEnd;
 
  602            if ( rangesAdded +1 >= maxRanges ) 
break;
 
  604            addRangeString( *currentZippedRange );
 
  605            currentZippedRange = std::make_pair( range._start, rangeEnd );
 
  612          range.bytesWritten = 0;
 
  614            range._digest->reset();
 
  617        if ( rangesAdded >= maxRanges ) {
 
  618          MIL << 
_easyHandle << 
" " << 
"Reached max nr of ranges (" << maxRanges << 
"), batching the request to not break the server" << std::endl;
 
  624      if ( currentZippedRange )
 
  625        addRangeString( *currentZippedRange );
 
  627      MIL << 
_easyHandle << 
" " << 
"Requesting Ranges: " << rangeDesc << std::endl;
 
  629      setCurlOption( CURLOPT_RANGE, rangeDesc.c_str() );
 
  631    } 
catch( 
const CurlMultiPartSetoptError &err ) {
 
  632      setCode( Code::InternalError, 
"" );
 
  633    } 
catch( 
const CurlMultInitRangeError &err ) {
 
  634      setCode( Code::InternalError, err.asUserString() );
 
  636      setCode( Code::InternalError, err.asUserString() );
 
  637    } 
catch( 
const std::exception &err ) {
 
 
  646    if ( 
_lastCode != Code::NoError && !force )
 
 
  661      DBG << 
_easyHandle << 
" " << 
"Invalid Content-Range Header format: '" << std::string(line) << std::endl;
 
 
  679      if ( what.
size() >= 2 ) {
 
 
  706      auto bytesHashed = rng.
_digest->bytesHashed ();
 
  710        rng.
_digest->update( padding.data(), padding.size() );
 
  712      auto digVec = rng.
_digest->digestVector();
 
 
static std::string digestVectorToString(const UByteArray &vec)
get hex string representation of the digest vector given as parameter
Base class for Exception.
@ icase
Do not differentiate case.
@ rxdefault
These are enforced even if you don't pass them as flag argument.
Regular expression match result.
size_t wrtcallback(char *ptr, size_t size, size_t nmemb)
unsigned _rangeAttemptIdx
static constexpr unsigned _rangeAttemptSize
ProtocolMode _protocolMode
CurlMultiPartHandler(ProtocolMode mode, void *easyHandle, std::vector< Range > &ranges, CurlMultiPartDataReceiver &receiver)
std::optional< Range > _currentSrvRange
std::string _seperatorString
The seperator string for multipart responses as defined in RFC 7233 Section 4.1.
NetworkRequestError::Type Code
std::string _lastErrorMsg
bool _gotContentRangeInfo
static size_t curl_wrtcallback(char *ptr, size_t size, size_t nmemb, void *userdata)
static size_t curl_hdrcallback(char *ptr, size_t size, size_t nmemb, void *userdata)
~CurlMultiPartHandler() override
const std::string & lastErrorMessage() const
static constexpr unsigned _rangeAttempt[]
bool parseContentRangeHeader(const std::string_view &line, size_t &start, size_t &len, size_t &fileLen)
void setCode(Code c, std::string msg, bool force=false)
std::vector< char > _rangePrefaceBuffer
Here we buffer.
std::optional< off_t > currentRange() const
std::optional< size_t > _reportedFileSize
Filesize as reported by the content range or byte range headers.
size_t hdrcallback(char *ptr, size_t size, size_t nmemb)
bool validateRange(Range &rng)
CurlMultiPartDataReceiver & _receiver
void setRangeState(Range &rng, State state)
std::optional< size_t > reportedFileSize() const
bool parseContentTypeMultiRangeHeader(const std::string_view &line, std::string &boundary)
void * easyHandle() const
bool checkIfRangeChkSumIsValid(Range &rng)
std::vector< Range > & _requestedRanges
the requested ranges that need to be downloaded
std::optional< off_t > _currentRange
const std::string & asString(const std::string &t)
Global asString() that works with std::string too.
bool regex_match(const std::string &s, smatch &matches, const regex ®ex)
\relates regex \ingroup ZYPP_STR_REGEX    \relates regex \ingroup ZYPP_STR_REGEX
std::string form(const char *format,...) __attribute__((format(printf
Printf style construction of std::string.
TInt strtonum(const C_Str &str)
Parsing numbers from string.
std::optional< zypp::Digest > _digest
Enables automated checking of downloaded contents against a checksum.
static Range make(size_t start, size_t len=0, std::optional< zypp::Digest > &&digest={}, CheckSumBytes &&expectedChkSum=CheckSumBytes(), std::any &&userData=std::any(), std::optional< size_t > digestCompareLen={}, std::optional< size_t > _dataBlockPadding={})
std::string _chksumtype
Enables automated checking of downloaded contents against a checksum.
std::optional< size_t > _relevantDigestLen
std::optional< size_t > _chksumPad