libzypp: zypp-curl/ng/network/curlmultiparthandler.cc Source File

libzypp 17.38.14

curlmultiparthandler.cc
Go to the documentation of this file.
1#include "curlmultiparthandler.h"
2
3#include <zypp-curl/ng/network/private/mediadebug_p.h>
4#include <zypp-core/base/StringV.h>
5
6#include <curl/curl.h>
7
8#include <utility>
9
10namespace zyppng {
11
12 namespace {
13
14 class CurlMultiPartSetoptError : public zypp::Exception
15 {
16 public:
17 CurlMultiPartSetoptError ( CURLcode code ) : _code(code){};
18 CURLcode code() const;
19 private:
20 CURLcode _code;
21 };
22
23 CURLcode CurlMultiPartSetoptError::code() const {
24 return _code;
25 }
26
27 class CurlMultInitRangeError : public zypp::Exception
28 {
29 public:
30 CurlMultInitRangeError ( std::string what ) : zypp::Exception( std::move(what) ){}
31 };
32
33 }
34
35 size_t CurlMultiPartHandler::curl_hdrcallback(char *ptr, size_t size, size_t nmemb, void *userdata)
36 {
37 if ( !userdata )
38 return 0;
39
40 CurlMultiPartHandler *that = reinterpret_cast<CurlMultiPartHandler *>( userdata );
41 return that->hdrcallback( ptr, size, nmemb );
42 }
43
44 size_t CurlMultiPartHandler::curl_wrtcallback(char *ptr, size_t size, size_t nmemb, void *userdata)
45 {
46 if ( !userdata )
47 return 0;
48
49 CurlMultiPartHandler *that = reinterpret_cast<CurlMultiPartHandler *>( userdata );
50 return that->wrtcallback( ptr, size, nmemb );
51 }
52
53 CurlMultiPartHandler::Range::Range(RangeDesc &&rd, std::optional<zypp::Digest> dig, std::any userD, State rangeState)
54 : RangeDesc( std::move(rd) )
55 , _digest( std::move(dig) )
56 , userData( std::move(userD) )
57 , _rangeState( rangeState )
58 { }
59
60 void CurlMultiPartHandler::Range::restart()
61 {
62 _rangeState = CurlMultiPartHandler::Pending;
63 bytesWritten = 0;
64 if ( _digest ) _digest->reset();
65 }
66
67 CurlMultiPartHandler::Range CurlMultiPartHandler::Range::clone() const
68 {
69 return Range (
70 RangeDesc {
71 ._start = _start,
72 ._len = _len,
73 ._chksumtype = _chksumtype,
74 ._checksum = _checksum,
75 ._relevantDigestLen = _relevantDigestLen,
76 ._chksumPad = _chksumPad
77 },
78 ( _digest ? _digest->clone() : std::optional<zypp::Digest>{} ),
79 userData,
80 _rangeState
81 );
82 }
83
84 CurlMultiPartHandler::Range CurlMultiPartHandler::Range::make(size_t start, size_t len, std::optional<zypp::Digest> &&digest, CheckSumBytes &&expectedChkSum, std::any &&userData, std::optional<size_t> digestCompareLen, std::optional<size_t> dataBlockPadding)
85 {
86 std::string chksumtype = ( digest ? digest->name() : std::string() );
87 return Range (
88 RangeDesc {
89 ._start = start,
90 ._len = len,
91 ._chksumtype = std::move(chksumtype),
92 ._checksum = std::move( expectedChkSum ),
93 ._relevantDigestLen = std::move( digestCompareLen ),
94 ._chksumPad = std::move( dataBlockPadding ),
95 },
96 std::move( digest ),
97 std::move( userData ),
98 State::Pending
99 );
100 }
101
102 CurlMultiPartHandler::CurlMultiPartHandler(ProtocolMode mode, void *easyHandle, std::vector<Range> &ranges, CurlMultiPartDataReceiver &receiver)
103 : _protocolMode( mode )
104 , _easyHandle( easyHandle )
105 , _receiver( receiver )
106 , _requestedRanges( ranges )
107 {
108 // non http can only do range by range
109 if ( _protocolMode == ProtocolMode::Basic ) {
110 WAR << "!!!! Downloading ranges without HTTP might be slow !!!!" << std::endl;
111 _rangeAttemptIdx = _rangeAttemptSize - 1;
112 }
113 }
114
115 CurlMultiPartHandler::~CurlMultiPartHandler()
116 {
117 if ( _easyHandle ) {
118 curl_easy_setopt( _easyHandle, CURLOPT_HEADERFUNCTION, nullptr );
119 curl_easy_setopt( _easyHandle, CURLOPT_HEADERDATA, nullptr );
120 curl_easy_setopt( _easyHandle, CURLOPT_WRITEFUNCTION, nullptr );
121 curl_easy_setopt( _easyHandle, CURLOPT_WRITEDATA, nullptr );
122 }
123 }
124
125 size_t CurlMultiPartHandler::hdrcallback(char *ptr, size_t size, size_t nmemb)
126 {
127 // it is valid to call this function with no data to read, just call the given handler
128 // or return ok
129 if ( size * nmemb == 0)
130 return _receiver.headerfunction( ptr, size * nmemb );
131
132 if ( _protocolMode == ProtocolMode::HTTP ) {
133
134 std::string_view hdr( ptr, size*nmemb );
135
136 hdr.remove_prefix( std::min( hdr.find_first_not_of(" \t\r\n"), hdr.size() ) );
137 const auto lastNonWhitespace = hdr.find_last_not_of(" \t\r\n");
138 if ( lastNonWhitespace != hdr.npos )
139 hdr.remove_suffix( hdr.size() - (lastNonWhitespace + 1) );
140 else
141 hdr = std::string_view();
142
143 // we just received whitespaces, wait for more
144 if ( !hdr.size() ) {
145 return ( size * nmemb );
146 }
147
148 if ( zypp::strv::hasPrefixCI( hdr, "HTTP/" ) ) {
149
150 long statuscode = 0;
151 (void)curl_easy_getinfo( _easyHandle, CURLINFO_RESPONSE_CODE, &statuscode);
152
153 const auto &doRangeFail = [&](){
154 WAR << _easyHandle << " " << "Range FAIL, trying with a smaller batch" << std::endl;
155
156 setCode ( Code::RangeFail, "Expected range status code 206, but got none." );
157
158 // reset all ranges we requested to pending, we never got the data for them
159 std::for_each( _requestedRanges.begin (), _requestedRanges.end(), []( auto &range ) {
160 if ( range._rangeState == Running )
161 range._rangeState = Pending;
162 });
163 return 0;
164 };
165
166 // ignore other status codes, maybe we are redirected etc.
167 if ( ( statuscode >= 200 && statuscode <= 299 && statuscode != 206 )
168 || statuscode == 416 ) {
169 return doRangeFail();
170 }
171
172 } else if ( zypp::strv::hasPrefixCI( hdr, "Content-Range:") ) {
173 Range r;
174
175 size_t fileLen = 0;
176 if ( !parseContentRangeHeader( hdr, r._start, r._len, fileLen ) ) {
177 //@TODO shouldn't we map this to a extra error? After all this is not our fault
178 setCode( Code::InternalError, "Invalid Content-Range header format." );
179 return 0;
180 }
181
182 if ( !_reportedFileSize ) {
183 WAR << "Setting request filesize to " << fileLen << std::endl;
184 _reportedFileSize = fileLen;
185 }
186
187 DBG << _easyHandle << " " << "Got content range :" << r._start << " len " << r._len << std::endl;
188 _gotContentRangeInfo = true;
189 _currentSrvRange = std::move(r);
190
191 } else if ( zypp::strv::hasPrefixCI( hdr, "Content-Type:") ) {
192 std::string sep;
193 if ( parseContentTypeMultiRangeHeader( hdr, sep ) ) {
194 _gotContentRangeInfo = true;
195 _isMuliPartResponse = true;
196 _seperatorString = "--"+sep;
197 }
198 }
199 }
200
201 return _receiver.headerfunction( ptr, size * nmemb );
202 }
203
204 size_t CurlMultiPartHandler::wrtcallback(char *ptr, size_t size, size_t nmemb)
205 {
206 const auto max = ( size * nmemb );
207
208 //it is valid to call this function with no data to write, just call the given handler
209 if ( max == 0)
210 return _receiver.writefunction( ptr, {}, max );
211
212 // try to consume all bytes that have been given to us
213 size_t bytesConsumedSoFar = 0;
214 while ( bytesConsumedSoFar != max ) {
215
216 std::optional<off_t> seekTo;
217
218 // this is called after all headers have been processed
219 if ( !_allHeadersReceived ) {
220 _allHeadersReceived = true;
221
222 // no ranges at all, this is a error
223 if ( _protocolMode == ProtocolMode::HTTP && !_gotContentRangeInfo ) {
224 //we got a invalid response, the status code pointed to being partial ( 206 ) but we got no range definition
225 setCode( Code::ServerReturnedError, "Invalid data from server, range respone was announced but there was no range definiton." );
226 return 0;
227 }
228 }
229
230 if ( _protocolMode == ProtocolMode::HTTP && _currentSrvRange && !_currentRange ) {
231 /*
232 this branch is responsible to find a range that overlaps the current server range, so we know which of our ranges to fill
233 Entering here means we are in one of two situations:
234
235 1) we just have finished writing a requested range but
236 still have not completely consumed a range that we have received from the server.
237 Since HTTP std allows the server to coalesce requested ranges in order to optimize downloads
238 we need to find the best match ( because the current offset might not even be in our requested ranges )
239
240 2) we just parsed a Content-Length header and start a new block
241 */
242
243 std::optional<uint> foundRange;
244 const size_t beginSrvRange = _currentSrvRange->_start + _currentSrvRange->bytesWritten;
245 const size_t endSrvRange = beginSrvRange + (_currentSrvRange->_len - _currentSrvRange->bytesWritten);
246 auto currDist = ULONG_MAX;
247 for ( uint i = 0; i < _requestedRanges.size(); i++ ) {
248 const auto &currR = _requestedRanges[i];
249
250 // do not allow double ranges
251 if ( currR._rangeState == Finished || currR._rangeState == Error )
252 continue;
253
254 // check if the range was already written
255 if ( currR._len && currR._len == currR.bytesWritten )
256 continue;
257
258 const auto currRBegin = currR._start + currR.bytesWritten;
259 if ( !( beginSrvRange <= currRBegin && endSrvRange >= currRBegin ) )
260 continue;
261
262 // calculate the distance of the current ranges offset+data written to the range we got back from the server
263 const auto newDist = currRBegin - beginSrvRange;
264
265 // exact match
266 if ( currRBegin == beginSrvRange && currR._len == _currentSrvRange->_len ) {
267 foundRange = i;
268 break;
269 }
270
271 if ( !foundRange ) {
272 foundRange = i;
273 currDist = newDist;
274 } else {
275 //pick the range with the closest distance
276 if ( newDist < currDist ) {
277 foundRange = i;
278 currDist = newDist;
279 }
280 }
281 }
282
283 if ( !foundRange ) {
284 // @TODO shouldn't we simply consume the rest of the range here and see if future data will contain a matchable range again?
285 setCode( Code::InternalError, "Unable to find a matching range for data returned by the server.");
286 return 0;
287 }
288
289 //set the found range as the current one
290 _currentRange = *foundRange;
291
292 //continue writing where we stopped
293 seekTo = _requestedRanges[*foundRange]._start + _requestedRanges[*foundRange].bytesWritten;
294
295 //if we skip bytes we need to advance our written bytecount
296 const auto skipBytes = *seekTo - beginSrvRange;
297 bytesConsumedSoFar += skipBytes;
298 _currentSrvRange->bytesWritten += skipBytes;
299
300 std::string errBuf = "Receiver cancelled starting the current range.";
301 if ( !_receiver.beginRange (*_currentRange, errBuf) ) {
302 setCode( Code::InternalError, "Receiver cancelled starting the current range.");
303 return 0;
304 }
305
306 } else if ( _protocolMode != ProtocolMode::HTTP && !_currentRange ) {
307 // if we are not running in HTTP mode we can only request a single range, that means we get our data
308 // in one continous stream. Since our
309 // ranges are ordered by start, we just pick the first one that is marked as running
310 if ( !_currentRange ) {
311 const auto i = std::find_if( _requestedRanges.begin (), _requestedRanges.end(), []( const Range &r){ return r._rangeState == Running; });
312 if ( i == _requestedRanges.end() ) {
313 setCode( Code::InternalError, "Received data but no range was marked as requested." );
314 return 0;
315 }
316
317 _currentRange = std::distance( _requestedRanges.begin(), i );
318
319 //continue writing where we stopped
320 seekTo = _requestedRanges[*_currentRange]._start + _requestedRanges[*_currentRange].bytesWritten;
321 }
322 }
323
324 if ( _currentRange ) {
325 /*
326 * We have data to write and know the target range
327 */
328
329 // make sure we do not read over the current server range
330 auto availableData = max - bytesConsumedSoFar;
331 if ( _currentSrvRange ) {
332 availableData = std::min( availableData, _currentSrvRange->_len - _currentSrvRange->bytesWritten );
333 }
334
335 auto &rng = _requestedRanges[*_currentRange];
336
337 // do only write what we need until the range is full
338 const auto bytesToWrite = rng._len > 0 ? std::min( rng._len - rng.bytesWritten, availableData ) : availableData;
339
340 auto written = _receiver.writefunction( ptr + bytesConsumedSoFar, seekTo, bytesToWrite );
341 if ( written <= 0 )
342 return 0;
343
344 if ( rng._digest && rng._checksum.size() ) {
345 if ( !rng._digest->update( ptr + bytesConsumedSoFar, written ) )
346 return 0;
347 }
348
349 rng.bytesWritten += written;
350 if ( _currentSrvRange ) _currentSrvRange->bytesWritten += written;
351
352 // range is done
353 if ( rng._len > 0 && rng.bytesWritten >= rng._len ) {
354 std::string errBuf = "Receiver cancelled after finishing the current range.";
355 bool rngIsValid = validateRange( rng );
356 if ( !_receiver.finishedRange (*_currentRange, rngIsValid, errBuf) ) {
357 setCode( Code::InternalError, "Receiver cancelled starting the current range.");
358 return 0;
359 }
360 _currentRange.reset();
361 }
362
363 if ( _currentSrvRange && _currentSrvRange->_len > 0 && _currentSrvRange->bytesWritten >= _currentSrvRange->_len ) {
364 _currentSrvRange.reset();
365 // we ran out of data in the current chunk, reset the target range if it is not a open range as well, because next data will be
366 // a chunk header again
367 if ( _isMuliPartResponse )
368 _currentRange.reset();
369 }
370
371 bytesConsumedSoFar += written;
372 }
373
374 if ( bytesConsumedSoFar == max )
375 return max;
376
377 if ( _protocolMode == ProtocolMode::HTTP && !_currentRange && !_currentSrvRange ) {
378 /*
379 This branch is reponsible to parse the multibyte response we got from the
380 server and parse the next target range from it
381 */
382
383 std::string_view incoming( ptr + bytesConsumedSoFar, max - bytesConsumedSoFar );
384 auto hdrEnd = incoming.find("\r\n\r\n");
385 if ( hdrEnd == incoming.npos ) {
386 //no header end in the data yet, push to buffer and return
387 _rangePrefaceBuffer.insert( _rangePrefaceBuffer.end(), incoming.begin(), incoming.end() );
388 return max;
389 }
390
391 //append the data of the current header to the buffer and parse it
392 _rangePrefaceBuffer.insert( _rangePrefaceBuffer.end(), incoming.begin(), incoming.begin() + ( hdrEnd + 4 ) );
393 bytesConsumedSoFar += ( hdrEnd + 4 ); //header data plus header end
394
395 std::string_view data( _rangePrefaceBuffer.data(), _rangePrefaceBuffer.size() );
396 auto sepStrIndex = data.find( _seperatorString );
397 if ( sepStrIndex == data.npos ) {
398 setCode( Code::InternalError, "Invalid multirange header format, seperator string missing." );
399 return 0;
400 }
401
402 auto startOfHeader = sepStrIndex + _seperatorString.length();
403 std::vector<std::string_view> lines;
404 zypp::strv::split( data.substr( startOfHeader ), "\r\n", zypp::strv::Trim::trim, [&]( std::string_view strv ) { lines.push_back(strv); } );
405 for ( const auto &hdrLine : lines ) {
406 if ( zypp::strv::hasPrefixCI(hdrLine, "Content-Range:") ) {
407 size_t fileLen = 0;
408 Range r;
409 //if we can not parse the header the message must be broken
410 if(! parseContentRangeHeader( hdrLine, r._start, r._len, fileLen ) ) {
411 setCode( Code::InternalError, "Invalid Content-Range header format." );
412 return 0;
413 }
414 if ( !_reportedFileSize ) {
415 _reportedFileSize = fileLen;
416 }
417 _currentSrvRange = std::move(r);
418 break;
419 }
420 }
421
422 if ( !_currentSrvRange ){
423 setCode( Code::InternalError, "Could not read a server range from the response." );
424 return 0;
425 }
426
427 //clear the buffer again
428 _rangePrefaceBuffer.clear();
429 }
430 }
431
432 return bytesConsumedSoFar;
433 }
434
435 void *CurlMultiPartHandler::easyHandle() const
436 {
437 return _easyHandle;
438 }
439
440 bool CurlMultiPartHandler::canRecover() const
441 {
442 // We can recover from RangeFail errors if we have more batch sizes to try
443 // we never auto downgrade to last range set ( which is 1 ) because in that case
444 // just downloading the full file is usually faster.
445 if ( _lastCode == Code::RangeFail )
446 return ( _rangeAttemptIdx + 1 < ( _rangeAttemptSize - 1 ) ) && hasMoreWork();
447 return false;
448 }
449
450 bool CurlMultiPartHandler::hasMoreWork() const
451 {
452 // check if we have ranges that have never been requested
453 return std::any_of( _requestedRanges.begin(), _requestedRanges.end(), []( const auto &range ){ return range._rangeState == Pending; });
454 }
455
456 bool CurlMultiPartHandler::hasError() const
457 {
458 return _lastCode != Code::NoError;
459 }
460
461 CurlMultiPartHandler::Code CurlMultiPartHandler::lastError() const
462 {
463 return _lastCode;
464 }
465
466 const std::string &CurlMultiPartHandler::lastErrorMessage() const
467 {
468 return _lastErrorMsg;
469 }
470
471 bool CurlMultiPartHandler::prepareToContinue( )
472 {
473 if ( hasMoreWork() ) {
474 // go to the next range batch level if we are restarted due to a failed range request
475 if ( _lastCode == Code::RangeFail ) {
476 if ( _rangeAttemptIdx + 1 >= _rangeAttemptSize ) {
477 setCode ( Code::RangeFail, "No more range batch sizes available", true );
478 return false;
479 }
480 _rangeAttemptIdx++;
481 }
482 return true;
483 }
484
485 setCode ( Code::NoError, "Request has no more work", true );
486 return false;
487
488 }
489
490 void CurlMultiPartHandler::finalize()
491 {
492 // if we still have a current range set it valid by checking the checksum
493 if ( _currentRange ) {
494 auto &currR = _requestedRanges[*_currentRange];
495 std::string errBuf;
496 bool rngIsValid = validateRange( currR );
497 _receiver.finishedRange (*_currentRange, rngIsValid, errBuf);
498 _currentRange.reset();
499 }
500 }
501
502 bool CurlMultiPartHandler::verifyData()
503 {
504 finalize();
505
506 for ( auto &r : _requestedRanges ) {
507 if ( r._rangeState != CurlMultiPartHandler::Finished ) {
508 if ( r._len > 0 && r.bytesWritten != r._len )
509 setCode( Code::MissingData, (zypp::str::Format("Did not receive all requested data from the server ( off: %1%, req: %2%, recv: %3% ).") % r._start % r._len % r.bytesWritten ) );
510 else if ( r._digest && r._checksum.size() && !checkIfRangeChkSumIsValid(r) ) {
511 setCode( Code::InvalidChecksum, (zypp::str::Format("Invalid checksum %1%, expected checksum %2%") % r._digest->digest() % zypp::Digest::digestVectorToString( r._checksum ) ) );
512 } else {
513 setCode( Code::InternalError, (zypp::str::Format("Download of block failed.") ) );
514 }
515 //we only report the first error
516 break;
517 }
518 }
519 return ( _lastCode == Code::NoError );
520 }
521
522 bool CurlMultiPartHandler::prepare( )
523 {
524 _lastCode = Code::NoError;
525 _lastErrorMsg.clear();
526 _seperatorString.clear();
527 _currentSrvRange.reset();
528 _reportedFileSize.reset();
529 _gotContentRangeInfo = false;
530 _allHeadersReceived = false;
531 _isMuliPartResponse = false;
532
533 if ( _requestedRanges.size() == 0 ) {
534 setCode( Code::InternalError, "Calling the CurlMultiPartHandler::prepare function without a range to download is not supported.");
535 return false;
536 }
537
538 const auto setCurlOption = [&]( CURLoption opt, auto &&data )
539 {
540 auto ret = curl_easy_setopt( _easyHandle, opt, data );
541 if ( ret != 0 ) {
542 throw CurlMultiPartSetoptError(ret);
543 }
544 };
545
546 try {
547 setCurlOption( CURLOPT_HEADERFUNCTION, CurlMultiPartHandler::curl_hdrcallback );
548 setCurlOption( CURLOPT_HEADERDATA, this );
549 setCurlOption( CURLOPT_WRITEFUNCTION, CurlMultiPartHandler::curl_wrtcallback );
550 setCurlOption( CURLOPT_WRITEDATA, this );
551
552 std::string rangeDesc;
553 uint rangesAdded = 0;
554 auto maxRanges = _rangeAttempt[_rangeAttemptIdx];
555
556 // helper function to build up the request string for the range
557 auto addRangeString = [ &rangeDesc, &rangesAdded ]( const std::pair<size_t, size_t> &range ) {
558 std::string rangeD = zypp::str::form("%llu-", static_cast<unsigned long long>( range.first ) );
559 if( range.second > 0 )
560 rangeD.append( zypp::str::form( "%llu", static_cast<unsigned long long>( range.second ) ) );
561
562 if ( rangeDesc.size() )
563 rangeDesc.append(",").append( rangeD );
564 else
565 rangeDesc = std::move( rangeD );
566
567 rangesAdded++;
568 };
569
570 std::optional<std::pair<size_t, size_t>> currentZippedRange;
571 bool closedRange = true;
572 for ( auto &range : _requestedRanges ) {
573
574 if ( range._rangeState != Pending )
575 continue;
576
577 //reset the download results
578 range.bytesWritten = 0;
579
580 //when we have a open range in the list of ranges we will get from start of range to end of file,
581 //all following ranges would never be marked as valid, so we have to fail early
582 if ( !closedRange )
583 throw CurlMultInitRangeError("It is not supported to request more ranges after a open range.");
584
585 const auto rangeEnd = range._len > 0 ? range._start + range._len - 1 : 0;
586 closedRange = (rangeEnd > 0);
587
588 bool added = false;
589
590 // we try to compress the requested ranges into as big chunks as possible for the request,
591 // when receiving we still track the original ranges so we can collect and test their checksums
592 if ( !currentZippedRange ) {
593 added = true;
594 currentZippedRange = std::make_pair( range._start, rangeEnd );
595 } else {
596 //range is directly consecutive to the previous range
597 if ( currentZippedRange->second + 1 == range._start ) {
598 added = true;
599 currentZippedRange->second = rangeEnd;
600 } else {
601 //this range does not directly follow the previous one, we build the string and start a new one
602 if ( rangesAdded +1 >= maxRanges ) break;
603 added = true;
604 addRangeString( *currentZippedRange );
605 currentZippedRange = std::make_pair( range._start, rangeEnd );
606 }
607 }
608
609 // remember range was already requested
610 if ( added ) {
611 setRangeState( range, Running );
612 range.bytesWritten = 0;
613 if ( range._digest )
614 range._digest->reset();
615 }
616
617 if ( rangesAdded >= maxRanges ) {
618 MIL << _easyHandle << " " << "Reached max nr of ranges (" << maxRanges << "), batching the request to not break the server" << std::endl;
619 break;
620 }
621 }
622
623 // add the last range too
624 if ( currentZippedRange )
625 addRangeString( *currentZippedRange );
626
627 MIL << _easyHandle << " " << "Requesting Ranges: " << rangeDesc << std::endl;
628
629 setCurlOption( CURLOPT_RANGE, rangeDesc.c_str() );
630
631 } catch( const CurlMultiPartSetoptError &err ) {
632 setCode( Code::InternalError, "" );
633 } catch( const CurlMultInitRangeError &err ) {
634 setCode( Code::InternalError, err.asUserString() );
635 } catch( const zypp::Exception &err ) {
636 setCode( Code::InternalError, err.asUserString() );
637 } catch( const std::exception &err ) {
638 setCode( Code::InternalError, zypp::str::asString(err.what()) );
639 }
640 return ( _lastCode == Code::NoError );
641 }
642
643 void CurlMultiPartHandler::setCode(Code c, std::string msg , bool force)
644 {
645 // never overwrite a error, this is reset when we restart
646 if ( _lastCode != Code::NoError && !force )
647 return;
648
649 _lastCode = c;
650 _lastErrorMsg = std::move(msg);
651 _receiver.notifyErrorCodeChanged();
652 }
653
654 bool CurlMultiPartHandler::parseContentRangeHeader( const std::string_view &line, size_t &start, size_t &len, size_t &fileLen )
655 {
656 //content-range: bytes 10485760-19147879/19147880
657 static const zypp::str::regex regex("^Content-Range:[[:space:]]+bytes[[:space:]]+([0-9]+)-([0-9]+)\\/([0-9]+)$", zypp::str::regex::rxdefault | zypp::str::regex::icase );
658
659 zypp::str::smatch what;
660 if( !zypp::str::regex_match( std::string(line), what, regex ) || what.size() != 4 ) {
661 DBG << _easyHandle << " " << "Invalid Content-Range Header format: '" << std::string(line) << std::endl;
662 return false;
663 }
664
665 size_t s = zypp::str::strtonum<size_t>( what[1]);
666 size_t e = zypp::str::strtonum<size_t>( what[2]);
667 fileLen = zypp::str::strtonum<size_t>( what[3]);
668 start = s;
669 len = ( e - s ) + 1;
670 return true;
671 }
672
673 bool CurlMultiPartHandler::parseContentTypeMultiRangeHeader( const std::string_view &line, std::string &boundary )
674 {
675 static const zypp::str::regex regex("^Content-Type:[[:space:]]+multipart\\/byteranges;[[:space:]]+boundary=(.*)$", zypp::str::regex::rxdefault | zypp::str::regex::icase );
676
677 zypp::str::smatch what;
678 if( zypp::str::regex_match( std::string(line), what, regex ) ) {
679 if ( what.size() >= 2 ) {
680 boundary = what[1];
681 return true;
682 }
683 }
684 return false;
685 }
686
687 bool CurlMultiPartHandler::validateRange( Range &rng )
688 {
689 if ( rng._digest && rng._checksum.size() ) {
690 if ( ( rng._len == 0 || rng.bytesWritten == rng._len ) && checkIfRangeChkSumIsValid(rng) )
691 setRangeState(rng, Finished);
692 else
693 setRangeState(rng, Error);
694 } else {
695 if ( rng._len == 0 ? true : rng.bytesWritten == rng._len )
696 setRangeState(rng, Finished);
697 else
698 setRangeState(rng, Error);
699 }
700 return ( rng._rangeState == Finished );
701 }
702
703 bool CurlMultiPartHandler::checkIfRangeChkSumIsValid ( Range &rng )
704 {
705 if ( rng._digest && rng._checksum.size() ) {
706 auto bytesHashed = rng._digest->bytesHashed ();
707 if ( rng._chksumPad && *rng._chksumPad > bytesHashed ) {
708 MIL_MEDIA << _easyHandle << " " << "Padding the digest to required block size" << std::endl;
709 zypp::ByteArray padding( *rng._chksumPad - bytesHashed, '\0' );
710 rng._digest->update( padding.data(), padding.size() );
711 }
712 auto digVec = rng._digest->digestVector();
713 if ( rng._relevantDigestLen ) {
714 digVec.resize( *rng._relevantDigestLen );
715 }
716 return ( digVec == rng._checksum );
717 }
718
719 // no checksum required
720 return true;
721 }
722
723 void CurlMultiPartHandler::setRangeState( Range &rng, State state )
724 {
725 if ( rng._rangeState != state ) {
726 rng._rangeState = state;
727 }
728 }
729
730 std::optional<off_t> CurlMultiPartHandler::currentRange() const
731 {
732 return _currentRange;
733 }
734
735 std::optional<size_t> CurlMultiPartHandler::reportedFileSize() const
736 {
737 return _reportedFileSize;
738 }
739
740} // namespace zyppng
DBG
#define DBG
Definition Logger.h:102
MIL
#define MIL
Definition Logger.h:103
WAR
#define WAR
Definition Logger.h:104
StringV.h
zypp::ByteArray
Definition ByteArray.h:20
zypp::Digest::digestVectorToString
static std::string digestVectorToString(const UByteArray &vec)
get hex string representation of the digest vector given as parameter
Definition Digest.cc:244
zypp::Exception
Base class for Exception.
Definition Exception.h:153
zypp::str::regex
Regular expression.
Definition Regex.h:95
zypp::str::regex::icase
@ icase
Do not differentiate case.
Definition Regex.h:99
zypp::str::regex::rxdefault
@ rxdefault
These are enforced even if you don't pass them as flag argument.
Definition Regex.h:103
zypp::str::smatch
Regular expression match result.
Definition Regex.h:168
zypp::str::smatch::size
unsigned size() const
Definition Regex.cc:106
zyppng::Base::data
void * data(uint32_t quark)
Definition base.cc:82
zyppng::CurlMultiPartDataReceiver
Definition curlmultiparthandler.h:19
zyppng::CurlMultiPartDataReceiver::headerfunction
virtual size_t headerfunction(char *ptr, size_t bytes)=0
zyppng::CurlMultiPartDataReceiver::notifyErrorCodeChanged
virtual void notifyErrorCodeChanged()
Definition curlmultiparthandler.h:59
zyppng::CurlMultiPartDataReceiver::writefunction
virtual size_t writefunction(char *ptr, std::optional< off_t > offset, size_t bytes)=0
zyppng::CurlMultiPartDataReceiver::finishedRange
virtual bool finishedRange(off_t range, bool validated, std::string &cancelReason)
Definition curlmultiparthandler.h:54
zyppng::CurlMultiPartDataReceiver::beginRange
virtual bool beginRange(off_t range, std::string &cancelReason)
Definition curlmultiparthandler.h:42
zyppng::CurlMultiPartHandler::prepare
bool prepare()
Definition curlmultiparthandler.cc:522
zyppng::CurlMultiPartHandler::lastError
Code lastError() const
Definition curlmultiparthandler.cc:461
zyppng::CurlMultiPartHandler::wrtcallback
size_t wrtcallback(char *ptr, size_t size, size_t nmemb)
Definition curlmultiparthandler.cc:204
zyppng::CurlMultiPartHandler::_rangeAttemptIdx
unsigned _rangeAttemptIdx
Definition curlmultiparthandler.h:181
zyppng::CurlMultiPartHandler::_rangeAttemptSize
static constexpr unsigned _rangeAttemptSize
Definition curlmultiparthandler.h:92
zyppng::CurlMultiPartHandler::_protocolMode
ProtocolMode _protocolMode
Definition curlmultiparthandler.h:163
zyppng::CurlMultiPartHandler::CurlMultiPartHandler
CurlMultiPartHandler(ProtocolMode mode, void *easyHandle, std::vector< Range > &ranges, CurlMultiPartDataReceiver &receiver)
Definition curlmultiparthandler.cc:102
zyppng::CurlMultiPartHandler::finalize
void finalize()
Definition curlmultiparthandler.cc:490
zyppng::CurlMultiPartHandler::_lastCode
Code _lastCode
Definition curlmultiparthandler.h:167
zyppng::CurlMultiPartHandler::_currentSrvRange
std::optional< Range > _currentSrvRange
Definition curlmultiparthandler.h:178
zyppng::CurlMultiPartHandler::_seperatorString
std::string _seperatorString
The seperator string for multipart responses as defined in RFC 7233 Section 4.1.
Definition curlmultiparthandler.h:174
zyppng::CurlMultiPartHandler::canRecover
bool canRecover() const
Definition curlmultiparthandler.cc:440
zyppng::CurlMultiPartHandler::prepareToContinue
bool prepareToContinue()
Definition curlmultiparthandler.cc:471
zyppng::CurlMultiPartHandler::Code
NetworkRequestError::Type Code
Definition curlmultiparthandler.h:102
zyppng::CurlMultiPartHandler::_isMuliPartResponse
bool _isMuliPartResponse
Definition curlmultiparthandler.h:172
zyppng::CurlMultiPartHandler::_lastErrorMsg
std::string _lastErrorMsg
Definition curlmultiparthandler.h:168
zyppng::CurlMultiPartHandler::_gotContentRangeInfo
bool _gotContentRangeInfo
Definition curlmultiparthandler.h:171
zyppng::CurlMultiPartHandler::curl_wrtcallback
static size_t curl_wrtcallback(char *ptr, size_t size, size_t nmemb, void *userdata)
Definition curlmultiparthandler.cc:44
zyppng::CurlMultiPartHandler::curl_hdrcallback
static size_t curl_hdrcallback(char *ptr, size_t size, size_t nmemb, void *userdata)
Definition curlmultiparthandler.cc:35
zyppng::CurlMultiPartHandler::~CurlMultiPartHandler
~CurlMultiPartHandler() override
Definition curlmultiparthandler.cc:115
zyppng::CurlMultiPartHandler::lastErrorMessage
const std::string & lastErrorMessage() const
Definition curlmultiparthandler.cc:466
zyppng::CurlMultiPartHandler::_rangeAttempt
static constexpr unsigned _rangeAttempt[]
Definition curlmultiparthandler.h:83
zyppng::CurlMultiPartHandler::parseContentRangeHeader
bool parseContentRangeHeader(const std::string_view &line, size_t &start, size_t &len, size_t &fileLen)
Definition curlmultiparthandler.cc:654
zyppng::CurlMultiPartHandler::setCode
void setCode(Code c, std::string msg, bool force=false)
Definition curlmultiparthandler.cc:643
zyppng::CurlMultiPartHandler::_rangePrefaceBuffer
std::vector< char > _rangePrefaceBuffer
Here we buffer.
Definition curlmultiparthandler.h:175
zyppng::CurlMultiPartHandler::currentRange
std::optional< off_t > currentRange() const
Definition curlmultiparthandler.cc:730
zyppng::CurlMultiPartHandler::_reportedFileSize
std::optional< size_t > _reportedFileSize
Filesize as reported by the content range or byte range headers.
Definition curlmultiparthandler.h:179
zyppng::CurlMultiPartHandler::hdrcallback
size_t hdrcallback(char *ptr, size_t size, size_t nmemb)
Definition curlmultiparthandler.cc:125
zyppng::CurlMultiPartHandler::validateRange
bool validateRange(Range &rng)
Definition curlmultiparthandler.cc:687
zyppng::CurlMultiPartHandler::State
State
Definition curlmultiparthandler.h:94
zyppng::CurlMultiPartHandler::Running
@ Running
Definition curlmultiparthandler.h:96
zyppng::CurlMultiPartHandler::Error
@ Error
Definition curlmultiparthandler.h:98
zyppng::CurlMultiPartHandler::Pending
@ Pending
Definition curlmultiparthandler.h:95
zyppng::CurlMultiPartHandler::Finished
@ Finished
Definition curlmultiparthandler.h:97
zyppng::CurlMultiPartHandler::_receiver
CurlMultiPartDataReceiver & _receiver
Definition curlmultiparthandler.h:165
zyppng::CurlMultiPartHandler::hasMoreWork
bool hasMoreWork() const
Definition curlmultiparthandler.cc:450
zyppng::CurlMultiPartHandler::hasError
bool hasError() const
Definition curlmultiparthandler.cc:456
zyppng::CurlMultiPartHandler::setRangeState
void setRangeState(Range &rng, State state)
Definition curlmultiparthandler.cc:723
zyppng::CurlMultiPartHandler::reportedFileSize
std::optional< size_t > reportedFileSize() const
Definition curlmultiparthandler.cc:735
zyppng::CurlMultiPartHandler::parseContentTypeMultiRangeHeader
bool parseContentTypeMultiRangeHeader(const std::string_view &line, std::string &boundary)
Definition curlmultiparthandler.cc:673
zyppng::CurlMultiPartHandler::_easyHandle
void * _easyHandle
Definition curlmultiparthandler.h:164
zyppng::CurlMultiPartHandler::_allHeadersReceived
bool _allHeadersReceived
Definition curlmultiparthandler.h:170
zyppng::CurlMultiPartHandler::easyHandle
void * easyHandle() const
Definition curlmultiparthandler.cc:435
zyppng::CurlMultiPartHandler::checkIfRangeChkSumIsValid
bool checkIfRangeChkSumIsValid(Range &rng)
Definition curlmultiparthandler.cc:703
zyppng::CurlMultiPartHandler::ProtocolMode
ProtocolMode
Definition curlmultiparthandler.h:73
zyppng::CurlMultiPartHandler::ProtocolMode::HTTP
@ HTTP
Definition curlmultiparthandler.h:75
zyppng::CurlMultiPartHandler::ProtocolMode::Basic
@ Basic
Definition curlmultiparthandler.h:74
zyppng::CurlMultiPartHandler::_requestedRanges
std::vector< Range > & _requestedRanges
the requested ranges that need to be downloaded
Definition curlmultiparthandler.h:182
zyppng::CurlMultiPartHandler::CheckSumBytes
UByteArray CheckSumBytes
Definition curlmultiparthandler.h:101
zyppng::CurlMultiPartHandler::verifyData
bool verifyData()
Definition curlmultiparthandler.cc:502
zyppng::CurlMultiPartHandler::_currentRange
std::optional< off_t > _currentRange
Definition curlmultiparthandler.h:177
curlmultiparthandler.h
zypp::str::regex_match
bool regex_match(const char *s, smatch &matches, const regex &regex) ZYPP_API
Regular expression matching.
Definition Regex.cc:80
mediadebug_p.h
MIL_MEDIA
#define MIL_MEDIA
Definition mediadebug_p.h:29
std
Definition ansi.h:855
zypp::str::asString
const std::string & asString(const std::string &t)
Global asString() that works with std::string too.
Definition String.h:140
zypp::str::form
std::string form(const char *format,...) __attribute__((format(printf
Printf style construction of std::string.
Definition String.cc:39
zypp::str::strtonum
TInt strtonum(const C_Str &str)
Parsing numbers from string.
zypp::strv
Definition StringV.cc:18
zyppng::ranges
Definition ranges.h:56
zyppng
Definition MediaNetworkRequestExecutor.h:16
zypp::str::Format
Convenient building of std::string with boost::format.
Definition String.h:254
zyppng::CurlMultiPartHandler::Range
Definition curlmultiparthandler.h:104
zyppng::CurlMultiPartHandler::Range::_rangeState
State _rangeState
Definition curlmultiparthandler.h:118
zyppng::CurlMultiPartHandler::Range::_digest
std::optional< zypp::Digest > _digest
Enables automated checking of downloaded contents against a checksum.
Definition curlmultiparthandler.h:114
zyppng::CurlMultiPartHandler::Range::clone
Range clone() const
Definition curlmultiparthandler.cc:67
zyppng::CurlMultiPartHandler::Range::Range
Range()=default
zyppng::CurlMultiPartHandler::Range::bytesWritten
size_t bytesWritten
Definition curlmultiparthandler.h:109
zyppng::CurlMultiPartHandler::Range::restart
void restart()
Definition curlmultiparthandler.cc:60
zyppng::CurlMultiPartHandler::Range::userData
std::any userData
Definition curlmultiparthandler.h:116
zyppng::CurlMultiPartHandler::Range::make
static Range make(size_t start, size_t len=0, std::optional< zypp::Digest > &&digest={}, CheckSumBytes &&expectedChkSum=CheckSumBytes(), std::any &&userData=std::any(), std::optional< size_t > digestCompareLen={}, std::optional< size_t > _dataBlockPadding={})
Definition curlmultiparthandler.cc:84
zyppng::RangeDesc
Definition rangedesc.h:27
zyppng::RangeDesc::_chksumtype
std::string _chksumtype
Enables automated checking of downloaded contents against a checksum.
Definition rangedesc.h:35
zyppng::RangeDesc::_start
size_t _start
Definition rangedesc.h:28
zyppng::RangeDesc::_len
size_t _len
Definition rangedesc.h:29
zyppng::RangeDesc::_checksum
UByteArray _checksum
Definition rangedesc.h:36
zyppng::RangeDesc::_relevantDigestLen
std::optional< size_t > _relevantDigestLen
Definition rangedesc.h:37
zyppng::RangeDesc::_chksumPad
std::optional< size_t > _chksumPad
Definition rangedesc.h:38