@@ -102,15 +102,16 @@ fd_snapshot_httpdl_reset( fd_snapshot_httpdl_t * self ) {
102
102
self -> hops = FD_SNAPSHOT_HTTPDL_DEFAULT_HOPS ;
103
103
self -> req_deadline = 0L ;
104
104
105
- self -> req_tail = 0UL ;
106
- self -> req_head = 0UL ;
107
- self -> resp_tail = 0UL ;
108
- self -> resp_head = 0UL ;
109
- self -> dl_total = 0UL ;
110
- self -> last_dl_total = 0UL ;
111
- self -> last_nanos = 0UL ;
112
- self -> write_total = 0UL ;
113
- self -> content_len = 0UL ;
105
+ self -> req_tail = 0UL ;
106
+ self -> req_head = 0UL ;
107
+ self -> resp_tail = 0UL ;
108
+ self -> file_resp_tail = 0UL ;
109
+ self -> resp_head = 0UL ;
110
+ self -> dl_total = 0UL ;
111
+ self -> last_dl_total = 0UL ;
112
+ self -> last_nanos = 0UL ;
113
+ self -> write_total = 0UL ;
114
+ self -> content_len = 0UL ;
114
115
115
116
fd_memset ( self -> req_buf , 0 , sizeof (self -> req_buf ) );
116
117
fd_memset ( self -> resp_buf , 0 , sizeof (self -> resp_buf ) );
@@ -300,7 +301,7 @@ fd_snapshot_httpdl_init_full_snapshot_file( fd_snapshot_httpdl_t * self,
300
301
fd_memcpy ( self -> snapshot_filename , self -> full_snapshot_entry -> filename , PATH_MAX );
301
302
302
303
/* open full snapshot save file */
303
- self -> current_snapshot_fd = open ( self -> snapshot_filename_temp , O_WRONLY |O_CREAT |O_TRUNC , S_IRUSR |S_IWUSR );
304
+ self -> current_snapshot_fd = open ( self -> snapshot_filename_temp , O_WRONLY |O_CREAT |O_TRUNC | O_NONBLOCK , S_IRUSR |S_IWUSR );
304
305
if ( FD_UNLIKELY ( self -> current_snapshot_fd < 0 ) ) {
305
306
FD_LOG_WARNING (( "open(%s) failed (%d-%s)" , self -> snapshot_filename_temp , errno , fd_io_strerror ( errno ) ));
306
307
self -> state = FD_SNAPSHOT_HTTPDL_STATE_FAIL ;
@@ -372,9 +373,10 @@ fd_snapshot_httpdl_init_incremental_snapshot_file( fd_snapshot_httpdl_t * self,
372
373
static void
373
374
fd_snapshot_httpdl_reset_req ( fd_snapshot_httpdl_t * self ) {
374
375
self -> req_deadline = fd_log_wallclock () + (long )FD_SNAPSHOT_HTTPDL_REQUEST_TIMEOUT ;
375
- self -> state = FD_SNAPSHOT_HTTPDL_STATE_REQ ;
376
- self -> resp_tail = 0UL ;
377
- self -> resp_head = 0UL ;
376
+ self -> state = FD_SNAPSHOT_HTTPDL_STATE_REQ ;
377
+ self -> resp_tail = 0UL ;
378
+ self -> file_resp_tail = 0UL ;
379
+ self -> resp_head = 0UL ;
378
380
}
379
381
380
382
static int
@@ -527,7 +529,8 @@ fd_snapshot_httpdl_resp( fd_snapshot_httpdl_t * self ) {
527
529
Remember where the leftover tail started so we can later reuse it
528
530
during response reading. */
529
531
530
- self -> resp_tail = (ulong )parse_res ;
532
+ self -> resp_tail = (ulong )parse_res ;
533
+ self -> file_resp_tail = self -> resp_tail ;
531
534
532
535
/* Is it a redirect? If so, start over. */
533
536
@@ -598,23 +601,25 @@ fd_snapshot_httpdl_resp( fd_snapshot_httpdl_t * self ) {
598
601
599
602
static int
600
603
fd_snapshot_httldl_write_snapshot_file ( fd_snapshot_httpdl_t * self ,
601
- ulong write_sz ) {
604
+ ulong sz ,
605
+ ulong * written_sz ) {
602
606
FD_TEST ( self -> current_snapshot_fd != -1 );
603
607
604
608
/* write out to snapshot file */
605
609
ulong src_sz ;
606
610
int err = fd_io_write ( self -> current_snapshot_fd ,
607
- self -> resp_buf + self -> resp_tail ,
608
- write_sz ,
609
- write_sz ,
611
+ self -> resp_buf + self -> file_resp_tail ,
612
+ sz ,
613
+ sz ,
610
614
& src_sz );
611
- if ( FD_UNLIKELY ( err != 0 ) ) {
615
+ if ( FD_UNLIKELY ( err != EAGAIN ) ) {
612
616
FD_LOG_WARNING (( "fd_io_write() failed (%d-%s) requested %lu bytes and wrote %lu bytes" , err , fd_io_strerror ( err ), write_sz , src_sz ));
613
617
self -> state = FD_SNAPSHOT_HTTPDL_STATE_FAIL ;
614
618
fd_snapshot_httpdl_cleanup_fds ( self );
615
619
return err ;
616
620
}
617
621
622
+ * written_sz = src_sz ;
618
623
return 0 ;
619
624
}
620
625
@@ -655,21 +660,30 @@ fd_snapshot_httpdl_write( fd_snapshot_httpdl_t * self,
655
660
656
661
/* write out to in memory buffer */
657
662
ulong write_sz = fd_ulong_min ( avail_sz , dst_max );
658
- fd_memcpy ( dst , self -> resp_buf + self -> resp_tail , write_sz );
659
- * sz = write_sz ;
663
+ if ( write_sz ) {
664
+ fd_memcpy ( dst , self -> resp_buf + self -> resp_tail , write_sz );
665
+ * sz = write_sz ;
666
+ }
667
+
668
+ ulong avail_file_sz = self -> resp_head - self -> file_resp_tail ;
669
+ ulong file_write_sz = fd_ulong_min ( avail_file_sz , dst_max );
660
670
661
671
/* save snapshot contents to file */
662
- int err = fd_snapshot_httldl_write_snapshot_file ( self , write_sz );
672
+ ulong file_written_sz = 0UL ;
673
+ int err = fd_snapshot_httldl_write_snapshot_file ( self , file_write_sz , & file_written_sz );
663
674
if ( FD_UNLIKELY ( err ) ) {
664
675
return err ;
665
676
}
666
677
667
678
self -> resp_tail += (uint )write_sz ;
679
+ self -> file_resp_tail += (uint )file_written_sz ;
668
680
self -> write_total += write_sz ;
681
+ self -> file_write_total += file_written_sz ;
669
682
self -> metrics .bytes_read = self -> dl_total ;
670
683
671
684
/* check if done downloading and writing */
672
- if ( self -> content_len == self -> write_total ) {
685
+ if ( self -> content_len == self -> write_total &&
686
+ self -> content_len == self -> file_write_total ) {
673
687
FD_LOG_NOTICE (( "Wrote out all %lu MB" , self -> write_total >>20 ));
674
688
675
689
self -> state = FD_SNAPSHOT_HTTPDL_STATE_DONE ;
@@ -696,9 +710,10 @@ fd_snapshot_httpdl_dl( fd_snapshot_httpdl_t * self,
696
710
else return 0 ;
697
711
}
698
712
699
- if ( self -> resp_head == self -> resp_tail ) {
713
+ if ( self -> resp_head == self -> resp_tail &&
714
+ self -> resp_head == self -> file_resp_tail ) {
700
715
/* Empty resp buffer means we can recv more bytes */
701
- self -> resp_tail = self -> resp_head = 0UL ;
716
+ self -> resp_tail = self -> file_resp_tail = self -> resp_head = 0UL ;
702
717
long recv_sz = recv ( self -> socket_fd , self -> resp_buf ,
703
718
fd_ulong_min ( self -> content_len - self -> dl_total , FD_SNAPSHOT_HTTPDL_RESP_BUF_MAX ),
704
719
MSG_DONTWAIT );
0 commit comments