Merge branch 'prng'

Author: dhardy

src/distributions/mod.rs

@@ -279,8 +279,8 @@ fn ziggurat<R: Rng, P, Z>(
 
 #[cfg(test)]
 mod tests {
-
     use {Rng, Rand};
+    use impls;
     use super::{RandSample, WeightedChoice, Weighted, Sample, IndependentSample};
 
     #[derive(PartialEq, Debug)]
@@ -301,6 +301,10 @@ mod tests {
         fn next_u64(&mut self) -> u64 {
             self.next_u32() as u64
         }
+        
+        fn fill_bytes(&mut self, dest: &mut [u8]) {
+            impls::fill_bytes_via_u32(self, dest)
+        }
     }
 
     #[test]

src/impls.rs

@@ -0,0 +1,170 @@
+// Copyright 2013-2017 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+//! Helper functions for implementing `Rng` functions.
+//! 
+//! For cross-platform reproducibility, these functions all use Little Endian:
+//! least-significant part first. For example, `next_u64_via_u32` takes `u32`
+//! values `x, y`, then outputs `(y << 32) | x`. To implement `next_u32`
+//! from `next_u64` in little-endian order, one should use `next_u64() as u32`.
+//! 
+//! Byte-swapping (like the std `to_le` functions) is only needed to convert
+//! to/from byte sequences, and since its purpose is reproducibility,
+//! non-reproducible sources (e.g. `OsRng`) need not bother with it.
+
+// TODO: eventually these should be exported somehow
+#![allow(unused)]
+
+use core::intrinsics::transmute;
+use core::slice;
+use core::cmp::min;
+use core::mem::size_of;
+use Rng;
+
+/// Implement `next_u64` via `next_u32`, little-endian order.
+pub fn next_u64_via_u32<R: Rng+?Sized>(rng: &mut R) -> u64 {
+    // Use LE; we explicitly generate one value before the next.
+    let x = rng.next_u32() as u64;
+    let y = rng.next_u32() as u64;
+    (y << 32) | x
+}
+
+macro_rules! fill_bytes_via {
+    ($rng:ident, $next_u:ident, $BYTES:expr, $dest:ident) => {{
+        let mut left = $dest;
+        while left.len() >= $BYTES {
+            let (l, r) = {left}.split_at_mut($BYTES);
+            left = r;
+            let chunk: [u8; $BYTES] = unsafe {
+                transmute($rng.$next_u().to_le())
+            };
+            l.copy_from_slice(&chunk);
+        }
+        let n = left.len();
+        if n > 0 {
+            let chunk: [u8; $BYTES] = unsafe {
+                transmute($rng.$next_u().to_le())
+            };
+            left.copy_from_slice(&chunk[..n]);
+        }
+    }}
+}
+
+/// Implement `fill_bytes` via `next_u32`, little-endian order.
+pub fn fill_bytes_via_u32<R: Rng+?Sized>(rng: &mut R, dest: &mut [u8]) {
+    fill_bytes_via!(rng, next_u32, 4, dest)
+}
+
+/// Implement `fill_bytes` via `next_u64`, little-endian order.
+pub fn fill_bytes_via_u64<R: Rng+?Sized>(rng: &mut R, dest: &mut [u8]) {
+    fill_bytes_via!(rng, next_u64, 8, dest)
+}
+
+macro_rules! impl_uint_from_fill {
+    ($rng:expr, $ty:ty, $N:expr) => ({
+        debug_assert!($N == size_of::<$ty>());
+
+        let mut int: $ty = 0;
+        unsafe {
+            let ptr = &mut int as *mut $ty as *mut u8;
+            let slice = slice::from_raw_parts_mut(ptr, $N);
+            $rng.fill_bytes(slice);
+        }
+        int
+    });
+}
+
+macro_rules! fill_via_chunks {
+    ($src:expr, $dest:expr, $N:expr) => ({
+        let chunk_size_u8 = min($src.len() * $N, $dest.len());
+        let chunk_size = (chunk_size_u8 + $N - 1) / $N;
+
+        // Convert to little-endian:
+        for ref mut x in $src[0..chunk_size].iter_mut() {
+            **x = (*x).to_le();
+        }
+
+        let bytes = unsafe { slice::from_raw_parts($src.as_ptr() as *const u8,
+                                                   $src.len() * $N) };
+
+        let dest_chunk = &mut $dest[0..chunk_size_u8];
+        dest_chunk.copy_from_slice(&bytes[0..chunk_size_u8]);
+
+        (chunk_size, chunk_size_u8)
+    });
+}
+
+/// Implement `fill_bytes` by reading chunks from the output buffer of a block
+/// based RNG.
+///
+/// The return values are `(consumed_u32, filled_u8)`.
+///
+/// `filled_u8` is the number of filled bytes in `dest`, which may be less than
+/// the length of `dest`.
+/// `consumed_u32` is the number of words consumed from `src`, which is the same
+/// as `filled_u8 / 4` rounded up.
+///
+/// Note that on big-endian systems values in the output buffer `src` are
+/// mutated. `src[0..consumed_u32]` get converted to little-endian before
+/// copying.
+///
+/// # Example
+/// (from `IsaacRng`)
+///
+/// ```rust,ignore
+/// fn fill_bytes(&mut self, dest: &mut [u8]) {
+///     let mut read_len = 0;
+///     while read_len < dest.len() {
+///         if self.index >= self.rsl.len() {
+///             self.isaac();
+///         }
+///
+///         let (consumed_u32, filled_u8) =
+///             impls::fill_via_u32_chunks(&mut self.rsl[self.index..],
+///                                        &mut dest[read_len..]);
+///
+///         self.index += consumed_u32;
+///         read_len += filled_u8;
+///     }
+/// }
+/// ```
+pub fn fill_via_u32_chunks(src: &mut [u32], dest: &mut [u8]) -> (usize, usize) {
+    fill_via_chunks!(src, dest, 4)
+}
+
+/// Implement `fill_bytes` by reading chunks from the output buffer of a block
+/// based RNG.
+///
+/// The return values are `(consumed_u64, filled_u8)`.
+/// `filled_u8` is the number of filled bytes in `dest`, which may be less than
+/// the length of `dest`.
+/// `consumed_u64` is the number of words consumed from `src`, which is the same
+/// as `filled_u8 / 8` rounded up.
+///
+/// Note that on big-endian systems values in the output buffer `src` are
+/// mutated. `src[0..consumed_u64]` get converted to little-endian before
+/// copying.
+///
+/// See `fill_via_u32_chunks` for an example.
+pub fn fill_via_u64_chunks(src: &mut [u64], dest: &mut [u8]) -> (usize, usize) {
+    fill_via_chunks!(src, dest, 8)
+}
+
+/// Implement `next_u32` via `fill_bytes`, little-endian order.
+pub fn next_u32_via_fill<R: Rng+?Sized>(rng: &mut R) -> u32 {
+    impl_uint_from_fill!(rng, u32, 4)
+}
+
+/// Implement `next_u64` via `fill_bytes`, little-endian order.
+pub fn next_u64_via_fill<R: Rng+?Sized>(rng: &mut R) -> u64 {
+    impl_uint_from_fill!(rng, u64, 8)
+}
+
+// TODO: implement tests for the above

src/jitter.rs

@@ -16,7 +16,7 @@
 
 //! Non-physical true random number generator based on timing jitter.
 
-use Rng;
+use {Rng, impls};
 
 use core::{fmt, mem, ptr};
 #[cfg(feature="std")]
@@ -731,22 +731,7 @@ impl Rng for JitterRng {
     }
 
     fn fill_bytes(&mut self, dest: &mut [u8]) {
-        let mut left = dest;
-        while left.len() >= 8 {
-            let (l, r) = {left}.split_at_mut(8);
-            left = r;
-            let chunk: [u8; 8] = unsafe {
-                mem::transmute(self.next_u64().to_le())
-            };
-            l.copy_from_slice(&chunk);
-        }
-        let n = left.len();
-        if n > 0 {
-            let chunk: [u8; 8] = unsafe {
-                mem::transmute(self.next_u64().to_le())
-            };
-            left.copy_from_slice(&chunk[..n]);
-        }
+        impls::fill_bytes_via_u64(self, dest)
     }
 }
 

src/lib.rs

@@ -277,6 +277,7 @@ use distributions::range::SampleRange;
 
 // public modules
 pub mod distributions;
+mod impls;
 pub mod jitter;
 #[cfg(feature="std")] pub mod os;
 #[cfg(feature="std")] pub mod read;
@@ -338,21 +339,28 @@ pub trait Rand : Sized {
 
 /// A random number generator.
 pub trait Rng {
-    /// Return the next random u32.
-    ///
-    /// This rarely needs to be called directly, prefer `r.gen()` to
-    /// `r.next_u32()`.
-    // FIXME #rust-lang/rfcs#628: Should be implemented in terms of next_u64
+    /// Return the next random `u32`.
+    /// 
+    /// Implementations of this trait must implement at least one of
+    /// `next_u32`, `next_u64` and `fill_bytes` directly. In the case this
+    /// function is not implemented directly, it can be implemented using
+    /// `self.next_u64() as u32` or via `fill_bytes` (TODO: expose helper
+    /// function).
     fn next_u32(&mut self) -> u32;
 
-    /// Return the next random u64.
-    ///
-    /// By default this is implemented in terms of `next_u32`. An
-    /// implementation of this trait must provide at least one of
-    /// these two methods. Similarly to `next_u32`, this rarely needs
-    /// to be called directly, prefer `r.gen()` to `r.next_u64()`.
+    /// Return the next random `u64`.
+    /// 
+    /// Implementations of this trait must implement at least one of
+    /// `next_u32`, `next_u64` and `fill_bytes` directly. In the case this
+    /// function is not implemented directly, the default implementation will
+    /// generate values via `next_u32` in little-endian fashion, or this
+    /// function can be implemented via `fill_bytes` (TODO: expose helper
+    /// function).
+    /// 
+    /// Types wrapping an inner RNG must not use the default implementation,
+    /// since the inner RNG's implementation may produce different values.
     fn next_u64(&mut self) -> u64 {
-        ((self.next_u32() as u64) << 32) | (self.next_u32() as u64)
+        impls::next_u64_via_u32(self)
     }
 
     /// Return the next random f32 selected from the half-open
@@ -408,18 +416,22 @@ pub trait Rng {
     }
 
     /// Fill `dest` with random data.
-    ///
-    /// This has a default implementation in terms of `next_u64` and
-    /// `next_u32`, but should be overridden by implementations that
-    /// offer a more efficient solution than just calling those
-    /// methods repeatedly.
-    ///
-    /// This method does *not* have a requirement to bear any fixed
-    /// relationship to the other methods, for example, it does *not*
-    /// have to result in the same output as progressively filling
-    /// `dest` with `self.gen::<u8>()`, and any such behaviour should
-    /// not be relied upon.
-    ///
+    /// 
+    /// Implementations of this trait must implement at least one of
+    /// `next_u32`, `next_u64` and `fill_bytes` directly. In the case this
+    /// function is not implemented directly, the default implementation will
+    /// generate values via `next_u64` in little-endian fashion.
+    /// (TODO: expose helper function to allow implementation via `next_u32`.)
+    /// 
+    /// There is no requirement on how this method generates values relative to
+    /// `next_u32` or `next_u64`; e.g. a `u64` cast to bytes is not required to
+    /// have the same value as eight bytes filled via this function. There *is*
+    /// a requirement of portability for reproducible generators which implies
+    /// that any seedable generator must fix endianness when generating bytes.
+    /// 
+    /// Types wrapping an inner RNG must not use the default implementation,
+    /// since the inner RNG's implementation may produce different values.
+    /// 
     /// This method should guarantee that `dest` is entirely filled
     /// with new data, and may panic if this is impossible
     /// (e.g. reading past the end of a file that is being used as the
@@ -435,27 +447,7 @@ pub trait Rng {
     /// println!("{:?}", &v[..]);
     /// ```
     fn fill_bytes(&mut self, dest: &mut [u8]) {
-        // this could, in theory, be done by transmuting dest to a
-        // [u64], but this is (1) likely to be undefined behaviour for
-        // LLVM, (2) has to be very careful about alignment concerns,
-        // (3) adds more `unsafe` that needs to be checked, (4)
-        // probably doesn't give much performance gain if
-        // optimisations are on.
-        let mut count = 0;
-        let mut num = 0;
-        for byte in dest.iter_mut() {
-            if count == 0 {
-                // we could micro-optimise here by generating a u32 if
-                // we only need a few more bytes to fill the vector
-                // (i.e. at most 4).
-                num = self.next_u64();
-                count = 8;
-            }
-
-            *byte = (num & 0xff) as u8;
-            num >>= 8;
-            count -= 1;
-        }
+        impls::fill_bytes_via_u64(self, dest)
     }
 
     /// Return a random value of a `Rand` type.
@@ -768,7 +760,7 @@ pub struct Closed01<F>(pub F);
 
 /// The standard RNG. This is designed to be efficient on the current
 /// platform.
-#[derive(Copy, Clone, Debug)]
+#[derive(Clone, Debug)]
 pub struct StdRng {
     rng: IsaacWordRng,
 }
@@ -811,6 +803,11 @@ impl Rng for StdRng {
     fn next_u64(&mut self) -> u64 {
         self.rng.next_u64()
     }
+    
+    #[inline]
+    fn fill_bytes(&mut self, dest: &mut [u8]) {
+        self.rng.fill_bytes(dest)
+    }
 }
 
 impl<'a> SeedableRng<&'a [usize]> for StdRng {
@@ -985,17 +982,21 @@ pub fn sample<T, I, R>(rng: &mut R, iterable: I, amount: usize) -> Vec<T>
 
 #[cfg(test)]
 mod test {
+    use impls;
     use super::{Rng, thread_rng, random, SeedableRng, StdRng, weak_rng};
     use std::iter::repeat;
 
     pub struct MyRng<R> { inner: R }
 
     impl<R: Rng> Rng for MyRng<R> {
         fn next_u32(&mut self) -> u32 {
-            fn next<T: Rng>(t: &mut T) -> u32 {
-                t.next_u32()
-            }
-            next(&mut self.inner)
+            self.inner.next_u32()
+        }
+        fn next_u64(&mut self) -> u64 {
+            self.inner.next_u64()
+        }
+        fn fill_bytes(&mut self, dest: &mut [u8]) {
+            self.inner.fill_bytes(dest)
         }
     }
 
@@ -1007,8 +1008,10 @@ mod test {
     impl Rng for ConstRng {
         fn next_u32(&mut self) -> u32 { self.i as u32 }
         fn next_u64(&mut self) -> u64 { self.i }
-
-        // no fill_bytes on purpose
+        
+        fn fill_bytes(&mut self, dest: &mut [u8]) {
+            impls::fill_bytes_via_u64(self, dest)
+        }
     }
 
     pub fn iter_eq<I, J>(i: I, j: J) -> bool