Merge pull request #162 from scala/support-for-transient

Adds ability to pickle/unpickle things marked @transient

Author: heathermiller

core/src/main/scala/pickling/Compat.scala

@@ -85,6 +85,12 @@ object Compat {
     c.Expr[FastTypeTag[T]](bundle.impl[T])
   }
 
+  def FastTypeTagMacros_implClassTag[T: c.WeakTypeTag](c: Context): c.Expr[FastTypeTag[T]] = {
+    val c0: c.type = c
+    val bundle = new { val c: c0.type = c0 } with FastTypeTagMacros
+    c.Expr[FastTypeTag[T]](bundle.implClassTag[T])
+  }
+
   def FastTypeTagMacros_apply(c: Context)(key: c.Expr[String]): c.Expr[FastTypeTag[t]] forSome { type t } = {
     val c0: c.type = c
     val bundle = new { val c: c0.type = c0 } with FastTypeTagMacros

core/src/main/scala/pickling/FastTags.scala

@@ -5,6 +5,7 @@ import scala.reflect.macros.Context
 import scala.reflect.api.{Universe => ApiUniverse}
 import scala.reflect.runtime.{universe => ru}
 import language.experimental.macros
+import scala.reflect.ClassTag
 
 trait FastTypeTag[T] extends Equals {
   def mirror: ru.Mirror
@@ -19,6 +20,8 @@ trait FastTypeTag[T] extends Equals {
 object FastTypeTag {
   implicit def materializeFastTypeTag[T]: FastTypeTag[T] = macro Compat.FastTypeTagMacros_impl[T]
 
+  implicit def materializeFastTypeTagOfClassTag[T]: FastTypeTag[ClassTag[T]] = macro Compat.FastTypeTagMacros_implClassTag[T]
+
   private def stdTag[T: ru.TypeTag]: FastTypeTag[T] = apply(scala.reflect.runtime.currentMirror, ru.typeOf[T], ru.typeOf[T].key).asInstanceOf[FastTypeTag[T]]
   implicit val Null    = stdTag[Null]
   implicit val Byte    = stdTag[Byte]
@@ -108,6 +111,17 @@ trait FastTypeTagMacros extends Macro {
       }
     """
   }
+  def implClassTag[T: c.WeakTypeTag]: c.Tree = {
+    import c.universe._
+    val T = weakTypeOf[T]
+    q"""
+      new FastTypeTag[ClassTag[$T]] {
+        def mirror = scala.pickling.internal.`package`.currentMirror
+        lazy val tpe = scala.reflect.runtime.universe.weakTypeOf[ClassTag[$T]]
+        def key = "ClassTag[" + ${T.key} + "]"
+      }
+    """
+  }
   def apply(key: c.Tree): c.Tree = {
     import c.universe._
     q"""scala.pickling.FastTypeTag(scala.pickling.internal.`package`.currentMirror, $key)"""

core/src/main/scala/pickling/Macros.scala

@@ -258,8 +258,26 @@ trait UnpicklerMacros extends Macro {
 
       val someConstructorIsPrivate = ctors.exists(_.isPrivate)
       // println(s"someConstructorIsPrivate: $someConstructorIsPrivate")
-      val canCallCtor = !someConstructorIsPrivate && !cir.fields.exists(_.isErasedParam) && isPreciseType
-      // println(s"canCallCtor: $canCallCtor")
+      val canCallCtor = !someConstructorIsPrivate && !cir.fields.exists(_.isErasedParam) && isPreciseType && {
+        // there must not be a transient ctor param
+        // STEP 1: we need to figure out if there is a transient ctor param
+        val primaryCtor = tpe.declaration(nme.CONSTRUCTOR) match {
+          case overloaded: TermSymbol => overloaded.alternatives.head.asMethod // NOTE: primary ctor is always the first in the list
+          case primaryCtor: MethodSymbol => primaryCtor
+          case NoSymbol => NoSymbol
+        }
+
+        val allAccessors = tpe.declarations.collect { case meth: MethodSymbol if meth.isAccessor || meth.isParamAccessor => meth }
+
+        val (filteredAccessors, transientAccessors) = allAccessors.partition(irs.notMarkedTransient) // shoulsai iaobjsobj aaaobjsecta insteeyd  m d daybe
+
+        val hasTransientParam = (primaryCtor != NoSymbol) && primaryCtor.asMethod.paramss.flatten.exists { sym =>
+          transientAccessors.exists(acc => acc.name.toString == sym.name.toString)
+        }
+
+        !hasTransientParam
+      }
+      // STEP 2: remove transient fields from the "pending fields", the fields that need to be restored.
 
       // TODO: for ultimate loop safety, pendingFields should be hoisted to the outermost unpickling scope
       // For example, in the snippet below, when unpickling C, we'll need to move the b.c assignment not

core/src/main/scala/pickling/ir/IRs.scala

@@ -24,7 +24,6 @@ class IRs[U <: Universe with Singleton](val uni: U) {
     // this part is interesting to unpicklers
     def hasSetter = setter.isDefined
     def isErasedParam = isParam && accessor.isEmpty // TODO: this should somehow communicate with the constructors phase!
-    def isReifiedParam = isParam && accessor.nonEmpty
     def isNonParam = !isParam
   }
   case class ClassIR(tpe: Type, parent: ClassIR, fields: List[FieldIR]) extends PickleIR
@@ -33,14 +32,33 @@ class IRs[U <: Universe with Singleton](val uni: U) {
   private type C = ClassIR
 
   // TODO: minimal versus verbose PickleFormat. i.e. someone might want all concrete inherited fields in their pickle
+
+  def notMarkedTransient(sym: TermSymbol): Boolean = {
+    val tr = scala.util.Try {
+      (sym.accessed == NoSymbol) || // if there is no backing field, then it cannot be marked transient
+      !sym.accessed.annotations.exists(_.tpe =:= typeOf[scala.transient])
+    }
+    tr.isFailure || tr.get
+  }
+
+  /** Creates FieldIRs for the given type, tp.
+  */
   private def fields(tp: Type): Q = {
     val ctor = tp.declaration(nme.CONSTRUCTOR) match {
       case overloaded: TermSymbol => overloaded.alternatives.head.asMethod // NOTE: primary ctor is always the first in the list
       case primaryCtor: MethodSymbol => primaryCtor
       case NoSymbol => NoSymbol
     }
-    val ctorParams = if (ctor != NoSymbol) ctor.asMethod.paramss.flatten.map(_.asTerm) else Nil
-    val allAccessors = tp.declarations.collect{ case meth: MethodSymbol if meth.isAccessor || meth.isParamAccessor => meth }
+
+    val allAccessors = tp.declarations.collect { case meth: MethodSymbol if meth.isAccessor || meth.isParamAccessor => meth }
+
+    val (filteredAccessors, transientAccessors) = allAccessors.partition(notMarkedTransient)
+
+    val ctorParams = if (ctor != NoSymbol) ctor.asMethod.paramss.flatten.flatMap { sym =>
+      if (transientAccessors.exists(acc => acc.name.toString == sym.name.toString)) List()
+      else List(sym.asTerm)
+    } else Nil
+
     val (paramAccessors, otherAccessors) = allAccessors.partition(_.isParamAccessor)
 
     def mkFieldIR(sym: TermSymbol, param: Option[TermSymbol], accessor: Option[MethodSymbol]) = {
@@ -62,7 +80,7 @@ class IRs[U <: Universe with Singleton](val uni: U) {
 
   private val f1 = (q1: Q, q2: Q) => q1 ++ q2
 
-  private val f2 = (c1: C, c2: C) => ClassIR(c2.tpe, c1, fields(c2.tpe))
+  private val f2 = (c1: C, c2: C) => ClassIR(c2.tpe, c1, c2.fields)
 
   private val f3 = (c: C) =>
     c.tpe.baseClasses

core/src/test/scala/pickling/run/implicit-params.scala

@@ -0,0 +1,21 @@
+package scala.pickling.implicitparams
+
+import org.scalatest.FunSuite
+import scala.pickling._
+import json._
+
+
+case class Person(implicit name: String, age: Int)
+object Test {
+  class SimpleImplParamTest extends FunSuite {
+    test("main") {
+      implicit val nme = "Harry"
+      implicit val age = 18
+
+      val per = Person()
+      val p = per.pickle
+      val up = p.unpickle[Person]
+      assert(per == up)
+    }
+  }
+}

core/src/test/scala/pickling/run/transient.scala

@@ -0,0 +1,49 @@
+package scala.pickling.transienttest
+
+import org.scalatest.FunSuite
+import scala.reflect.ClassTag
+import scala.pickling._
+import json._
+
+case class Person(val name: String , @transient val ssNumber: Int) {
+  override def toString = s"Person($name)"
+}
+
+class Dependency[T]
+
+class SparkConf(loadDefaults: Boolean)
+class SparkContext(config: SparkConf)
+
+class RDD[T: ClassTag](
+    @transient private var sc: SparkContext,
+    @transient private var deps: Seq[Dependency[_]]
+  )
+
+class RangePartitioner[K : ClassTag, V](
+    @transient val partitions: Int,
+    @transient val rdd: RDD[_ <: Product2[K,V]],
+    private var ascending: Boolean = true) {
+  override def toString = s"RangePartitioner(ascending = $ascending)"
+}
+
+class TransientSimpleTest extends FunSuite {
+  test("main") {
+    val per = Person("Jenny", 123)
+    val p: JSONPickle = per.pickle
+    val up = p.unpickle[Person]
+    assert(up.ssNumber == 0)
+    assert(per.toString == up.toString)
+  }
+}
+
+class TransientSparkTest extends FunSuite {
+  test("main") {
+    val sc = new SparkContext(new SparkConf(true))
+    val rdd = new RDD[(Int, Int)](sc, Seq(new Dependency()))
+    val rp = new RangePartitioner[Int, Int](2, rdd)
+    val p: JSONPickle = rp.pickle
+    val up = p.unpickle[RangePartitioner[Int, Int]]
+    assert(rp.toString == up.toString)
+  }
+}
+