problems with test sync pushed changes to segregate client side of in…

…tegration tests into 2 separate apps for RPC and WWS, plus the order of tests and timing facilities have been improved in order to better use collected data and provide better estimates for clients

3FD.sln

@@ -57,7 +57,9 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "IntegrationTestsApp.WinRT.W
 EndProject
 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "IntegrationTestsApp.WinRT.WindowsPhone", "IntegrationTestsApp.WinRT\IntegrationTestsApp.WinRT.WindowsPhone\IntegrationTestsApp.WinRT.WindowsPhone.vcxproj", "{CEB9F9FD-5E2A-49FB-AC14-2DEF01DC80B8}"
 EndProject
-Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "TestClient", "TestClient\TestClient.vcxproj", "{6ADF3A4B-1E22-41BA-9293-96593E6A45BE}"
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "TestRpcClient", "TestRpcClient\TestRpcClient.vcxproj", "{6ADF3A4B-1E22-41BA-9293-96593E6A45BE}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "TestWwsClient", "TestWwsClient\TestWwsClient.vcxproj", "{4913E097-136E-4BC4-BF8C-D3341FB1425A}"
 EndProject
 Global
 	GlobalSection(SharedMSBuildProjectFiles) = preSolution
@@ -291,6 +293,20 @@ Global
 		{6ADF3A4B-1E22-41BA-9293-96593E6A45BE}.Release|Win32.Build.0 = Release|Win32
 		{6ADF3A4B-1E22-41BA-9293-96593E6A45BE}.Release|x64.ActiveCfg = Release|x64
 		{6ADF3A4B-1E22-41BA-9293-96593E6A45BE}.Release|x64.Build.0 = Release|x64
+		{4913E097-136E-4BC4-BF8C-D3341FB1425A}.Debug|ARM.ActiveCfg = Debug|Win32
+		{4913E097-136E-4BC4-BF8C-D3341FB1425A}.Debug|Mixed Platforms.ActiveCfg = Debug|Win32
+		{4913E097-136E-4BC4-BF8C-D3341FB1425A}.Debug|Mixed Platforms.Build.0 = Debug|Win32
+		{4913E097-136E-4BC4-BF8C-D3341FB1425A}.Debug|Win32.ActiveCfg = Debug|Win32
+		{4913E097-136E-4BC4-BF8C-D3341FB1425A}.Debug|Win32.Build.0 = Debug|Win32
+		{4913E097-136E-4BC4-BF8C-D3341FB1425A}.Debug|x64.ActiveCfg = Debug|x64
+		{4913E097-136E-4BC4-BF8C-D3341FB1425A}.Debug|x64.Build.0 = Debug|x64
+		{4913E097-136E-4BC4-BF8C-D3341FB1425A}.Release|ARM.ActiveCfg = Release|Win32
+		{4913E097-136E-4BC4-BF8C-D3341FB1425A}.Release|Mixed Platforms.ActiveCfg = Release|Win32
+		{4913E097-136E-4BC4-BF8C-D3341FB1425A}.Release|Mixed Platforms.Build.0 = Release|Win32
+		{4913E097-136E-4BC4-BF8C-D3341FB1425A}.Release|Win32.ActiveCfg = Release|Win32
+		{4913E097-136E-4BC4-BF8C-D3341FB1425A}.Release|Win32.Build.0 = Release|Win32
+		{4913E097-136E-4BC4-BF8C-D3341FB1425A}.Release|x64.ActiveCfg = Release|x64
+		{4913E097-136E-4BC4-BF8C-D3341FB1425A}.Release|x64.Build.0 = Release|x64
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -315,6 +331,7 @@ Global
 		{09ED5B36-D345-4D62-9F01-DC41168BCB35} = {35A12F43-25FF-406A-8CCF-B203F688A791}
 		{CEB9F9FD-5E2A-49FB-AC14-2DEF01DC80B8} = {35A12F43-25FF-406A-8CCF-B203F688A791}
 		{6ADF3A4B-1E22-41BA-9293-96593E6A45BE} = {8E00B911-43B2-49D5-9B4A-AB6E1ED097A1}
+		{4913E097-136E-4BC4-BF8C-D3341FB1425A} = {8E00B911-43B2-49D5-9B4A-AB6E1ED097A1}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		VisualSVNWorkingCopyRoot = .

IntegrationTests/IntegrationTests.vcxproj

@@ -107,7 +107,7 @@
     </Link>
     <PostBuildEvent>
       <Command>copy "$(ProjectDir)application.config" "$(TargetPath).3fd.config"
-copy "$(ProjectDir)calculator.wsdl" "$(TargetDir)"</Command>
+copy "$(SolutionDir)TestShared\calculator.wsdl" "$(TargetDir)"</Command>
     </PostBuildEvent>
   </ItemDefinitionGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
@@ -127,7 +127,7 @@ copy "$(ProjectDir)calculator.wsdl" "$(TargetDir)"</Command>
     </Link>
     <PostBuildEvent>
       <Command>copy "$(ProjectDir)application.config" "$(TargetPath).3fd.config"
-copy "$(ProjectDir)calculator.wsdl" "$(TargetDir)"</Command>
+copy "$(SolutionDir)TestShared\calculator.wsdl" "$(TargetDir)"</Command>
     </PostBuildEvent>
   </ItemDefinitionGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
@@ -151,7 +151,7 @@ copy "$(ProjectDir)calculator.wsdl" "$(TargetDir)"</Command>
     </Link>
     <PostBuildEvent>
       <Command>copy "$(ProjectDir)application.config" "$(TargetPath).3fd.config"
-copy "$(ProjectDir)calculator.wsdl" "$(TargetDir)"</Command>
+copy "$(SolutionDir)TestShared\calculator.wsdl" "$(TargetDir)"</Command>
     </PostBuildEvent>
   </ItemDefinitionGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
@@ -175,7 +175,7 @@ copy "$(ProjectDir)calculator.wsdl" "$(TargetDir)"</Command>
     </Link>
     <PostBuildEvent>
       <Command>copy "$(ProjectDir)application.config" "$(TargetPath).3fd.config"
-copy "$(ProjectDir)calculator.wsdl" "$(TargetDir)"</Command>
+copy "$(SolutionDir)TestShared\calculator.wsdl" "$(TargetDir)"</Command>
     </PostBuildEvent>
   </ItemDefinitionGroup>
   <ItemGroup>

IntegrationTests/tests_rpc.cpp

@@ -220,7 +220,7 @@ namespace integration_tests
 
             auto shutdownTimeSpan = duration_cast<milliseconds>(
                 now - startTimeForSrvShutdown
-                );
+            );
 
             if (shutdownTimeSpan > maxTimeSpanForSrvShutdown)
                 maxTimeSpanForSrvShutdown = shutdownTimeSpan;
@@ -246,12 +246,15 @@ namespace integration_tests
         /// </return>
         static uint32_t GetMaxCycleTime()
         {
-            auto maxCycleTime = static_cast<uint32_t> (
-                maxTimeSpanForSrvShutdown.count()
-                + maxTimeSpanForSrvSetupAndStart.count()
-            );
-
-            return maxCycleTime > 0 ? maxCycleTime : 32U;
+            auto maxCycleTime =
+                maxTimeSpanForSrvShutdown.count() +
+                maxTimeSpanForSrvSetupAndStart.count();
+
+            /* In practice, measured time must be linearly augmented
+            for adjustment (using field data), because apparently the
+            server takes longer to be available, which is a little
+            after RpcServer::Start returns... */
+            return maxCycleTime > 0 ? static_cast<uint32_t> (100 + 1.2 * maxCycleTime) : 150;
         }
     };
 
@@ -325,6 +328,58 @@ namespace integration_tests
         }
     }
 
+    class Framework_RpcNoAuth2_TestCase : public ::testing::Test {};
+
+    /// <summary>
+    /// Tests the RPC server normal operation (responding requests), trying
+    /// several combinations of protocol sequence and authentication level.
+    /// </summary>
+    TEST_F(Framework_RpcNoAuth2_TestCase, ServerRun_ResponseTest)
+    {
+        // Ensures proper initialization/finalization of the framework
+        FrameworkInstance _framework;
+
+        CALL_STACK_TRACE;
+
+        try
+        {
+            // Initialize the RPC server (resource allocation takes place)
+            RpcServer::Initialize(ProtocolSequence::Local, "TestClient3FD");
+
+            // RPC interface implementation 1:
+            AcmeTesting_v1_0_epv_t intfImplFuncTable1 = { Operate, ChangeCase, WriteOnStorage, Shutdown };
+
+            // RPC interface implementation 2:
+            AcmeTesting_v1_0_epv_t intfImplFuncTable2 = { Operate2, ChangeCase2, WriteOnStorage, Shutdown };
+
+            std::vector<RpcSrvObject> objects;
+            objects.reserve(2);
+
+            // This object will run impl 1:
+            objects.emplace_back(
+                objectsUuidsImpl1[5],
+                AcmeTesting_v1_0_s_ifspec, // this is the interface (generated from IDL)
+                &intfImplFuncTable1
+            );
+
+            // This object will run impl 2:
+            objects.emplace_back(
+                objectsUuidsImpl2[5],
+                AcmeTesting_v1_0_s_ifspec, // this is the interface (generated from IDL)
+                &intfImplFuncTable2
+            );
+
+            EXPECT_EQ(STATUS_OKAY, RpcServer::Start(objects));
+            EXPECT_EQ(STATUS_OKAY, RpcServer::Wait());
+            EXPECT_EQ(STATUS_OKAY, RpcServer::Finalize());
+        }
+        catch (...)
+        {
+            RpcServer::Finalize();
+            HandleException();
+        }
+    }
+
     // The set of options for some test template instantiations
     struct AuthnTestOptions
     {
@@ -418,13 +473,15 @@ namespace integration_tests
         )
     );
 
-    class Framework_RpcNoAuth2_TestCase : public ::testing::Test {};
+    class Framework_RpcAuthn2_TestCase :
+        public ::testing::TestWithParam<AuthnTestOptions> {};
 
     /// <summary>
     /// Tests the RPC server normal operation (responding requests), trying
-    /// several combinations of protocol sequence and authentication level.
+    /// several combinations of protocol sequence and authentication level
+    /// using Microsoft NTLM/Negotiate/Kerberos SSP's.
     /// </summary>
-    TEST_F(Framework_RpcNoAuth2_TestCase, ServerRun_ResponseTest)
+    TEST_P(Framework_RpcAuthn2_TestCase, ServerRun_ResponseTest)
     {
         // Ensures proper initialization/finalization of the framework
         FrameworkInstance _framework;
@@ -433,8 +490,12 @@ namespace integration_tests
 
         try
         {
-            // Initialize the RPC server (resource allocation takes place)
-            RpcServer::Initialize(ProtocolSequence::Local, "TestClient3FD");
+            // Initialize the RPC server (authn svc reg & resource allocation takes place)
+            RpcServer::Initialize(
+                GetParam().protocolSequence,
+                "TestClient3FD",
+                GetParam().authenticationLevel
+            );
 
             // RPC interface implementation 1:
             AcmeTesting_v1_0_epv_t intfImplFuncTable1 = { Operate, ChangeCase, WriteOnStorage, Shutdown };
@@ -447,14 +508,14 @@ namespace integration_tests
 
             // This object will run impl 1:
             objects.emplace_back(
-                objectsUuidsImpl1[5],
+                GetParam().objectUUID1,
                 AcmeTesting_v1_0_s_ifspec, // this is the interface (generated from IDL)
                 &intfImplFuncTable1
             );
 
             // This object will run impl 2:
             objects.emplace_back(
-                objectsUuidsImpl2[5],
+                GetParam().objectUUID2,
                 AcmeTesting_v1_0_s_ifspec, // this is the interface (generated from IDL)
                 &intfImplFuncTable2
             );
@@ -470,15 +531,42 @@ namespace integration_tests
         }
     }
 
-    class Framework_RpcAuthn2_TestCase :
-        public ::testing::TestWithParam<AuthnTestOptions> {};
+    /* Implementation of test template takes care of switching
+    protocol sequences and authentication level: */
+    INSTANTIATE_TEST_CASE_P(
+        SwitchProtAndAuthLevel,
+        Framework_RpcAuthn2_TestCase,
+        ::testing::Values(
+            AuthnTestOptions{ ProtocolSequence::Local, objectsUuidsImpl1[6], objectsUuidsImpl2[6], AuthenticationLevel::Integrity },
+            AuthnTestOptions{ ProtocolSequence::Local, objectsUuidsImpl1[7], objectsUuidsImpl2[7], AuthenticationLevel::Privacy },
+            AuthnTestOptions{ ProtocolSequence::Local, objectsUuidsImpl1[8], objectsUuidsImpl2[8], AuthenticationLevel::Integrity },
+            AuthnTestOptions{ ProtocolSequence::Local, objectsUuidsImpl1[9], objectsUuidsImpl2[9], AuthenticationLevel::Privacy }
+            /*,
+            AuthnTestOptions{ ProtocolSequence::Local, objectsUuidsImpl1[10], objectsUuidsImpl2[10], AuthenticationLevel::Integrity },
+            AuthnTestOptions{ ProtocolSequence::Local, objectsUuidsImpl1[11], objectsUuidsImpl2[11], AuthenticationLevel::Privacy },
+            */
+            /*
+            AuthnTestOptions{ ProtocolSequence::TCP, objectsUuidsImpl1[6], objectsUuidsImpl2[6], AuthenticationLevel::Integrity },
+            AuthnTestOptions{ ProtocolSequence::TCP, objectsUuidsImpl1[7], objectsUuidsImpl2[7], AuthenticationLevel::Privacy },
+            AuthnTestOptions{ ProtocolSequence::TCP, objectsUuidsImpl1[8], objectsUuidsImpl2[8], AuthenticationLevel::Integrity },
+            AuthnTestOptions{ ProtocolSequence::TCP, objectsUuidsImpl1[9], objectsUuidsImpl2[9], AuthenticationLevel::Privacy }
+            */
+            /*,
+            AuthnTestOptions{ ProtocolSequence::TCP, objectsUuidsImpl1[10], objectsUuidsImpl2[10], AuthenticationLevel::Integrity },
+            AuthnTestOptions{ ProtocolSequence::TCP, objectsUuidsImpl1[11], objectsUuidsImpl2[11], AuthenticationLevel::Privacy }
+            */
+        )
+    );
+
+    class Framework_RpcSchannel_TestCase :
+        public ::testing::TestWithParam<SchannelTestOptions> {};
 
     /// <summary>
-    /// Tests the RPC server normal operation (responding requests), trying
-    /// several combinations of protocol sequence and authentication level
-    /// using Microsoft NTLM/Negotiate/Kerberos SSP's.
+    /// Tests the cycle init/start/stop/resume/stop/finalize of the RPC server,
+    /// for several combinations of protocol sequence and authentication level
+    /// using Schannel SSP.
     /// </summary>
-    TEST_P(Framework_RpcAuthn2_TestCase, ServerRun_ResponseTest)
+    TEST_P(Framework_RpcSchannel_TestCase, ServerRun_StatesCycleTest)
     {
         // Ensures proper initialization/finalization of the framework
         FrameworkInstance _framework;
@@ -487,10 +575,17 @@ namespace integration_tests
 
         try
         {
-            // Initialize the RPC server (authn svc reg & resource allocation takes place)
+            CertInfo certInfo(
+                CERT_SYSTEM_STORE_LOCAL_MACHINE,
+                "My",
+                "MySelfSignedCert4DevTestsServer",
+                GetParam().useStrongSec
+            );
+
+            // Initialize the RPC server (resource allocation takes place)
             RpcServer::Initialize(
-                GetParam().protocolSequence,
                 "TestClient3FD",
+                &certInfo,
                 GetParam().authenticationLevel
             );
 
@@ -517,9 +612,21 @@ namespace integration_tests
                 &intfImplFuncTable2
             );
 
+            // Now cycle through the states:
+
             EXPECT_EQ(STATUS_OKAY, RpcServer::Start(objects));
-            EXPECT_EQ(STATUS_OKAY, RpcServer::Wait());
+            RpcTestTimer::StopTimeCountServerSetupAndStart();
+
+            EXPECT_EQ(STATUS_OKAY, RpcServer::Stop());
+            EXPECT_EQ(STATUS_OKAY, RpcServer::Resume());
+
+            RpcTestTimer::StartTimeCountServerShutdown();
+
+            // Upon finalization (shutdown), resources will be released:
+            EXPECT_EQ(STATUS_OKAY, RpcServer::Stop());
             EXPECT_EQ(STATUS_OKAY, RpcServer::Finalize());
+
+            RpcTestTimer::StopTimeCountServerShutdown();
         }
         catch (...)
         {
@@ -532,47 +639,24 @@ namespace integration_tests
     protocol sequences and authentication level: */
     INSTANTIATE_TEST_CASE_P(
         SwitchProtAndAuthLevel,
-        Framework_RpcAuthn2_TestCase,
+        Framework_RpcSchannel_TestCase,
         ::testing::Values(
-            AuthnTestOptions{ ProtocolSequence::Local, objectsUuidsImpl1[6], objectsUuidsImpl2[6], AuthenticationLevel::Integrity },
-            AuthnTestOptions{ ProtocolSequence::Local, objectsUuidsImpl1[7], objectsUuidsImpl2[7], AuthenticationLevel::Privacy },
-            AuthnTestOptions{ ProtocolSequence::Local, objectsUuidsImpl1[8], objectsUuidsImpl2[8], AuthenticationLevel::Integrity },
-            AuthnTestOptions{ ProtocolSequence::Local, objectsUuidsImpl1[9], objectsUuidsImpl2[9], AuthenticationLevel::Privacy }
-            /*,
-            AuthnTestOptions{ ProtocolSequence::Local, objectsUuidsImpl1[10], objectsUuidsImpl2[10], AuthenticationLevel::Integrity },
-            AuthnTestOptions{ ProtocolSequence::Local, objectsUuidsImpl1[11], objectsUuidsImpl2[11], AuthenticationLevel::Privacy },
-            */
-            /*
-            AuthnTestOptions{ ProtocolSequence::TCP, objectsUuidsImpl1[6], objectsUuidsImpl2[6], AuthenticationLevel::Integrity },
-            AuthnTestOptions{ ProtocolSequence::TCP, objectsUuidsImpl1[7], objectsUuidsImpl2[7], AuthenticationLevel::Privacy },
-            AuthnTestOptions{ ProtocolSequence::TCP, objectsUuidsImpl1[8], objectsUuidsImpl2[8], AuthenticationLevel::Integrity },
-            AuthnTestOptions{ ProtocolSequence::TCP, objectsUuidsImpl1[9], objectsUuidsImpl2[9], AuthenticationLevel::Privacy }
-            */
-            /*,
-            AuthnTestOptions{ ProtocolSequence::TCP, objectsUuidsImpl1[10], objectsUuidsImpl2[10], AuthenticationLevel::Integrity },
-            AuthnTestOptions{ ProtocolSequence::TCP, objectsUuidsImpl1[11], objectsUuidsImpl2[11], AuthenticationLevel::Privacy }
-            */
+            SchannelTestOptions{ objectsUuidsImpl1[12], objectsUuidsImpl2[12], AuthenticationLevel::Privacy, false },
+            SchannelTestOptions{ objectsUuidsImpl1[13], objectsUuidsImpl2[13], AuthenticationLevel::Privacy, true },
+            SchannelTestOptions{ objectsUuidsImpl1[14], objectsUuidsImpl2[14], AuthenticationLevel::Integrity, false },
+            SchannelTestOptions{ objectsUuidsImpl1[15], objectsUuidsImpl2[15], AuthenticationLevel::Integrity, true }
         )
     );
 
-    // The set of options for some test template instantiations
-    struct SchannelTestOptions
-    {
-        const char *objectUUID1;
-        const char *objectUUID2;
-        AuthenticationLevel authenticationLevel;
-        bool useStrongSec;
-    };
-
-    class Framework_RpcSchannel_TestCase :
+    class Framework_RpcSchannel2_TestCase :
         public ::testing::TestWithParam<SchannelTestOptions> {};
 
     /// <summary>
     /// Tests the RPC server normal operation (responding requests), trying
     /// several combinations of protocol sequence and authentication level
     /// using Schannel SSP.
     /// </summary>
-    TEST_P(Framework_RpcSchannel_TestCase, ServerRun_Schannel_ResponseTest)
+    TEST_P(Framework_RpcSchannel2_TestCase, ServerRun_ResponseTest)
     {
         // Ensures proper initialization/finalization of the framework
         FrameworkInstance _framework;
@@ -629,18 +713,13 @@ namespace integration_tests
         }
     }
 
-    /* Implementation of test template takes care of switching
-    protocol sequences and authentication level: */
+    // Implementation of test template takes care of switching parameters:
     INSTANTIATE_TEST_CASE_P(
         SwitchProtAndAuthLevel,
-        Framework_RpcSchannel_TestCase,
-        ::testing::Values(
-            SchannelTestOptions{ objectsUuidsImpl1[12], objectsUuidsImpl2[12], AuthenticationLevel::Privacy, false },
-            SchannelTestOptions{ objectsUuidsImpl1[13], objectsUuidsImpl2[13], AuthenticationLevel::Privacy, true },
-            SchannelTestOptions{ objectsUuidsImpl1[14], objectsUuidsImpl2[14], AuthenticationLevel::Integrity, false },
-            SchannelTestOptions{ objectsUuidsImpl1[15], objectsUuidsImpl2[15], AuthenticationLevel::Integrity, true }
-        )
+        Framework_RpcSchannel2_TestCase,
+        paramsForSchannelTests // shared by both client & server side
     );
+
 }// end of namespace integration_tests
 }// end of namespace _3fd