diff --git a/tests/otTestRunner/App.config b/tests/otTestRunner/App.config deleted file mode 100644 index 88fa4027b..000000000 --- a/tests/otTestRunner/App.config +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/tests/otTestRunner/AssemblyInfo.cs b/tests/otTestRunner/AssemblyInfo.cs deleted file mode 100644 index c3f2b7cd3..000000000 --- a/tests/otTestRunner/AssemblyInfo.cs +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright (c) 2016, The OpenThread Authors. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. Neither the name of the copyright holder nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - - using System.Reflection; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; - -// General Information about an assembly is controlled through the following -// set of attributes. Change these attribute values to modify the information -// associated with an assembly. -[assembly: AssemblyTitle("otTestRunner")] -[assembly: AssemblyDescription("")] -[assembly: AssemblyConfiguration("")] -[assembly: AssemblyCompany("")] -[assembly: AssemblyProduct("otTestRunner")] -[assembly: AssemblyCopyright("Copyright © The OpenThread Authors 2016")] -[assembly: AssemblyTrademark("")] -[assembly: AssemblyCulture("")] - -// Setting ComVisible to false makes the types in this assembly not visible -// to COM components. If you need to access a type in this assembly from -// COM, set the ComVisible attribute to true on that type. -[assembly: ComVisible(false)] - -// The following GUID is for the ID of the typelib if this project is exposed to COM -[assembly: Guid("d5577e51-fa31-4802-8669-1db32805935e")] - -// Version information for an assembly consists of the following four values: -// -// Major Version -// Minor Version -// Build Number -// Revision -// -// You can specify all the values or you can default the Build and Revision Numbers -// by using the '*' as shown below: -// [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("1.0.0.0")] -[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/tests/otTestRunner/Program.cs b/tests/otTestRunner/Program.cs deleted file mode 100644 index eb57fd1dc..000000000 --- a/tests/otTestRunner/Program.cs +++ /dev/null @@ -1,392 +0,0 @@ -/* - * Copyright (c) 2016, The OpenThread Authors. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. Neither the name of the copyright holder nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.IO; -using System.Net; -using System.Text; -using System.Threading; -using System.Threading.Tasks; - -namespace otTestRunner -{ - class Program - { - struct TestResults - { - public bool Pass; - public List Output; - public string Error; - } - - static string EscapeJson(string data) - { - return data.Replace("\\", "\\\\").Replace("\"", "\\\""); - } - - /// - /// Executes an exe with the given args and captures the output - /// - static async Task ExecuteAsync(string name, string args = null, int timeoutMilliseconds = -1, int instanceIndex = -1) - { - ProcessStartInfo startInfo = new ProcessStartInfo(); - startInfo.WindowStyle = ProcessWindowStyle.Hidden; - startInfo.CreateNoWindow = true; - startInfo.UseShellExecute = false; - startInfo.RedirectStandardError = true; - startInfo.RedirectStandardOutput = true; - startInfo.FileName = name; - startInfo.Arguments = args; - - startInfo.EnvironmentVariables["NODE_TYPE"] = "win-sim"; - - if (instanceIndex != -1) - { - startInfo.EnvironmentVariables["INSTANCE"] = instanceIndex.ToString(); - } - - TestResults Results = new TestResults(); - Results.Pass = false; - Results.Output = new List(); - Results.Error = null; - var Errors = new List(); - - Results.Output.Add(string.Format("> set NODE_TYPE=win-sim")); - Results.Output.Add(string.Format("> {0} {1}", name, args)); - Results.Output.Add("----------------------------------------------------------------------"); - - try - { - // Execute process - using (Process process = Process.Start(startInfo)) - { - process.OutputDataReceived += - (object sender, DataReceivedEventArgs e) => { - if (e.Data != null && e.Data.Length > 0) - lock (Results.Output) { Results.Output.Add(e.Data); } - }; - process.ErrorDataReceived += - (object sender, DataReceivedEventArgs e) => { - if (e.Data != null && e.Data.Length > 0) - lock (Results.Output) { Errors.Add(e.Data); } - }; - - process.BeginErrorReadLine(); - process.BeginOutputReadLine(); - -#if DEBUG - Console.WriteLine("Starting {0} {1}", name, args); -#endif - - // Wait for process to complete - await Task.Run( - () => { - if (timeoutMilliseconds == -1) - process.WaitForExit(); - else if (!process.WaitForExit(timeoutMilliseconds)) - { - process.Kill(); - Results.Output.Add(string.Format("Killed {0} on execution timeout!", name)); - } - }); - - // Wait a bit for any output to collect - await Task.Delay(1000); - - process.CancelOutputRead(); - process.CancelErrorRead(); - - Results.Output.AddRange(Errors); - Results.Pass = Errors.Count > 0 && Errors[Errors.Count - 1] == "OK"; - - if (!Results.Pass) Results.Error = EscapeJson(string.Join("\\r\\n", Errors)); - - // Make sure the process is killed - try { process.Kill(); } catch (Exception) { } - -#if DEBUG - Console.WriteLine("Completed {0} {1}", name, args); -#endif - } - } - catch (Exception e) - { - Results.Output.Add("Encountered exception: " + e.Message); - Results.Output.Add(e.StackTrace); - } - - return Results; - } - - static bool VerboseOutput = false; - static int RetiresOnFailure = 0; - static bool AppVeyorMode = false; - static string AppVeyorApiUrl = null; - static string ResultsFolder = "Results_" + DateTime.Now.ToString("yyyyMMdd_HH.mm.ss"); - - static void UploadAppVeyorTestResult(string name, bool passed, long durationMS, string error = null) - { - if (AppVeyorApiUrl == null) return; - - string jsonData = null; - - try - { - var request = (HttpWebRequest)WebRequest.Create(Path.Combine(AppVeyorApiUrl, "api/tests")); - - jsonData = - string.Format( - "{{" + - "\"testName\": \"{0}\", " + - "\"testFramework\": \"MSTest\", " + - "\"fileName\": \"{0}.py\", " + - "\"outcome\": \"{1}\", " + - "\"durationMilliseconds\": \"{2}\", " + - "\"ErrorMessage\": \"{3}\"" + - "}}", - name, - passed ? "Passed" : "Failed", - durationMS, - error == null ? "" : error - ); - - var data = Encoding.UTF8.GetBytes(jsonData); - - request.Method = "POST"; - request.ContentType = "application/json"; - request.ContentLength = data.Length; - - using (var stream = request.GetRequestStream()) - { - stream.Write(data, 0, data.Length); - } - - var response = (HttpWebResponse)request.GetResponse(); - var responseString = new StreamReader(response.GetResponseStream()).ReadToEnd(); - } - catch (Exception e) - { - Console.WriteLine("Encountered exception for http post:"); - Console.WriteLine(e.Message); - Console.WriteLine(e.StackTrace); - - Console.WriteLine("Json content:"); - Console.WriteLine(jsonData); - } - } - - enum TestResult - { - Fail, - Pass, - PassWithRetry - } - - /// - /// Runs a python test file and returns success/failure - /// - static async Task RunTest(string file, int index) - { - string pythonPath = "python.exe"; - if (AppVeyorMode) - { - if (Environment.GetEnvironmentVariable("Platform").ToLower() == "x64") - { - pythonPath = @"c:\python35-x64\python.exe"; - } - else - { - pythonPath = @"c:\python35\python.exe"; - } - } - - int tries = 0; - Stopwatch Timer; - TestResults Results; - - do - { - Timer = new Stopwatch(); - Timer.Start(); - - Results = await ExecuteAsync(pythonPath, file, 30 * 60 * 1000, index); - - Timer.Stop(); - - } while (++tries < RetiresOnFailure + 1 && Results.Pass == false); - - if (VerboseOutput) - { - lock (ResultsFolder) - { - foreach (var line in Results.Output) - Console.WriteLine(line); - } - } - - UploadAppVeyorTestResult(Path.GetFileNameWithoutExtension(file), Results.Pass, Timer.ElapsedMilliseconds, Results.Error); - - // Write the output to a file - var filePrefix = Results.Pass ? "P_" : "F_"; - var outputFilePath = Path.Combine(ResultsFolder, filePrefix + Path.GetFileNameWithoutExtension(file) + ".txt"); - try { - File.WriteAllLines(outputFilePath, Results.Output); - } catch (Exception e) { - Console.WriteLine("Exception while trying to write {0}:\n{1}!", outputFilePath, e.Message); - } - - return Results.Pass ? - (tries == 1 ? TestResult.Pass : TestResult.PassWithRetry) : - TestResult.Fail; - } - - /// - /// Runs all the tests as indicated by input arguments - /// - static void Main(string[] args) - { - if (args.Length < 2) - { - Console.WriteLine("Usage: otTestRunner.exe [path] [search pattern] (parallel:n) (verbose)"); - return; - } - - var files = Directory.GetFiles(args[0], args[1]); - if (files.Length == 0) - { - Console.WriteLine("No tests found with that path & pattern!"); - return; - } - - var NumberOfTestsToRunInParallel = 1; - for (var i = 2; i < args.Length; i++) - { - if (args[i].StartsWith("parallel:")) - NumberOfTestsToRunInParallel = int.Parse(args[i].Substring(9)); - else if (args[i].StartsWith("retry:")) - RetiresOnFailure = int.Parse(args[i].Substring(6)); - else if (args[i].StartsWith("verbose")) - VerboseOutput = true; - else if (args[i].StartsWith("appveyor")) - { - AppVeyorMode = true; - AppVeyorApiUrl = Environment.GetEnvironmentVariable("APPVEYOR_API_URL"); - //Console.WriteLine("AppVeyorApiUrl = {0}", AppVeyorApiUrl); - } - } - - var CurNumTestsRunning = 0; - var ReadyToRunEvent = new ManualResetEvent(true); - - var TestPassCount = 0; - Stopwatch Timer = new Stopwatch(); - - Directory.CreateDirectory(ResultsFolder); - Console.WriteLine("Test results saved: .\\{0}", ResultsFolder); - - Console.WriteLine("Running {0} tests, {1} at a time:", files.Length, NumberOfTestsToRunInParallel); - /*for (var i = 0; i < files.Length; i++) - Console.WriteLine(Path.GetFileName(files[i]));*/ - Console.WriteLine(""); - - Timer.Start(); - for (var i = 0; i < files.Length; i++) - { - // Wait for the event to be set, if not already - ReadyToRunEvent.WaitOne(); - - if (i != 0) - { - // Wait a bit to stagger the starts - Task.Delay(1000).Wait(); - } - - lock (ResultsFolder) - { - if (++CurNumTestsRunning == NumberOfTestsToRunInParallel) - ReadyToRunEvent.Reset(); - } - - var index = i; - var fileName = files[i]; - - // Start the test, but don't wait for it - Task.Run( - async () => - { - var result = await RunTest(fileName, index); - lock (ResultsFolder) - { - var PrevColor = Console.ForegroundColor; - if (result != TestResult.Fail) - { - if (result == TestResult.Pass) - { - Console.ForegroundColor = ConsoleColor.Green; - } - else - { - Console.ForegroundColor = ConsoleColor.Yellow; - } - Console.Write("PASS"); - TestPassCount++; - } - else - { - Console.ForegroundColor = ConsoleColor.Red; - Console.Write("FAIL"); - } - Console.ForegroundColor = PrevColor; - Console.WriteLine(": {0}", Path.GetFileNameWithoutExtension(fileName)); - - CurNumTestsRunning--; - ReadyToRunEvent.Set(); - } - }); - } - - // Wait for all the tests to complete - while (CurNumTestsRunning != 0) - ReadyToRunEvent.WaitOne(); - - Timer.Stop(); - TimeSpan ts = Timer.Elapsed; - string elapsedTime = - String.Format("{0:00}:{1:00}:{2:00}.{3:00}", - ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds / 10); - - Console.WriteLine("{0} tests run in {1}", files.Length, elapsedTime); - Console.WriteLine("{0} passed and {1} failed", TestPassCount, files.Length - TestPassCount); - - if (!AppVeyorMode) - Environment.ExitCode = files.Length == TestPassCount ? 0 : 1; - } - } -}