From 5770212d59d02e348535b287a8792d509ac25764 Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Mon, 22 Sep 2014 22:45:56 +0700 Subject: [PATCH 1/2] SourceFileRequest transport-only changes --- DistCL.Core/Compiler.cs | 148 ++++++++++-------- DistCL.Core/Contracts/ICompiler.cs | 93 +++++++---- DistCL.Core/Contracts/ILocalCompiler.cs | 4 +- DistCL.Core/DistCL.Core.csproj | 2 +- DistCL.Core/Proxies/AgentProxy.cs | 19 ++- ...eCompilerService.CompileResult.datasource} | 4 +- .../RemoteCompilerService/Reference.cs | 146 +++++++++++------ .../RemoteCompilerService/Reference.svcmap | 2 +- .../RemoteCompilerService/bindings.wsdl | 8 +- .../configuration.svcinfo | 2 +- .../configuration91.svcinfo | 10 +- .../RemoteCompilerService/distcl.wsdl | 5 +- .../RemoteCompilerService/distcl.xsd | 3 +- .../RemoteCompilerService/messages1.xsd | 30 +++- .../RemoteCompilerService/service.wsdl | 12 -- DistCL.Service/DistCL.Service.csproj | 3 - 16 files changed, 307 insertions(+), 184 deletions(-) rename DistCL.Core/Service References/RemoteCompilerService/{DistCL.RemoteCompilerService.CompileStatus.datasource => DistCL.RemoteCompilerService.CompileResult.datasource} (80%) diff --git a/DistCL.Core/Compiler.cs b/DistCL.Core/Compiler.cs index 96aa460..d5816da 100644 --- a/DistCL.Core/Compiler.cs +++ b/DistCL.Core/Compiler.cs @@ -178,81 +178,97 @@ LocalCompileOutput ILocalCompiler.LocalCompile(LocalCompileInput input) }); } - using (var inputStream = File.OpenRead(input.Src)) + string compilationToken = null; + var inputFileName = input.Src; + + string agentName; + var remoteCompiler = CompilerServices.AgentPool.GetRandomCompiler(input.CompilerVersion, out agentName); + var compilerSearchDuration = stopwatch.Elapsed; + + while (true) { - var remoteInput = new CompileInput + using (var inputStream = File.OpenRead(inputFileName)) { - CompilerVersion = input.CompilerVersion, - Arguments = input.Arguments, - Src = inputStream, - SrcLength = inputStream.Length, - SrcName = input.SrcName - }; - - string agentName; - var remoteCompiler = CompilerServices.AgentPool.GetRandomCompiler(input.CompilerVersion, out agentName); - var compilerSearchDuration = stopwatch.Elapsed; - using (var remoteOutput = remoteCompiler.Compile(remoteInput)) - { - var remoteStreams = new Dictionary(); - var cookies = new List(); - foreach (var artifact in remoteOutput.Status.Cookies) - { - switch (artifact.Type) + var remoteInput = new CompileInput { - case CompileArtifactType.Out: - case CompileArtifactType.Err: - var artifactStream = new MemoryStream(); - remoteStreams.Add(artifact.Type, artifactStream); - cookies.Add(artifact); - break; - - default: - var fileStream = File.Open( - Path.Combine(Path.GetDirectoryName(input.Src), artifact.Name), - FileMode.Create, - FileAccess.Write, - FileShare.None); - remoteStreams.Add(artifact.Type, fileStream); - break; - } - } - CompileResultHelper.Unpack(remoteOutput.ResultData, remoteOutput.Status.Cookies, remoteStreams); - - var localStreams = new Dictionary(); - var localFiles = new List(); - foreach (var artifact in remoteOutput.Status.Cookies) + CompilerVersion = input.CompilerVersion, + Arguments = input.Arguments, + Src = inputStream, + SrcLength = inputStream.Length, + SrcName = input.SrcName, + CompilationToken = compilationToken + }; + + using (var remoteOutput = remoteCompiler.Compile(remoteInput)) { - var stream = remoteStreams[artifact.Type]; - switch (artifact.Type) + if (remoteOutput.Result.Finished) { - case CompileArtifactType.Out: - case CompileArtifactType.Err: - stream.Position = 0; - localStreams.Add(artifact, stream); - break; - - default: - stream.Close(); - localFiles.Add(artifact); - break; - } - } - - LocalLogger.InfoFormat("Completed local compile '{0}'", input.SrcName); - stopwatch.Stop(); - LocalLogger.DebugFormat(@"Stats for '{0}': + var compileResult = (FinishedCompileResult) remoteOutput.Result; + var remoteStreams = new Dictionary(); + var cookies = new List(); + foreach (var artifact in compileResult.Cookies) + { + switch (artifact.Type) + { + case CompileArtifactType.Out: + case CompileArtifactType.Err: + var artifactStream = new MemoryStream(); + remoteStreams.Add(artifact.Type, artifactStream); + cookies.Add(artifact); + break; + + default: + var fileStream = File.Open( + Path.Combine(Path.GetDirectoryName(input.Src), artifact.Name), + FileMode.Create, + FileAccess.Write, + FileShare.None); + remoteStreams.Add(artifact.Type, fileStream); + break; + } + } + CompileResultHelper.Unpack(remoteOutput.ResultData, compileResult.Cookies, remoteStreams); + + var localStreams = new Dictionary(); + var localFiles = new List(); + foreach (var artifact in compileResult.Cookies) + { + var stream = remoteStreams[artifact.Type]; + switch (artifact.Type) + { + case CompileArtifactType.Out: + case CompileArtifactType.Err: + stream.Position = 0; + localStreams.Add(artifact, stream); + break; + + default: + stream.Close(); + localFiles.Add(artifact); + break; + } + } + + LocalLogger.InfoFormat("Completed local compile '{0}'", input.SrcName); + stopwatch.Stop(); + LocalLogger.DebugFormat(@"Stats for '{0}': agent {1}, preprocess {2}, compiler search {3} compile {4}", - input.SrcName, - agentName, - preprocessDuration, - compilerSearchDuration, - stopwatch.Elapsed); + input.SrcName, + agentName, + preprocessDuration, + compilerSearchDuration, + stopwatch.Elapsed); - return new LocalCompileOutput(true, remoteOutput.Status.ExitCode, localStreams, localFiles); + return new LocalCompileOutput(true, compileResult.ExitCode, localStreams, localFiles); + } + + var sourceFileRequest = (SourceFileRequest) remoteOutput.Result; + compilationToken = sourceFileRequest.CompilationToken; + inputFileName = sourceFileRequest.FileName; + } } } } @@ -319,7 +335,7 @@ CompileOutput ICompiler.Compile(CompileInput input) Dictionary streams; var errorCode = RunCompiler(clPath, input.Arguments, srcName, tmpPath, out streams); - return new CompileOutput(errorCode == 0, errorCode, streams, null); + return new CompileOutput(errorCode, streams, null); } finally { diff --git a/DistCL.Core/Contracts/ICompiler.cs b/DistCL.Core/Contracts/ICompiler.cs index d380258..eef0cfd 100644 --- a/DistCL.Core/Contracts/ICompiler.cs +++ b/DistCL.Core/Contracts/ICompiler.cs @@ -16,30 +16,10 @@ public interface ICompiler bool IsReady(); [OperationContract] - [FaultContract(typeof(CompilerNotFoundFaultContract), Namespace = GeneralSettings.CompilerMessageNamespace)] + [FaultContract(typeof (CompilerNotFoundFaultContract), Namespace = GeneralSettings.CompilerMessageNamespace)] CompileOutput Compile(CompileInput input); } - [DataContract(Namespace = GeneralSettings.CompilerMessageNamespace)] - public class CompileStatus - { - public CompileStatus(bool success, int exitCode, CompileArtifactCookie[] cookies) - { - ExitCode = exitCode; - Success = success; - Cookies = cookies; - } - - [DataMember] - public bool Success { get; private set; } - - [DataMember] - public int ExitCode { get; private set; } - - [DataMember] - public CompileArtifactCookie[] Cookies { get; private set; } - } - [MessageContract(WrapperNamespace = GeneralSettings.CompilerMessageNamespace)] public class CompileInput : IStreamedMessage { @@ -53,6 +33,9 @@ public CompileInput(IStreamedMessage message) Src = new ProxyStream(message); } + [MessageHeader] + public string CompilationToken { get; set; } + [MessageHeader] public string CompilerVersion { get; set; } @@ -86,24 +69,28 @@ public long StreamLength [MessageContract(WrapperNamespace = GeneralSettings.CompilerMessageNamespace)] public class CompileOutput : IDisposable { - private readonly CompileStatus _status; + private readonly CompileResult _result; private Stream _resultData; - public CompileOutput(CompileStatus status, Stream resultData) + public CompileOutput(string compilationToken, string requiredFileName) + { + _result = new SourceFileRequest(compilationToken, requiredFileName); + } + + public CompileOutput(CompileResult result, Stream resultData) { - _status = status; + _result = result; _resultData = resultData; } public CompileOutput( - bool success, int exitCode, IDictionary streams, IEnumerable artifacts) { CompileArtifactCookie[] cookies; _resultData = CompileResultHelper.Pack(streams, out cookies); - + // TODO remove redundant collections copy if (artifacts != null) { @@ -112,13 +99,13 @@ public CompileOutput( cookies = list.ToArray(); } - _status = new CompileStatus(success, exitCode, cookies); + _result = new FinishedCompileResult(exitCode, cookies); } [MessageHeader] - public CompileStatus Status + public CompileResult Result { - get { return _status; } + get { return _result; } } [MessageBodyMember] @@ -136,4 +123,52 @@ public void Dispose() } } } + + [DataContract(Namespace = GeneralSettings.CompilerMessageNamespace)] + [KnownType(typeof(FinishedCompileResult))] + [KnownType(typeof(SourceFileRequest))] + public abstract class CompileResult + { + protected CompileResult(bool finished) + { + Finished = finished; + } + + [DataMember] + public bool Finished { get; private set; } + } + + [DataContract(Namespace = GeneralSettings.CompilerMessageNamespace)] + public class FinishedCompileResult : CompileResult + { + public FinishedCompileResult(int exitCode, CompileArtifactCookie[] cookies) + : base(true) + { + ExitCode = exitCode; + Cookies = cookies; + } + + [DataMember] + public int ExitCode { get; private set; } + + [DataMember] + public CompileArtifactCookie[] Cookies { get; private set; } + } + + [DataContract(Namespace = GeneralSettings.CompilerMessageNamespace)] + public class SourceFileRequest : CompileResult + { + public SourceFileRequest(string compilationToken, string fileName) + : base(false) + { + CompilationToken = compilationToken; + FileName = fileName; + } + + [DataMember] + public string FileName { get; private set; } + + [DataMember] + public string CompilationToken { get; private set; } + } } diff --git a/DistCL.Core/Contracts/ILocalCompiler.cs b/DistCL.Core/Contracts/ILocalCompiler.cs index 9f6cb1a..5f17d11 100644 --- a/DistCL.Core/Contracts/ILocalCompiler.cs +++ b/DistCL.Core/Contracts/ILocalCompiler.cs @@ -65,7 +65,7 @@ public PreprocessToken(Guid guid, string accountName, DateTime requested, DateTi [MessageContract(WrapperNamespace = GeneralSettings.LocalCompilerMessageNamespace)] public class LocalCompileOutput : CompileOutput { - public LocalCompileOutput(CompileStatus status, Stream resultData) : base(status, resultData) + public LocalCompileOutput(CompileResult result, Stream resultData) : base(result, resultData) { } @@ -74,7 +74,7 @@ public LocalCompileOutput( int exitCode, IDictionary streams, IEnumerable artifacts) - : base(success, exitCode, streams, artifacts) + : base(exitCode, streams, artifacts) { } } diff --git a/DistCL.Core/DistCL.Core.csproj b/DistCL.Core/DistCL.Core.csproj index aaaa7c5..a520618 100644 --- a/DistCL.Core/DistCL.Core.csproj +++ b/DistCL.Core/DistCL.Core.csproj @@ -92,7 +92,7 @@ Reference.svcmap - + Reference.svcmap diff --git a/DistCL.Core/Proxies/AgentProxy.cs b/DistCL.Core/Proxies/AgentProxy.cs index eac00b7..8b80ea4 100644 --- a/DistCL.Core/Proxies/AgentProxy.cs +++ b/DistCL.Core/Proxies/AgentProxy.cs @@ -249,13 +249,20 @@ public CompileOutput Compile(CompileInput localInput) RemoteCompilerService.CompileOutput remoteOutput = _compiler.Compile(remoteInput); - return - new CompileOutput( - new CompileStatus( - remoteOutput.Status.Success, - remoteOutput.Status.ExitCode, - remoteOutput.Status.Cookies), + if (remoteOutput.Result.Finished) + { + var finishedCompileResult = ((RemoteCompilerService.FinishedCompileResult) remoteOutput.Result); + return new CompileOutput( + new FinishedCompileResult( + finishedCompileResult.ExitCode, + finishedCompileResult.Cookies), remoteOutput.ResultData); + } + + var sourceFileRequest = ((RemoteCompilerService.SourceFileRequest) remoteOutput.Result); + return new CompileOutput( + new SourceFileRequest(sourceFileRequest.CompilationToken, sourceFileRequest.FileName), + remoteOutput.ResultData); } public void Dispose() diff --git a/DistCL.Core/Service References/RemoteCompilerService/DistCL.RemoteCompilerService.CompileStatus.datasource b/DistCL.Core/Service References/RemoteCompilerService/DistCL.RemoteCompilerService.CompileResult.datasource similarity index 80% rename from DistCL.Core/Service References/RemoteCompilerService/DistCL.RemoteCompilerService.CompileStatus.datasource rename to DistCL.Core/Service References/RemoteCompilerService/DistCL.RemoteCompilerService.CompileResult.datasource index 1483fc5..1e8f06c 100644 --- a/DistCL.Core/Service References/RemoteCompilerService/DistCL.RemoteCompilerService.CompileStatus.datasource +++ b/DistCL.Core/Service References/RemoteCompilerService/DistCL.RemoteCompilerService.CompileResult.datasource @@ -5,6 +5,6 @@ Renaming the file extension or editing the content of this file may cause the file to be unrecognizable by the program. --> - - DistCL.RemoteCompilerService.CompileStatus, Service References.RemoteCompilerService.Reference.cs, Version=0.0.0.0, Culture=neutral, PublicKeyToken=null + + DistCL.RemoteCompilerService.CompileResult, Service References.RemoteCompilerService.Reference.cs, Version=0.0.0.0, Culture=neutral, PublicKeyToken=null \ No newline at end of file diff --git a/DistCL.Core/Service References/RemoteCompilerService/Reference.cs b/DistCL.Core/Service References/RemoteCompilerService/Reference.cs index 6bc3a0c..1f4fb46 100644 --- a/DistCL.Core/Service References/RemoteCompilerService/Reference.cs +++ b/DistCL.Core/Service References/RemoteCompilerService/Reference.cs @@ -1,7 +1,7 @@ //------------------------------------------------------------------------------ // // This code was generated by a tool. -// Runtime Version:4.0.30319.18034 +// Runtime Version:4.0.30319.34014 // // Changes to this file may cause incorrect behavior and will be lost if // the code is regenerated. @@ -153,21 +153,17 @@ protected void RaisePropertyChanged(string propertyName) { [System.Diagnostics.DebuggerStepThroughAttribute()] [System.CodeDom.Compiler.GeneratedCodeAttribute("System.Runtime.Serialization", "4.0.0.0")] - [System.Runtime.Serialization.DataContractAttribute(Name="CompileStatus", Namespace="urn:distcl:compiler:messages")] + [System.Runtime.Serialization.DataContractAttribute(Name="CompileResult", Namespace="urn:distcl:compiler:messages")] [System.SerializableAttribute()] - public partial class CompileStatus : object, System.Runtime.Serialization.IExtensibleDataObject, System.ComponentModel.INotifyPropertyChanged { + [System.Runtime.Serialization.KnownTypeAttribute(typeof(DistCL.RemoteCompilerService.SourceFileRequest))] + [System.Runtime.Serialization.KnownTypeAttribute(typeof(DistCL.RemoteCompilerService.FinishedCompileResult))] + public partial class CompileResult : object, System.Runtime.Serialization.IExtensibleDataObject, System.ComponentModel.INotifyPropertyChanged { [System.NonSerializedAttribute()] private System.Runtime.Serialization.ExtensionDataObject extensionDataField; [System.Runtime.Serialization.OptionalFieldAttribute()] - private DistCL.Utils.CompileArtifactCookie[] CookiesField; - - [System.Runtime.Serialization.OptionalFieldAttribute()] - private int ExitCodeField; - - [System.Runtime.Serialization.OptionalFieldAttribute()] - private bool SuccessField; + private bool FinishedField; [global::System.ComponentModel.BrowsableAttribute(false)] public System.Runtime.Serialization.ExtensionDataObject ExtensionData { @@ -180,50 +176,102 @@ public System.Runtime.Serialization.ExtensionDataObject ExtensionData { } [System.Runtime.Serialization.DataMemberAttribute()] - public DistCL.Utils.CompileArtifactCookie[] Cookies { + public bool Finished { get { - return this.CookiesField; + return this.FinishedField; } set { - if ((object.ReferenceEquals(this.CookiesField, value) != true)) { - this.CookiesField = value; - this.RaisePropertyChanged("Cookies"); + if ((this.FinishedField.Equals(value) != true)) { + this.FinishedField = value; + this.RaisePropertyChanged("Finished"); } } } + public event System.ComponentModel.PropertyChangedEventHandler PropertyChanged; + + protected void RaisePropertyChanged(string propertyName) { + System.ComponentModel.PropertyChangedEventHandler propertyChanged = this.PropertyChanged; + if ((propertyChanged != null)) { + propertyChanged(this, new System.ComponentModel.PropertyChangedEventArgs(propertyName)); + } + } + } + + [System.Diagnostics.DebuggerStepThroughAttribute()] + [System.CodeDom.Compiler.GeneratedCodeAttribute("System.Runtime.Serialization", "4.0.0.0")] + [System.Runtime.Serialization.DataContractAttribute(Name="SourceFileRequest", Namespace="urn:distcl:compiler:messages")] + [System.SerializableAttribute()] + public partial class SourceFileRequest : DistCL.RemoteCompilerService.CompileResult { + + [System.Runtime.Serialization.OptionalFieldAttribute()] + private string CompilationTokenField; + + [System.Runtime.Serialization.OptionalFieldAttribute()] + private string FileNameField; + [System.Runtime.Serialization.DataMemberAttribute()] - public int ExitCode { + public string CompilationToken { get { - return this.ExitCodeField; + return this.CompilationTokenField; } set { - if ((this.ExitCodeField.Equals(value) != true)) { - this.ExitCodeField = value; - this.RaisePropertyChanged("ExitCode"); + if ((object.ReferenceEquals(this.CompilationTokenField, value) != true)) { + this.CompilationTokenField = value; + this.RaisePropertyChanged("CompilationToken"); } } } [System.Runtime.Serialization.DataMemberAttribute()] - public bool Success { + public string FileName { get { - return this.SuccessField; + return this.FileNameField; } set { - if ((this.SuccessField.Equals(value) != true)) { - this.SuccessField = value; - this.RaisePropertyChanged("Success"); + if ((object.ReferenceEquals(this.FileNameField, value) != true)) { + this.FileNameField = value; + this.RaisePropertyChanged("FileName"); } } } + } + + [System.Diagnostics.DebuggerStepThroughAttribute()] + [System.CodeDom.Compiler.GeneratedCodeAttribute("System.Runtime.Serialization", "4.0.0.0")] + [System.Runtime.Serialization.DataContractAttribute(Name="FinishedCompileResult", Namespace="urn:distcl:compiler:messages")] + [System.SerializableAttribute()] + public partial class FinishedCompileResult : DistCL.RemoteCompilerService.CompileResult { - public event System.ComponentModel.PropertyChangedEventHandler PropertyChanged; + [System.Runtime.Serialization.OptionalFieldAttribute()] + private DistCL.Utils.CompileArtifactCookie[] CookiesField; - protected void RaisePropertyChanged(string propertyName) { - System.ComponentModel.PropertyChangedEventHandler propertyChanged = this.PropertyChanged; - if ((propertyChanged != null)) { - propertyChanged(this, new System.ComponentModel.PropertyChangedEventArgs(propertyName)); + [System.Runtime.Serialization.OptionalFieldAttribute()] + private int ExitCodeField; + + [System.Runtime.Serialization.DataMemberAttribute()] + public DistCL.Utils.CompileArtifactCookie[] Cookies { + get { + return this.CookiesField; + } + set { + if ((object.ReferenceEquals(this.CookiesField, value) != true)) { + this.CookiesField = value; + this.RaisePropertyChanged("Cookies"); + } + } + } + + [System.Runtime.Serialization.DataMemberAttribute()] + public int ExitCode { + get { + return this.ExitCodeField; + } + set { + if ((this.ExitCodeField.Equals(value) != true)) { + this.ExitCodeField = value; + this.RaisePropertyChanged("ExitCode"); + } } } } @@ -429,7 +477,7 @@ public LocalCompileInput(string Arguments, string CompilerVersion, DistCL.Remote public partial class LocalCompileOutput { [System.ServiceModel.MessageHeaderAttribute(Namespace="urn:distcl")] - public DistCL.RemoteCompilerService.CompileStatus Status; + public DistCL.RemoteCompilerService.CompileResult Result; [System.ServiceModel.MessageBodyMemberAttribute(Namespace="urn:distcl", Order=0)] public System.IO.Stream ResultData; @@ -437,8 +485,8 @@ public partial class LocalCompileOutput { public LocalCompileOutput() { } - public LocalCompileOutput(DistCL.RemoteCompilerService.CompileStatus Status, System.IO.Stream ResultData) { - this.Status = Status; + public LocalCompileOutput(DistCL.RemoteCompilerService.CompileResult Result, System.IO.Stream ResultData) { + this.Result = Result; this.ResultData = ResultData; } } @@ -483,7 +531,7 @@ DistCL.RemoteCompilerService.LocalCompileOutput DistCL.RemoteCompilerService.ILo return base.Channel.LocalCompile(request); } - public DistCL.RemoteCompilerService.CompileStatus LocalCompile(string Arguments, string CompilerVersion, DistCL.RemoteCompilerService.PreprocessToken PreprocessToken, string Src, string SrcName, out System.IO.Stream ResultData) { + public DistCL.RemoteCompilerService.CompileResult LocalCompile(string Arguments, string CompilerVersion, DistCL.RemoteCompilerService.PreprocessToken PreprocessToken, string Src, string SrcName, out System.IO.Stream ResultData) { DistCL.RemoteCompilerService.LocalCompileInput inValue = new DistCL.RemoteCompilerService.LocalCompileInput(); inValue.Arguments = Arguments; inValue.CompilerVersion = CompilerVersion; @@ -492,7 +540,7 @@ public DistCL.RemoteCompilerService.CompileStatus LocalCompile(string Arguments, inValue.SrcName = SrcName; DistCL.RemoteCompilerService.LocalCompileOutput retVal = ((DistCL.RemoteCompilerService.ILocalCompiler)(this)).LocalCompile(inValue); ResultData = retVal.ResultData; - return retVal.Status; + return retVal.Result; } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Advanced)] @@ -557,6 +605,9 @@ public partial class CompileInput { [System.ServiceModel.MessageHeaderAttribute(Namespace="urn:distcl")] public string Arguments; + [System.ServiceModel.MessageHeaderAttribute(Namespace="urn:distcl")] + public string CompilationToken; + [System.ServiceModel.MessageHeaderAttribute(Namespace="urn:distcl")] public string CompilerVersion; @@ -572,8 +623,9 @@ public partial class CompileInput { public CompileInput() { } - public CompileInput(string Arguments, string CompilerVersion, long SrcLength, string SrcName, System.IO.Stream Src) { + public CompileInput(string Arguments, string CompilationToken, string CompilerVersion, long SrcLength, string SrcName, System.IO.Stream Src) { this.Arguments = Arguments; + this.CompilationToken = CompilationToken; this.CompilerVersion = CompilerVersion; this.SrcLength = SrcLength; this.SrcName = SrcName; @@ -588,7 +640,7 @@ public CompileInput(string Arguments, string CompilerVersion, long SrcLength, st public partial class CompileOutput { [System.ServiceModel.MessageHeaderAttribute(Namespace="urn:distcl")] - public DistCL.RemoteCompilerService.CompileStatus Status; + public DistCL.RemoteCompilerService.CompileResult Result; [System.ServiceModel.MessageBodyMemberAttribute(Namespace="urn:distcl", Order=0)] public System.IO.Stream ResultData; @@ -596,8 +648,8 @@ public partial class CompileOutput { public CompileOutput() { } - public CompileOutput(DistCL.RemoteCompilerService.CompileStatus Status, System.IO.Stream ResultData) { - this.Status = Status; + public CompileOutput(DistCL.RemoteCompilerService.CompileResult Result, System.IO.Stream ResultData) { + this.Result = Result; this.ResultData = ResultData; } } @@ -642,16 +694,17 @@ DistCL.RemoteCompilerService.CompileOutput DistCL.RemoteCompilerService.ICompile return base.Channel.Compile(request); } - public DistCL.RemoteCompilerService.CompileStatus Compile(string Arguments, string CompilerVersion, long SrcLength, string SrcName, System.IO.Stream Src, out System.IO.Stream ResultData) { + public DistCL.RemoteCompilerService.CompileResult Compile(string Arguments, string CompilationToken, string CompilerVersion, long SrcLength, string SrcName, System.IO.Stream Src, out System.IO.Stream ResultData) { DistCL.RemoteCompilerService.CompileInput inValue = new DistCL.RemoteCompilerService.CompileInput(); inValue.Arguments = Arguments; + inValue.CompilationToken = CompilationToken; inValue.CompilerVersion = CompilerVersion; inValue.SrcLength = SrcLength; inValue.SrcName = SrcName; inValue.Src = Src; DistCL.RemoteCompilerService.CompileOutput retVal = ((DistCL.RemoteCompilerService.ICompileManager)(this)).Compile(inValue); ResultData = retVal.ResultData; - return retVal.Status; + return retVal.Result; } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Advanced)] @@ -659,9 +712,10 @@ public DistCL.RemoteCompilerService.CompileStatus Compile(string Arguments, stri return base.Channel.CompileAsync(request); } - public System.Threading.Tasks.Task CompileAsync(string Arguments, string CompilerVersion, long SrcLength, string SrcName, System.IO.Stream Src) { + public System.Threading.Tasks.Task CompileAsync(string Arguments, string CompilationToken, string CompilerVersion, long SrcLength, string SrcName, System.IO.Stream Src) { DistCL.RemoteCompilerService.CompileInput inValue = new DistCL.RemoteCompilerService.CompileInput(); inValue.Arguments = Arguments; + inValue.CompilationToken = CompilationToken; inValue.CompilerVersion = CompilerVersion; inValue.SrcLength = SrcLength; inValue.SrcName = SrcName; @@ -889,16 +943,17 @@ DistCL.RemoteCompilerService.CompileOutput DistCL.RemoteCompilerService.ICompile return base.Channel.Compile(request); } - public DistCL.RemoteCompilerService.CompileStatus Compile(string Arguments, string CompilerVersion, long SrcLength, string SrcName, System.IO.Stream Src, out System.IO.Stream ResultData) { + public DistCL.RemoteCompilerService.CompileResult Compile(string Arguments, string CompilationToken, string CompilerVersion, long SrcLength, string SrcName, System.IO.Stream Src, out System.IO.Stream ResultData) { DistCL.RemoteCompilerService.CompileInput inValue = new DistCL.RemoteCompilerService.CompileInput(); inValue.Arguments = Arguments; + inValue.CompilationToken = CompilationToken; inValue.CompilerVersion = CompilerVersion; inValue.SrcLength = SrcLength; inValue.SrcName = SrcName; inValue.Src = Src; DistCL.RemoteCompilerService.CompileOutput retVal = ((DistCL.RemoteCompilerService.ICompiler)(this)).Compile(inValue); ResultData = retVal.ResultData; - return retVal.Status; + return retVal.Result; } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Advanced)] @@ -906,9 +961,10 @@ public DistCL.RemoteCompilerService.CompileStatus Compile(string Arguments, stri return base.Channel.CompileAsync(request); } - public System.Threading.Tasks.Task CompileAsync(string Arguments, string CompilerVersion, long SrcLength, string SrcName, System.IO.Stream Src) { + public System.Threading.Tasks.Task CompileAsync(string Arguments, string CompilationToken, string CompilerVersion, long SrcLength, string SrcName, System.IO.Stream Src) { DistCL.RemoteCompilerService.CompileInput inValue = new DistCL.RemoteCompilerService.CompileInput(); inValue.Arguments = Arguments; + inValue.CompilationToken = CompilationToken; inValue.CompilerVersion = CompilerVersion; inValue.SrcLength = SrcLength; inValue.SrcName = SrcName; diff --git a/DistCL.Core/Service References/RemoteCompilerService/Reference.svcmap b/DistCL.Core/Service References/RemoteCompilerService/Reference.svcmap index 1bf4ea8..917a75b 100644 --- a/DistCL.Core/Service References/RemoteCompilerService/Reference.svcmap +++ b/DistCL.Core/Service References/RemoteCompilerService/Reference.svcmap @@ -30,7 +30,7 @@ - + diff --git a/DistCL.Core/Service References/RemoteCompilerService/bindings.wsdl b/DistCL.Core/Service References/RemoteCompilerService/bindings.wsdl index fbf8d2e..732c682 100644 --- a/DistCL.Core/Service References/RemoteCompilerService/bindings.wsdl +++ b/DistCL.Core/Service References/RemoteCompilerService/bindings.wsdl @@ -107,7 +107,7 @@ - + @@ -132,13 +132,14 @@ + - + @@ -236,13 +237,14 @@ + - + diff --git a/DistCL.Core/Service References/RemoteCompilerService/configuration.svcinfo b/DistCL.Core/Service References/RemoteCompilerService/configuration.svcinfo index 0d8716b..4fafb9a 100644 --- a/DistCL.Core/Service References/RemoteCompilerService/configuration.svcinfo +++ b/DistCL.Core/Service References/RemoteCompilerService/configuration.svcinfo @@ -9,7 +9,7 @@ - + diff --git a/DistCL.Core/Service References/RemoteCompilerService/configuration91.svcinfo b/DistCL.Core/Service References/RemoteCompilerService/configuration91.svcinfo index be431b7..23d3ecb 100644 --- a/DistCL.Core/Service References/RemoteCompilerService/configuration91.svcinfo +++ b/DistCL.Core/Service References/RemoteCompilerService/configuration91.svcinfo @@ -1,5 +1,5 @@ - + @@ -607,14 +607,14 @@ System.ServiceModel.Configuration.UserPrincipalNameElement - - npopov\profiler + + System.ServiceModel.Configuration.ServicePrincipalNameElement - - + + host/laika System.ServiceModel.Configuration.DnsElement diff --git a/DistCL.Core/Service References/RemoteCompilerService/distcl.wsdl b/DistCL.Core/Service References/RemoteCompilerService/distcl.wsdl index 61cee31..0fe90da 100644 --- a/DistCL.Core/Service References/RemoteCompilerService/distcl.wsdl +++ b/DistCL.Core/Service References/RemoteCompilerService/distcl.wsdl @@ -28,7 +28,7 @@ - + @@ -44,6 +44,7 @@ + @@ -52,7 +53,7 @@ - + diff --git a/DistCL.Core/Service References/RemoteCompilerService/distcl.xsd b/DistCL.Core/Service References/RemoteCompilerService/distcl.xsd index 6168b29..45f66f7 100644 --- a/DistCL.Core/Service References/RemoteCompilerService/distcl.xsd +++ b/DistCL.Core/Service References/RemoteCompilerService/distcl.xsd @@ -20,7 +20,7 @@ - + @@ -35,6 +35,7 @@ + diff --git a/DistCL.Core/Service References/RemoteCompilerService/messages1.xsd b/DistCL.Core/Service References/RemoteCompilerService/messages1.xsd index 430f817..c41a2c9 100644 --- a/DistCL.Core/Service References/RemoteCompilerService/messages1.xsd +++ b/DistCL.Core/Service References/RemoteCompilerService/messages1.xsd @@ -2,14 +2,34 @@ - + - - - + - + + + + + + + + + + + + + + + + + + + + + + + diff --git a/DistCL.Core/Service References/RemoteCompilerService/service.wsdl b/DistCL.Core/Service References/RemoteCompilerService/service.wsdl index dc078e6..cbf45d7 100644 --- a/DistCL.Core/Service References/RemoteCompilerService/service.wsdl +++ b/DistCL.Core/Service References/RemoteCompilerService/service.wsdl @@ -16,36 +16,24 @@ net.tcp://localhost:8734/Compile - - host/laika - net.tcp://localhost:8734/Compile/CompileCoordinator - - host/laika - net.tcp://localhost:8734/Compile/AgentPool - - host/laika - net.tcp://localhost:8734/Compile/Compiler - - host/laika - diff --git a/DistCL.Service/DistCL.Service.csproj b/DistCL.Service/DistCL.Service.csproj index 5493670..ea40704 100644 --- a/DistCL.Service/DistCL.Service.csproj +++ b/DistCL.Service/DistCL.Service.csproj @@ -84,9 +84,6 @@ ProjectInstaller.cs - - - {15b3e0d2-6217-493a-a690-158c497f5318} From 1d3953061c51a241c6e19b229101bcac2753ee19 Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Tue, 30 Sep 2014 22:46:39 +0700 Subject: [PATCH 2/2] SourceFileExchange and CompilationProcess files exchange between compilation process and client --- DistCL.Core/Compiler.cs | 230 +++++++++++++++++++++++++++++++++------- 1 file changed, 192 insertions(+), 38 deletions(-) diff --git a/DistCL.Core/Compiler.cs b/DistCL.Core/Compiler.cs index d5816da..d5fbcd1 100644 --- a/DistCL.Core/Compiler.cs +++ b/DistCL.Core/Compiler.cs @@ -2,6 +2,7 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics; +using System.Globalization; using System.IO; using System.ServiceModel; using System.Text; @@ -23,7 +24,7 @@ public class Compiler : ICompileManager, ILocalCompiler private readonly Semaphore _semaphore; private readonly object _acquireWorkersSyncRoot = new object(); private readonly int _maxWorkersCount; - private ManualResetEventSlim _preprocessWaitEventSlim = new ManualResetEventSlim(); + private readonly ManualResetEventSlim _preprocessWaitEventSlim = new ManualResetEventSlim(); private readonly ICompilerServicesCollection _compilerServices; private readonly Logger _compilerLogger = new Logger("COMPILER"); @@ -289,6 +290,86 @@ bool ICompiler.IsReady() return ready; } + private class SourceFileExchange : IDisposable + { + readonly ManualResetEventSlim _sourceFileIsRequested = new ManualResetEventSlim(); + private string _requestedSourceFile; + + readonly ManualResetEventSlim _sourceFileIsReady = new ManualResetEventSlim(); + private string _readySourceFile; + + private bool _isDisposed; + + public string RequestSourceFile(string fileName) + { + _sourceFileIsReady.Reset(); + _requestedSourceFile = fileName; + _sourceFileIsRequested.Set(); + + _sourceFileIsReady.Wait(); + return _readySourceFile; + } + + public string GetSourceFileRequest() + { + lock (_sourceFileIsRequested) + { + _sourceFileIsRequested.Wait(); + return _requestedSourceFile; + } + } + + public void SetSourceFileResponse(string fileName) + { + _sourceFileIsRequested.Reset(); + _readySourceFile = fileName; + _sourceFileIsReady.Set(); + } + + public bool IsDisposed + { + get { return _isDisposed; } + } + + public void Dispose() + { + _requestedSourceFile = null; + _sourceFileIsRequested.Set(); + + lock (_sourceFileIsRequested) + { + _sourceFileIsRequested.Dispose(); + } + + _sourceFileIsReady.Dispose(); + _isDisposed = true; + } + } + + private class CompilationProcess + { + private readonly SourceFileExchange _fileExchange; + private readonly Task _compilationTask; + + public CompilationProcess(SourceFileExchange fileExchange, Task compilationTask) + { + _fileExchange = fileExchange; + _compilationTask = compilationTask; + } + + public SourceFileExchange FileExchange + { + get { return _fileExchange; } + } + + public Task CompilationTask + { + get { return _compilationTask; } + } + } + + readonly Dictionary _compilationProcesses = new Dictionary(); + CompileOutput ICompiler.Compile(CompileInput input) { CompilerLogger.DebugFormat("Received compile request '{0}'", input.SrcName); @@ -303,63 +384,136 @@ CompileOutput ICompiler.Compile(CompileInput input) }); } - AcquireWorkers(CompileWorkerCount); + var compilationToken = input.CompilationToken ?? Guid.NewGuid().ToString(); + + var tmpPath = Path.Combine(Path.GetTempPath(), compilationToken); - CompilerLogger.InfoFormat("Processing '{0}'...", input.SrcName); + if (!Directory.Exists(tmpPath)) + Directory.CreateDirectory(tmpPath); - try + var pathRoot = Path.GetPathRoot(input.SrcName); + var srcName = Path.Combine( + tmpPath, + pathRoot == null + ? "_" + : Path.GetInvalidPathChars() + .Aggregate(pathRoot, (current, ipc) => current.Replace(ipc.ToString(CultureInfo.InvariantCulture), string.Empty)), + Path.GetDirectoryName(input.SrcName), + Path.GetFileName(input.SrcName)); + using (var src = File.OpenWrite(srcName)) { - string clPath; - if (!CompilerVersions.TryGetValue(input.CompilerVersion, out clPath)) - { - var error = string.Format("Compiler with specified version not found ({0})", input.CompilerVersion); - CompilerLogger.Warn(error); - throw new Exception(error); - } + CompilerLogger.DebugFormat("Copying source to {0}...", srcName); + input.Src.CopyTo(src); + CompilerLogger.Debug("Source copied to local file"); + } - var tmpPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); + CompilationProcess compilationProcess; + lock (_compilationProcesses) + { + _compilationProcesses.TryGetValue(compilationToken, out compilationProcess); + } - if (!Directory.Exists(tmpPath)) - Directory.CreateDirectory(tmpPath); + if (compilationProcess != null) + { + compilationProcess.FileExchange.SetSourceFileResponse(srcName); + } + else + { + var fileExchange = new SourceFileExchange(); - try - { - var srcName = Path.Combine(tmpPath, Path.GetFileName(input.SrcName)); - using (var src = File.OpenWrite(srcName)) + compilationProcess = new CompilationProcess( + fileExchange, + Task.Run(delegate { - CompilerLogger.DebugFormat("Copying source to {0}...", srcName); - input.Src.CopyTo(src); - CompilerLogger.Debug("Source copied to local file"); - } + AcquireWorkers(CompileWorkerCount); - Dictionary streams; - var errorCode = RunCompiler(clPath, input.Arguments, srcName, tmpPath, out streams); + CompilerLogger.InfoFormat("Processing '{0}'...", input.SrcName); - return new CompileOutput(errorCode, streams, null); - } - finally + try + { + string clPath; + if (!CompilerVersions.TryGetValue(input.CompilerVersion, out clPath)) + { + var error = string.Format("Compiler with specified version not found ({0})", input.CompilerVersion); + CompilerLogger.Warn(error); + throw new Exception(error); + } + + try + { + Dictionary streams; + var errorCode = RunCompiler( + clPath, + input.Arguments, + srcName, + tmpPath, + fileExchange.RequestSourceFile, + out streams); + + return new CompileOutput(errorCode, streams, null); + } + finally + { + Directory.Delete(tmpPath, true); + } + } + catch (Exception e) + { + CompilerLogger.LogException(string.Format("Exception in CompileInternal({0})", input.SrcName), e); + throw; + } + finally + { + ReleaseWorkers(CompileWorkerCount); + stopwatch.Stop(); + CompilerLogger.InfoFormat("Processing of '{0}' completed ({1})", input.SrcName, stopwatch.Elapsed); + } + })); + + lock (_compilationProcesses) { - Directory.Delete(tmpPath, true); + _compilationProcesses.Add(compilationToken, compilationProcess); } } - catch (Exception e) + + var tasksSyncRoot = new object(); + + var fileRequestWaitTask = Task.Run(delegate { - CompilerLogger.LogException(string.Format("Exception in CompileInternal({0})", input.SrcName), e); - throw; - } - finally + lock (tasksSyncRoot) + { + // ReSharper disable once AccessToDisposedClosure + return compilationProcess.FileExchange.IsDisposed ? null : compilationProcess.FileExchange.GetSourceFileRequest(); + } + }); + + if (Task.WaitAny(compilationProcess.CompilationTask, fileRequestWaitTask) == 0) { - ReleaseWorkers(CompileWorkerCount); - stopwatch.Stop(); - CompilerLogger.InfoFormat("Processing of '{0}' completed ({1})", input.SrcName, stopwatch.Elapsed); + var compileOutput = compilationProcess.CompilationTask.Result; + + lock (_compilationProcesses) + { + _compilationProcesses.Remove(compilationToken); + } + lock (tasksSyncRoot) + { + compilationProcess.FileExchange.Dispose(); + } + + return compileOutput; } + + return new CompileOutput(compilationToken, fileRequestWaitTask.Result); } + private delegate string GetCachedFileNameDelegate(string sourceFileName); + private int RunCompiler( string clPath, string commmandLine, - string inputPath, - string outputPath, + string inputPath, // obsolete ? + string outputPath, // obsolete ? + GetCachedFileNameDelegate getFileName, out Dictionary streams) { streams = new Dictionary();