message: Implement minimum extension version enforcement

diff:
using System.Collections.Generic;
using System.IO;
using System.Linq;
+using System.Reflection;
using System.Threading.Tasks;
using Microsoft.Azure.WebJobs.Hosting;
using Microsoft.Azure.WebJobs.Script.Description;
using Microsoft.Azure.WebJobs.Script.Diagnostics;
using Microsoft.Azure.WebJobs.Script.Diagnostics.Extensions;
using Microsoft.Azure.WebJobs.Script.ExtensionBundle;
+using Microsoft.Azure.WebJobs.Script.ExtensionRequirements;
using Microsoft.Azure.WebJobs.Script.Models;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
@@ -31,6 +33,7 @@ public class ScriptStartupTypeLocator : IWebJobsStartupTypeLocator
private readonly IMetricsLogger _metricsLogger;
private readonly Lazy<IEnumerable<Type>> _startupTypes;
+ private static readonly ExtensionRequirementsInfo _extensionRequirements = DependencyHelper.GetExtensionRequirements();
private static string[] _builtinExtensionAssemblies = GetBuiltinExtensionAssemblies();
public ScriptStartupTypeLocator(string rootScriptPath, ILogger<ScriptStartupTypeLocator> logger, IExtensionBundleManager extensionBundleManager,
@@ -72,6 +75,9 @@ public async Task<IEnumerable<Type>> GetExtensionsStartupTypesAsync()
if (bundleConfigured)
{
+ ExtensionBundleDetails bundleDetails = await _extensionBundleManager.GetExtensionBundleDetails();
+ ValidateBundleRequirements(bundleDetails);
+
bindingsSet = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
// Generate a Hashset of all the binding types used in the function app
@@ -199,6 +205,8 @@ public async Task<IEnumerable<Type>> GetExtensionsStartupTypesAsync()
}
}
+ ValidateExtensionRequirements(startupTypes);
+
return startupTypes;
}
@@ -233,6 +241,88 @@ private ExtensionReference[] ParseExtensions(string metadataFilePath)
}
}
+ private void ValidateBundleRequirements(ExtensionBundleDetails bundleDetails)
+ {
+ if (_extensionRequirements.BundleRequirementsByBundleId.TryGetValue(bundleDetails.Id, out BundleRequirement requirement))
+ {
+ var bundleVersion = new Version(bundleDetails.Version);
+ var minimumVersion = new Version(requirement.MinimumVersion);
+
+ if (bundleVersion < minimumVersion)
+ {
+ _logger.MinimumBundleVersionNotSatisfied(bundleDetails.Id, bundleDetails.Version, requirement.MinimumVersion);
+ throw new HostInitializationException($"Referenced bundle {bundleDetails.Id} of version {bundleDetails.Version} does not meet the required minimum version of {requirement.MinimumVersion}. Update your extension bundle reference in host.json to reference {requirement.MinimumVersion} or later. For more information see https://aka.ms/func-min-bundle-versions.");
+ }
+ }
+ }
+
+ private void ValidateExtensionRequirements(List<Type> startupTypes)
+ {
+ var errors = new List<string>();
+
+ void CollectError(Type extensionType, Version minimumVersion, ExtensionStartupTypeRequirement requirement)
+ {
+ _logger.MinimumExtensionVersionNotSatisfied(extensionType.Name, extensionType.Assembly.FullName, minimumVersion, requirement.PackageName, requirement.MinimumPackageVersion);
+ string requirementNotMetError = $"ExtensionStartupType {extensionType.Name} from assembly '{extensionType.Assembly.FullName}' does not meet the required minimum version of {minimumVersion}. Update your NuGet package reference for {requirement.PackageName} to {requirement.MinimumPackageVersion} or later.";
+ errors.Add(requirementNotMetError);
+ }
+
+ foreach (var extensionType in startupTypes)
+ {
+ if (_extensionRequirements.ExtensionRequirementsByStartupType.TryGetValue(extensionType.Name, out ExtensionStartupTypeRequirement requirement))
+ {
+ Version minimumAssemblyVersion = new Version(requirement.MinimumAssemblyVersion);
+
+ AssemblyName assemblyName = extensionType.Assembly.GetName();
+ Version extensionAssemblyVersion = assemblyName.Version;
+ string extensionAssemblySimpleName = assemblyName.Name;
+
+ if (extensionAssemblySimpleName != requirement.AssemblyName)
+ {
+ // We recognized this type, but it did not come from the assembly that we expect, skipping validation
+ continue;
+ }
+
+ if (requirement.MinimumAssemblyFileVersion != null && !string.IsNullOrEmpty(extensionType.Assembly.Location))
+ {
+ Version minimumAssemblyFileVersion = new Version(requirement.MinimumAssemblyFileVersion);
+
+ try
+ {
+ Version extensionAssemblyFileVersion = new Version(System.Diagnostics.FileVersionInfo.GetVersionInfo(extensionType.Assembly.Location).FileVersion);
+ if (extensionAssemblyFileVersion < minimumAssemblyFileVersion)
+ {
+ CollectError(extensionType, minimumAssemblyFileVersion, requirement);
+ continue;
+ }
+ }
+ catch (FormatException)
+ {
+ // We failed to parse the assembly file version as a Version. We can't do a version check - give up
+ continue;
+ }
+ }
+
+ if (extensionAssemblyVersion < minimumAssemblyVersion)
+ {
+ CollectError(extensionType, minimumAssemblyVersion, requirement);
+ }
+ }
+ }
+
+ if (errors.Count > 0)
+ {
+ var builder = new System.Text.StringBuilder();
+ builder.AppendLine("One or more loaded extensions do not meet the minimum requirements. For more information see https://aka.ms/func-min-extension-versions.");
+ foreach (string e in errors)
+ {
+ builder.AppendLine(e);
+ }
+
+ throw new HostInitializationException(builder.ToString());
+ }
+ }
+
private class TypeNameEqualityComparer : IEqualityComparer<Type>
{
public bool Equals(Type x, Type y)

diff:
using System.IO;
using System.Linq;
using System.Runtime.InteropServices;
+using Microsoft.Azure.WebJobs.Script.ExtensionRequirements;
using Microsoft.Extensions.DependencyModel;
using Newtonsoft.Json.Linq;
using static Microsoft.DotNet.PlatformAbstractions.RuntimeEnvironment;
@@ -90,6 +91,20 @@ private static string GetResourceFileContents(string fileName)
.ToDictionary(a => a.Name, StringComparer.OrdinalIgnoreCase);
}
+ internal static ExtensionRequirementsInfo GetExtensionRequirements()
+ {
+ string requirementsJson = GetResourceFileContents("extensionrequirements.json");
+ JObject requirements = JObject.Parse(requirementsJson);
+
+ var bundleRequirements = requirements["bundles"]
+ .ToObject<BundleRequirement[]>();
+
+ var extensionRequirements = requirements["types"]
+ .ToObject<ExtensionStartupTypeRequirement[]>();
+
+ return new ExtensionRequirementsInfo(bundleRequirements, extensionRequirements);
+ }
+
/// <summary>
/// Gets the default runtime fallback RIDs for a given RID.
/// The graph used to build the fallback list is static and

diff:
@@ -197,6 +197,18 @@ internal static class LoggerExtension
new EventId(333, nameof(IncorrectAutorestGeneratedJsonFile)),
"autorest_generated.json file found is incorrect (https://aka.ms/stencil) | exception:\n{contents}");
+ private static readonly Action<ILogger, string, string, Version, string, string, Exception> _minimumExtensionVersionNotSatisfied =
+ LoggerMessage.Define<string, string, Version, string, string>(
+ LogLevel.Error,
+ new EventId(334, nameof(MinimumExtensionVersionNotSatisfied)),
+ "ExtensionStartupType {extensionTypeName} from assembly '{extensionTypeAssemblyFullName}' does not meet the required minimum version of {minimumAssemblyVersion}. Update your NuGet package reference for {requirementPackageName} to {requirementMinimumPackageVersion} or later.");
+
+ private static readonly Action<ILogger, string, string, string, string, Exception> _minimumBundleVersionNotSatisfied =
+ LoggerMessage.Define<string, string, string, string>(
+ LogLevel.Error,
+ new EventId(335, nameof(MinimumBundleVersionNotSatisfied)),
+ "Referenced bundle {bundleId} of version {bundleVersion} does not meet the required minimum version of {minimumVersion}. Update your extension bundle reference in host.json to reference {minimumVersion2} or later.");
+
public static void ExtensionsManagerRestoring(this ILogger logger)
{
_extensionsManagerRestoring(logger, null);
@@ -358,5 +370,15 @@ public static void IncorrectAutorestGeneratedJsonFile(this ILogger logger, strin
{
_incorrectAutorestGeneratedJSONFile(logger, contents, null);
}
+
+ public static void MinimumExtensionVersionNotSatisfied(this ILogger logger, string extensionTypeName, string extensionTypeAssemblyFullName, Version minimumAssemblyVersion, string requirementPackageName, string requirementMinimumPackageVersion)
+ {
+ _minimumExtensionVersionNotSatisfied(logger, extensionTypeName, extensionTypeAssemblyFullName, minimumAssemblyVersion, requirementPackageName, requirementMinimumPackageVersion, null);
+ }
+
+ public static void MinimumBundleVersionNotSatisfied(this ILogger logger, string bundleId, string bundleVersion, string minimumVersion)
+ {
+ _minimumBundleVersionNotSatisfied(logger, bundleId, bundleVersion, minimumVersion, minimumVersion, null);
+ }
}
}

diff:
<None Remove="runtimes.json" />
</ItemGroup>
<ItemGroup>
+ <EmbeddedResource Include="extensionrequirements.json">
+ <CopyToOutputDirectory>Never</CopyToOutputDirectory>
+ </EmbeddedResource>
<EmbeddedResource Include="FileProvisioning\PowerShell\profile.ps1" />
<EmbeddedResource Include="FileProvisioning\PowerShell\requirements.psd1" />
<EmbeddedResource Include="runtimeassemblies-relaxed.json">

diff:
<_FunctionsSkipCleanOutput>true</_FunctionsSkipCleanOutput>
</PropertyGroup>
<ItemGroup>
- <PackageReference Include="Microsoft.Azure.WebJobs.Extensions.Storage" Version="4.0.3" />
+ <PackageReference Include="Microsoft.Azure.WebJobs.Extensions.Storage" Version="4.0.4" />
<PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" Version="2.2.0" />
<PackageReference Include="Microsoft.NET.Sdk.Functions" Version="3.0.11" />
</ItemGroup>

diff: Binary files /dev/null and b/test/WebJobs.Script.Tests.Integration/TestScripts/OutOfDateExtension/bin/Microsoft.Azure.WebJobs.Extensions.DurableTask.dll differ

diff: Binary files /dev/null and b/test/WebJobs.Script.Tests.Integration/TestScripts/OutOfDateExtension/bin/Microsoft.Azure.WebJobs.Extensions.Storage.dll differ

diff: Binary files /dev/null and b/test/WebJobs.Script.Tests.Integration/TestScripts/OutOfDateExtension/bin/Microsoft.Azure.WebJobs.ServiceBus.dll differ

diff:
+{
+ "extensions":[
+ { "name": "AzureStorage", "typeName":"Microsoft.Azure.WebJobs.Extensions.Storage.AzureStorageWebJobsStartup, Microsoft.Azure.WebJobs.Extensions.Storage, Version=3.0.10.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35"},
+ { "name": "ServiceBus", "typeName":"Microsoft.Azure.WebJobs.ServiceBus.ServiceBusWebJobsStartup, Microsoft.Azure.WebJobs.ServiceBus, Version=4.2.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35"},
+ { "name": "DurableTask", "typeName":"Microsoft.Azure.WebJobs.Extensions.DurableTask.DurableTaskWebJobsStartup, Microsoft.Azure.WebJobs.Extensions.DurableTask, Version=2.0.0.0, Culture=neutral, PublicKeyToken=014045d636e89289"}
+ ]
+}
\ No newline at end of file

diff:
+{
+ "version": "2.0"
+}
\ No newline at end of file


message: Temporarily relax the minimum extension versions to match bundle 2.6.1

change_type: MODIFY

diff:
"bundles": [
{
"id": "Microsoft.Azure.Functions.ExtensionBundle",
- "minimumVersion": "2.7.0"
+ "minimumVersion": "2.6.1"
}
],
"types": [
"name": "ServiceBusWebJobsStartup",
"assemblyName": "Microsoft.Azure.WebJobs.ServiceBus",
"packageName": "Microsoft.Azure.WebJobs.Extensions.ServiceBus",
- "minimumAssemblyVersion": "4.3.0.0",
- "minimumPackageVersion": "4.3.0"
+ "minimumAssemblyVersion": "4.2.1.0",
+ "minimumPackageVersion": "4.2.1"
},
{
"name": "EventHubsWebJobsStartup",
"name": "CosmosDBWebJobsStartup",
"assemblyName": "Microsoft.Azure.WebJobs.Extensions.CosmosDB",
"packageName": "Microsoft.Azure.WebJobs.Extensions.CosmosDB",
- "minimumAssemblyVersion": "3.0.10.0",
- "minimumPackageVersion": "3.0.10"
+ "minimumAssemblyVersion": "3.0.9.0",
+ "minimumPackageVersion": "3.0.9"
},
{
"name": "DurableTaskWebJobsStartup",
"assemblyName": "Microsoft.Azure.WebJobs.Extensions.DurableTask",
"packageName": "Microsoft.Azure.WebJobs.Extensions.DurableTask",
"minimumAssemblyVersion": "2.0.0.0",
- "minimumAssemblyFileVersion": "2.5.0",
- "minimumPackageVersion": "2.5.0"
+ "minimumAssemblyFileVersion": "2.4.1",
+ "minimumPackageVersion": "2.4.1"
},
{
"name": "EventGridWebJobsStartup",
"name": "KafkaWebJobsStartup",
"assemblyName": "Microsoft.Azure.WebJobs.Extensions.Kafka",
"packageName": "Microsoft.Azure.WebJobs.Extensions.Kafka",
- "minimumAssemblyVersion": "3.3.2.0",
- "minimumPackageVersion": "3.3.2"
+ "minimumAssemblyVersion": "3.2.1.0",
+ "minimumPackageVersion": "3.2.1"
}
]
}
\ No newline at end of file

diff:
@@ -490,7 +490,7 @@ public async Task GetExtensionsStartupTypes_RejectsBundleBelowMinimumVersion()
var traces = testLoggerProvider.GetAllLogMessages();
// Assert
- Assert.True(traces.Any(m => string.Equals(m.FormattedMessage, $"Referenced bundle Microsoft.Azure.Functions.ExtensionBundle of version 2.1.0 does not meet the required minimum version of 2.7.0. Update your extension bundle reference in host.json to reference 2.7.0 or later.")));
+ Assert.True(traces.Any(m => string.Equals(m.FormattedMessage, $"Referenced bundle Microsoft.Azure.Functions.ExtensionBundle of version 2.1.0 does not meet the required minimum version of 2.6.1. Update your extension bundle reference in host.json to reference 2.6.1 or later.")));
}
}
@@ -536,16 +536,6 @@ void CopyToBin(string path)
Assert.NotNull(storageTrace);
Assert.Equal("ExtensionStartupType AzureStorageWebJobsStartup from assembly 'Microsoft.Azure.WebJobs.Extensions.Storage, Version=3.0.10.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35' does not meet the required minimum version of 4.0.4.0. Update your NuGet package reference for Microsoft.Azure.WebJobs.Extensions.Storage to 4.0.4 or later.",
storageTrace.FormattedMessage);
-
- var serviceBusTrace = traces.FirstOrDefault(m => m.FormattedMessage.StartsWith("ExtensionStartupType ServiceBusWebJobsStartup"));
- Assert.NotNull(serviceBusTrace);
- Assert.Equal("ExtensionStartupType ServiceBusWebJobsStartup from assembly 'Microsoft.Azure.WebJobs.ServiceBus, Version=4.2.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35' does not meet the required minimum version of 4.3.0.0. Update your NuGet package reference for Microsoft.Azure.WebJobs.Extensions.ServiceBus to 4.3.0 or later.",
- serviceBusTrace.FormattedMessage);
-
- var durableTrace = traces.FirstOrDefault(m => m.FormattedMessage.StartsWith("ExtensionStartupType DurableTaskWebJobsStartup"));
- Assert.NotNull(durableTrace);
- Assert.Equal("ExtensionStartupType DurableTaskWebJobsStartup from assembly 'Microsoft.Azure.WebJobs.Extensions.DurableTask, Version=2.0.0.0, Culture=neutral, PublicKeyToken=014045d636e89289' does not meet the required minimum version of 2.5.0. Update your NuGet package reference for Microsoft.Azure.WebJobs.Extensions.DurableTask to 2.5.0 or later.",
- durableTrace.FormattedMessage);
}
}

license: MIT License

repo: azure/azure-functions-host


diff:
@@ -190,7 +190,7 @@ internal void BuildAndLogConsoleLog(string msg, LogLevel level)
};
if (WorkerProcessUtilities.IsConsoleLog(msg))
{
- _workerProcessLogger?.LogDebug(WorkerProcessUtilities.RemoveLogPrefix(msg));
+ _workerProcessLogger?.Log(level, WorkerProcessUtilities.RemoveLogPrefix(msg));
}
else
{

diff:
@@ -17,7 +17,8 @@ public class WorkerConsoleLogServiceTests
{
private IScriptEventManager _eventManager;
private IProcessRegistry _processRegistry;
- private TestLogger _testLogger = new TestLogger("test");
+ private TestLogger _testUserLogger = new TestLogger("Host.Function.Console");
+ private TestLogger _testSystemLogger = new TestLogger("Worker.rpcWorkerProcess");
private WorkerConsoleLogService _workerConsoleLogService;
private WorkerConsoleLogSource _workerConsoleLogSource;
private Mock<IServiceProvider> _serviceProviderMock;
@@ -30,31 +31,36 @@ public async Task WorkerConsoleLogService_ConsoleLogs_LogLevel_Expected(bool use
_workerConsoleLogSource = new WorkerConsoleLogSource();
_eventManager = new ScriptEventManager();
_processRegistry = new EmptyProcessRegistry();
- _workerConsoleLogService = new WorkerConsoleLogService(_testLogger, _workerConsoleLogSource);
+ _workerConsoleLogService = new WorkerConsoleLogService(_testUserLogger, _workerConsoleLogSource);
_serviceProviderMock = new Mock<IServiceProvider>(MockBehavior.Strict);
- WorkerProcess workerProcess = new TestWorkerProcess(_eventManager, _processRegistry, _testLogger, _workerConsoleLogSource, null, _serviceProviderMock.Object, useStdErrForErroLogsOnly);
+ WorkerProcess workerProcess = new TestWorkerProcess(_eventManager, _processRegistry, _testSystemLogger, _workerConsoleLogSource, null, _serviceProviderMock.Object, useStdErrForErroLogsOnly);
workerProcess.ParseErrorMessageAndLog("Test Message No keyword");
workerProcess.ParseErrorMessageAndLog("Test Error Message");
workerProcess.ParseErrorMessageAndLog("Test Warning Message");
-
- workerProcess.BuildAndLogConsoleLog("LanguageWorkerConsoleLog[Test worker log]", LogLevel.Information);
+ workerProcess.ParseErrorMessageAndLog("LanguageWorkerConsoleLog[Test Worker Message No keyword]");
+ workerProcess.ParseErrorMessageAndLog("LanguageWorkerConsoleLog[Test Worker Error Message]");
+ workerProcess.ParseErrorMessageAndLog("LanguageWorkerConsoleLog[Test Worker Warning Message]");
_ = _workerConsoleLogService.ProcessLogs().ContinueWith(t => { });
await _workerConsoleLogService.StopAsync(System.Threading.CancellationToken.None);
- var allLogs = _testLogger.GetLogMessages();
- Assert.True(allLogs.Count == 4);
- VerifyLogLevel(allLogs, "Test Error Message", LogLevel.Error);
- VerifyLogLevel(allLogs, "Test Warning Message", LogLevel.Warning);
+ var userLogs = _testUserLogger.GetLogMessages();
+ var systemLogs = _testSystemLogger.GetLogMessages();
+ Assert.True(userLogs.Count == 3);
+ Assert.True(systemLogs.Count == 3);
+ VerifyLogLevel(userLogs, "Test Error Message", LogLevel.Error);
+ VerifyLogLevel(systemLogs, "[Test Worker Error Message]", LogLevel.Error);
+ VerifyLogLevel(userLogs, "Test Warning Message", LogLevel.Warning);
+ VerifyLogLevel(systemLogs, "[Test Worker Warning Message]", LogLevel.Warning);
if (useStdErrForErroLogsOnly)
{
- VerifyLogLevel(allLogs, "Test Message No keyword", LogLevel.Error);
+ VerifyLogLevel(userLogs, "Test Message No keyword", LogLevel.Error);
+ VerifyLogLevel(systemLogs, "[Test Worker Message No keyword]", LogLevel.Error);
}
else
{
- VerifyLogLevel(allLogs, "Test Message No keyword", LogLevel.Information);
+ VerifyLogLevel(userLogs, "Test Message No keyword", LogLevel.Information);
+ VerifyLogLevel(systemLogs, "[Test Worker Message No keyword]", LogLevel.Information);
}
-
- VerifyLogLevel(allLogs, "[Test worker log]", LogLevel.Debug);
}
private static void VerifyLogLevel(IList<LogMessage> allLogs, string msg, LogLevel expectedLevel)

original_message: Use appropriate log level for worker console logs (#8102)


message:
Update Node.js Worker Version to 3.1.0
[Release notes](https://github.com/Azure/azure-functions-nodejs-worker/releases/tag/v3.1.0)

diff:
-->
- Fixing race during language worker start (#7979)
- Updated Java Worker Version to [2.2.1-SNAPSHOT](https://github.com/Azure/azure-functions-java-worker/releases/tag/2.2.1-SNAPSHOT)
+- Updated Node.js Worker Version to [3.1.0](https://github.com/Azure/azure-functions-nodejs-worker/releases/tag/v3.1.0)
**Release sprint:** Sprint 113
[ [bugs](https://github.com/Azure/azure-functions-host/issues?q=is%3Aissue+milestone%3A%22Functions+Sprint+113%22+label%3Abug+is%3Aclosed) | [features](https://github.com/Azure/azure-functions-host/issues?q=is%3Aissue+milestone%3A%22Functions+Sprint+113%22+label%3Afeature+is%3Aclosed) ]

diff:
<!-- Workers -->
<PackageReference Include="Microsoft.Azure.Functions.JavaWorker" Version="2.2.1-SNAPSHOT" />
- <PackageReference Include="Microsoft.Azure.Functions.NodeJsWorker" Version="3.0.0" />
+ <PackageReference Include="Microsoft.Azure.Functions.NodeJsWorker" Version="3.1.0" />
<PackageReference Include="Microsoft.Azure.Functions.PowerShellWorker.PS7.0" Version="4.0.1468" />
<PackageReference Include="Microsoft.Azure.Functions.PowerShellWorker.PS7.2" Version="4.0.1479" />
<PackageReference Include="Microsoft.Azure.WebJobs.Extensions" Version="5.0.0-beta.1" />

diff:
<PackageReference Include="Microsoft.AspNetCore.TestHost" Version="6.0.0" />
<PackageReference Include="Microsoft.Azure.DocumentDB.Core" Version="2.11.2" />
<PackageReference Include="Microsoft.Azure.EventHubs" Version="2.1.0" />
- <PackageReference Include="Microsoft.Azure.Functions.NodeJsWorker" Version="3.0.0" />
+ <PackageReference Include="Microsoft.Azure.Functions.NodeJsWorker" Version="3.1.0" />
<PackageReference Include="Microsoft.Azure.WebJobs.Extensions.Storage" Version="4.0.5-11874" />
<PackageReference Include="Microsoft.Azure.Functions.JavaWorker" Version="2.2.1-SNAPSHOT" />
<PackageReference Include="Microsoft.Azure.Mobile.Client" Version="4.0.2" />

original_message:
Update Node.js Worker Version to 3.1.0 (#8131)

[Release notes](https://github.com/Azure/azure-functions-nodejs-worker/releases/tag/v3.1.0)


message:
Add function app dir to worker init request
Updated protobuf subtree to tag [v1.5.5-protofile](https://github.com/Azure/azure-functions-language-worker-protobuf/releases/tag/v1.5.5-protofile)

diff:
@@ -228,7 +228,8 @@ internal WorkerInitRequest GetWorkerInitRequest()
return new WorkerInitRequest()
{
HostVersion = ScriptHost.Version,
- WorkerDirectory = _workerConfig.Description.WorkerDirectory
+ WorkerDirectory = _workerConfig.Description.WorkerDirectory,
+ FunctionAppDirectory = _applicationHostOptions.CurrentValue.ScriptPath
};
}

diff:
@@ -113,6 +113,9 @@ message WorkerInitRequest {
// Full path of worker.config.json location
string worker_directory = 4;
+
+ // base directory for function app
+ string function_app_directory = 5;
}
// Worker responds with the result of initializing itself

diff:
@@ -225,8 +225,10 @@ public void WorkerInitRequest_Expected()
{
WorkerInitRequest initRequest = _workerChannel.GetWorkerInitRequest();
Assert.NotNull(initRequest.WorkerDirectory);
+ Assert.NotNull(initRequest.FunctionAppDirectory);
Assert.NotNull(initRequest.HostVersion);
Assert.Equal("testDir", initRequest.WorkerDirectory);
+ Assert.Equal(_scriptRootPath, initRequest.FunctionAppDirectory);
Assert.Equal(ScriptHost.Version, initRequest.HostVersion);
}

original_message:
Add function app dir to worker init request (#8144)

Updated protobuf subtree to tag [v1.5.5-protofile](https://github.com/Azure/azure-functions-language-worker-protobuf/releases/tag/v1.5.5-protofile)


message: Update Node.js Worker Version to 3.2.0

diff:
-->
- Add admin/host/config API (#7394)
- Updated Java Worker Version to [2.2.2-SNAPSHOT](https://github.com/Azure/azure-functions-java-worker/releases/tag/2.2.2-SNAPSHOT)
+- Updated Node.js Worker Version to [3.2.0](https://github.com/Azure/azure-functions-nodejs-worker/releases/tag/v3.2.0)
**Release sprint:** Sprint 118
[ [bugs](https://github.com/Azure/azure-functions-host/issues?q=is%3Aissue+milestone%3A%22Functions+Sprint+118%22+label%3Abug+is%3Aclosed) | [features](https://github.com/Azure/azure-functions-host/issues?q=is%3Aissue+milestone%3A%22Functions+Sprint+118%22+label%3Afeature+is%3Aclosed) ]

diff:
<!-- Workers -->
<PackageReference Include="Microsoft.Azure.Functions.JavaWorker" Version="2.2.2-SNAPSHOT" />
- <PackageReference Include="Microsoft.Azure.Functions.NodeJsWorker" Version="3.1.0" />
+ <PackageReference Include="Microsoft.Azure.Functions.NodeJsWorker" Version="3.2.0" />
<PackageReference Include="Microsoft.Azure.Functions.PowerShellWorker.PS7.0" Version="4.0.1570" />
<PackageReference Include="Microsoft.Azure.Functions.PowerShellWorker.PS7.2" Version="4.0.1567" />
<PackageReference Include="Microsoft.Azure.WebJobs.Extensions" Version="5.0.0-beta.1" />

diff:
@@ -193,7 +193,7 @@ public async Task Scenario_Logging()
{
userLogs = Fixture.Host.GetScriptHostLogMessages(userCategory).Select(p => p.FormattedMessage).ToList();
consoleLog = Fixture.Host.GetScriptHostLogMessages(WorkerConstants.FunctionConsoleLogCategoryName).Select(p => p.FormattedMessage).SingleOrDefault();
- return userLogs.Count == 10 && consoleLog != null;
+ return userLogs.Count == 11 && consoleLog != null;
}, userMessageCallback: Fixture.Host.GetLog);
// verify use of context.log to log complex objects

diff:
<PackageReference Include="Microsoft.AspNetCore.TestHost" Version="6.0.0" />
<PackageReference Include="Microsoft.Azure.DocumentDB.Core" Version="2.11.2" />
<PackageReference Include="Microsoft.Azure.EventHubs" Version="2.1.0" />
- <PackageReference Include="Microsoft.Azure.Functions.NodeJsWorker" Version="3.1.0" />
+ <PackageReference Include="Microsoft.Azure.Functions.NodeJsWorker" Version="3.2.0" />
<PackageReference Include="Microsoft.Azure.WebJobs.Extensions.Storage" Version="4.0.5-11874" />
<PackageReference Include="Microsoft.Azure.Functions.JavaWorker" Version="2.2.2-SNAPSHOT" />
<PackageReference Include="Microsoft.Azure.Mobile.Client" Version="4.0.2" />

original_message: Update Node.js Worker Version to 3.2.0 (#8259)


message: Update Node.js Worker Version to 3.3.0

diff:
- My change description (#PR)
-->
- Updated Java Worker Version to [2.2.3](https://github.com/Azure/azure-functions-java-worker/releases/tag/2.2.3)
+- Updated Node.js Worker Version to [3.3.0](https://github.com/Azure/azure-functions-nodejs-worker/releases/tag/v3.3.0)
**Release sprint:** Sprint 118
[ [bugs](https://github.com/Azure/azure-functions-host/issues?q=is%3Aissue+milestone%3A%22Functions+Sprint+118%22+label%3Abug+is%3Aclosed) | [features](https://github.com/Azure/azure-functions-host/issues?q=is%3Aissue+milestone%3A%22Functions+Sprint+118%22+label%3Afeature+is%3Aclosed) ]

diff:
<!-- Workers -->
<PackageReference Include="Microsoft.Azure.Functions.JavaWorker" Version="2.2.3" />
- <PackageReference Include="Microsoft.Azure.Functions.NodeJsWorker" Version="3.2.0" />
+ <PackageReference Include="Microsoft.Azure.Functions.NodeJsWorker" Version="3.3.0" />
<PackageReference Include="Microsoft.Azure.Functions.PowerShellWorker.PS7.0" Version="4.0.1570" />
<PackageReference Include="Microsoft.Azure.Functions.PowerShellWorker.PS7.2" Version="4.0.1567" />
<PackageReference Include="Microsoft.Azure.WebJobs.Extensions" Version="5.0.0-beta.1" />

diff:
<PackageReference Include="Microsoft.AspNetCore.TestHost" Version="6.0.0" />
<PackageReference Include="Microsoft.Azure.DocumentDB.Core" Version="2.11.2" />
<PackageReference Include="Microsoft.Azure.EventHubs" Version="2.1.0" />
- <PackageReference Include="Microsoft.Azure.Functions.NodeJsWorker" Version="3.2.0" />
+ <PackageReference Include="Microsoft.Azure.Functions.NodeJsWorker" Version="3.3.0" />
<PackageReference Include="Microsoft.Azure.WebJobs.Extensions.Storage" Version="4.0.5-11874" />
<PackageReference Include="Microsoft.Azure.Functions.JavaWorker" Version="2.2.3" />
<PackageReference Include="Microsoft.Azure.Mobile.Client" Version="4.0.2" />

original_message: Update Node.js Worker Version to 3.3.0 (#8293)


diff:
@@ -158,7 +158,7 @@ private void OnProcessExited(object sender, EventArgs e)
}
else
{
- var processExitEx = new WorkerProcessExitException($"{Process.StartInfo.FileName} exited with code {Process.ExitCode}", new Exception(exceptionMessage));
+ var processExitEx = new WorkerProcessExitException($"{Process.StartInfo.FileName} exited with code {Process.ExitCode} (0x{Process.ExitCode.ToString("X")})", new Exception(exceptionMessage));
processExitEx.ExitCode = Process.ExitCode;
processExitEx.Pid = Process.Id;
HandleWorkerProcessExitError(processExitEx);

original_message: Add hex exit code to message when worker fails (#8260)


diff:
@@ -100,14 +100,6 @@ function CleanOutput([string] $rootPath) {
Write-Host " Removing python worker"
Remove-Item -Recurse -Force "$rootPath\workers\python" -ErrorAction SilentlyContinue
- Write-Host " Removing non-win32 node grpc binaries"
- Get-ChildItem "$rootPath\workers\node\grpc\src\node\extension_binary" -ErrorAction SilentlyContinue |
- Foreach-Object {
- if (-Not ($_.FullName -Match "win32")) {
- Remove-Item -Recurse -Force $_.FullName
- }
- }
-
$keepRuntimes = @('win', 'win-x86', 'win10-x86', 'win-x64', 'win10-x64')
Write-Host " Removing all powershell runtimes except $keepRuntimes"
Get-ChildItem "$rootPath\workers\powershell" -Directory -ErrorAction SilentlyContinue |

original_message: No longer need to remove node grpc binaries in v4 (#8306)


message: Update Node.js Worker Version to 3.4.0

diff:
- Update Python Worker Version to [4.2.0-hotfix](https://github.com/Azure/azure-functions-python-worker/releases/tag/4.2.0-hotfix)
- Return 500 instead of 403 when writing secrets to storage fails (#8362)
- Updated Java Worker Version to [2.2.3](https://github.com/Azure/azure-functions-java-worker/releases/tag/2.2.3)
-- Updated Node.js Worker Version to [3.3.0](https://github.com/Azure/azure-functions-nodejs-worker/releases/tag/v3.3.0)
+- Updated Node.js Worker Version to [3.4.0](https://github.com/Azure/azure-functions-nodejs-worker/releases/tag/v3.4.0)
- Add Include minimum durable payload to sync trigger (#8292)
- Enable diagnostic events collection
- Added support for no storage account scenarios (#8083)

diff:
<!-- Workers -->
<PackageReference Include="Microsoft.Azure.Functions.JavaWorker" Version="2.2.3" />
- <PackageReference Include="Microsoft.Azure.Functions.NodeJsWorker" Version="3.3.0" />
+ <PackageReference Include="Microsoft.Azure.Functions.NodeJsWorker" Version="3.4.0" />
<PackageReference Include="Microsoft.Azure.Functions.PowerShellWorker.PS7.0" Version="4.0.2045" />
<PackageReference Include="Microsoft.Azure.Functions.PowerShellWorker.PS7.2" Version="4.0.2040" />
<PackageReference Include="Microsoft.Azure.WebJobs.Extensions" Version="5.0.0-beta.2-10871" />

diff:
<PackageReference Include="Microsoft.AspNetCore.TestHost" Version="6.0.0" />
<PackageReference Include="Microsoft.Azure.DocumentDB.Core" Version="2.11.2" />
<PackageReference Include="Microsoft.Azure.EventHubs" Version="2.1.0" />
- <PackageReference Include="Microsoft.Azure.Functions.NodeJsWorker" Version="3.3.0" />
+ <PackageReference Include="Microsoft.Azure.Functions.NodeJsWorker" Version="3.4.0" />
<PackageReference Include="Microsoft.Azure.WebJobs.Extensions.Storage" Version="4.0.5-11874" />
<PackageReference Include="Microsoft.Azure.Functions.JavaWorker" Version="2.2.3" />
<PackageReference Include="Microsoft.Azure.Mobile.Client" Version="4.0.2" />

original_message: Update Node.js Worker Version to 3.4.0 (#8467)


message: Update Node.js Worker Version to 3.5.0

diff:
-->
- Update Python Worker Version to [4.6.0](https://github.com/Azure/azure-functions-python-worker/releases/tag/4.6.0)
- Update Python Worker Version to [4.5.0](https://github.com/Azure/azure-functions-python-worker/releases/tag/4.5.0)
-
+- Update Node.js Worker Version to [3.5.0](https://github.com/Azure/azure-functions-nodejs-worker/releases/tag/v3.5.0)
- Host support for out-of-proc cancellation tokens ([#2153](https://github.com/Azure/azure-functions-host/issues/2152))
- Updated Java Worker Version to [2.4.0](https://github.com/Azure/azure-functions-java-worker/releases/tag/2.4.0)

diff:
<!-- Workers -->
<PackageReference Include="Microsoft.Azure.Functions.JavaWorker" Version="2.4.0" />
- <PackageReference Include="Microsoft.Azure.Functions.NodeJsWorker" Version="3.4.0" />
+ <PackageReference Include="Microsoft.Azure.Functions.NodeJsWorker" Version="3.5.0" />
<PackageReference Include="Microsoft.Azure.Functions.PowerShellWorker.PS7.0" Version="4.0.2045" />
<PackageReference Include="Microsoft.Azure.Functions.PowerShellWorker.PS7.2" Version="4.0.2040" />
<PackageReference Include="Microsoft.Azure.WebJobs.Extensions" Version="5.0.0-beta.2-10879" />

diff:
<PackageReference Include="Microsoft.AspNetCore.TestHost" Version="6.0.0" />
<PackageReference Include="Microsoft.Azure.DocumentDB.Core" Version="2.11.2" />
<PackageReference Include="Microsoft.Azure.EventHubs" Version="2.1.0" />
- <PackageReference Include="Microsoft.Azure.Functions.NodeJsWorker" Version="3.4.0" />
+ <PackageReference Include="Microsoft.Azure.Functions.NodeJsWorker" Version="3.5.0" />
<PackageReference Include="Microsoft.Azure.WebJobs.Extensions.Storage" Version="4.0.5-11874" />
<PackageReference Include="Microsoft.Azure.Functions.JavaWorker" Version="2.4.0" />
<PackageReference Include="Microsoft.Azure.Mobile.Client" Version="4.0.2" />

original_message: Update Node.js Worker Version to 3.5.0 (#8668)


message: Update Node.js Worker Version to 3.5.1

diff:
- My change description (#PR)
-->
- Update Java Worker Version to [2.6.0](https://github.com/Azure/azure-functions-java-worker/releases/tag/2.6.0)
-- Revert Node.js Worker Version to [3.4.0](https://github.com/Azure/azure-functions-nodejs-worker/releases/tag/v3.4.0)
+- Update Node.js Worker Version to [3.5.1](https://github.com/Azure/azure-functions-nodejs-worker/releases/tag/v3.5.1)
- Update PowerShell Worker 7.0 to 4.0.2302 [Release Note](https://github.com/Azure/azure-functions-powershell-worker/releases/tag/v4.0.2302)
- Update PowerShell Worker 7.2 to 4.0.2304 [Release Note](https://github.com/Azure/azure-functions-powershell-worker/releases/tag/v4.0.2304)

diff:
<!-- Workers -->
<PackageReference Include="Microsoft.Azure.AppService.Proxy.Client" Version="2.2.20220831.41" />
<PackageReference Include="Microsoft.Azure.Functions.JavaWorker" Version="2.6.0" />
- <PackageReference Include="Microsoft.Azure.Functions.NodeJsWorker" Version="3.4.0" />
+ <PackageReference Include="Microsoft.Azure.Functions.NodeJsWorker" Version="3.5.1" />
<PackageReference Include="Microsoft.Azure.Functions.PowerShellWorker.PS7.0" Version="4.0.2302" />
<PackageReference Include="Microsoft.Azure.Functions.PowerShellWorker.PS7.2" Version="4.0.2304" />
<PackageReference Include="Microsoft.Azure.WebJobs.Extensions" Version="5.0.0-beta.2-10879" />

diff:
<PackageReference Include="Microsoft.AspNetCore.TestHost" Version="6.0.0" />
<PackageReference Include="Microsoft.Azure.DocumentDB.Core" Version="2.11.2" />
<PackageReference Include="Microsoft.Azure.EventHubs" Version="2.1.0" />
- <PackageReference Include="Microsoft.Azure.Functions.NodeJsWorker" Version="3.4.0" />
+ <PackageReference Include="Microsoft.Azure.Functions.NodeJsWorker" Version="3.5.1" />
<PackageReference Include="Microsoft.Azure.WebJobs.Extensions.Storage" Version="4.0.5-11874" />
<PackageReference Include="Microsoft.Azure.Functions.JavaWorker" Version="2.6.0" />
<PackageReference Include="Microsoft.Azure.Mobile.Client" Version="4.0.2" />

original_message: Update Node.js Worker Version to 3.5.1 (#8817)


message: Architecture settings and readme updates

change_type: ADD

diff:
+# Contributing to FAIR Sequence-to-Sequence Toolkit (PyTorch)
+We want to make contributing to this project as easy and transparent as
+possible.
+
+## Pull Requests
+We actively welcome your pull requests.
+
+1. Fork the repo and create your branch from `master`.
+2. If you've added code that should be tested, add tests.
+3. If you've changed APIs, update the documentation.
+4. Ensure the test suite passes.
+5. Make sure your code lints.
+6. If you haven't already, complete the Contributor License Agreement ("CLA").
+
+## Contributor License Agreement ("CLA")
+In order to accept your pull request, we need you to submit a CLA. You only need
+to do this once to work on any of Facebook's open source projects.
+
+Complete your CLA here: <https://code.facebook.com/cla>
+
+## Issues
+We use GitHub issues to track public bugs. Please ensure your description is
+clear and has sufficient instructions to be able to reproduce the issue.
+
+## Coding Style
+We try to follow the PEP style guidelines and encourage you to as well.
+
+## License
+By contributing to FAIR Sequence-to-Sequence Toolkit, you agree that your contributions will be licensed
+under the LICENSE file in the root directory of this source tree.
\ No newline at end of file

diff:
# Introduction
FAIR Sequence-to-Sequence Toolkit (PyTorch)
-This is a PyTorch version of [fairseq](https://github.com/facebookresearch/fairseq), a sequence-to-sequence learning toolkit from Facebook AI Research. The original authors of this reimplementation are (in no particular order) Sergey Edunov, Myle Ott, and Sam Gross. The toolkit implements the fully convolutional model described in [Convolutional Sequence to Sequence Learning](https://arxiv.org/abs/1705.03122). The toolkit features multi-GPU training on a single machine as well as fast beam search generation on both CPU and GPU. We provide pre-trained models for English to French and English to German translation.
+This is a PyTorch version of [fairseq](https://github.com/facebookresearch/fairseq), a sequence-to-sequence learning toolkit from Facebook AI Research. The original authors of this reimplementation are (in no particular order) Sergey Edunov, Myle Ott, and Sam Gross. The toolkit implements the fully convolutional model described in [Convolutional Sequence to Sequence Learning](https://arxiv.org/abs/1705.03122) and features multi-GPU training on a single machine as well as fast beam search generation on both CPU and GPU. We provide pre-trained models for English to French and English to German translation.
![Model](fairseq.gif)
@@ -27,8 +27,9 @@ If you use the code in your paper, then please cite it as:
Currently fairseq-py requires PyTorch from the GitHub repository. There are multiple ways of installing it.
We suggest using [Miniconda3](https://conda.io/miniconda.html) and the following instructions.
-* Install Miniconda3 from https://conda.io/miniconda.html create and activate python 3 environment.
+* Install Miniconda3 from https://conda.io/miniconda.html; create and activate a Python 3 environment.
+* Install PyTorch:
```
conda install gcc numpy cudnn nccl
conda install magma-cuda80 -c soumith
@@ -44,15 +45,15 @@ pip install -r requirements.txt
NO_DISTRIBUTED=1 python setup.py install
```
-
-Install fairseq by cloning the GitHub repository and by running
-
+* Install fairseq-py by cloning the GitHub repository and running:
```
pip install -r requirements.txt
python setup.py build
python setup.py develop
```
+# Quick Start
+
The following command-line tools are available:
* `python preprocess.py`: Data pre-processing: build vocabularies and binarize training data
* `python train.py`: Train a new model on one or multiple GPUs
@@ -60,9 +61,6 @@ The following command-line tools are available:
* `python generate.py -i`: Translate raw text with a trained model
* `python score.py`: BLEU scoring of generated translations against reference translations
-
-# Quick Start
-
## Evaluating Pre-trained Models [TO BE ADAPTED]
First, download a pre-trained model along with its vocabularies:
```
@@ -100,7 +98,7 @@ Check [below](#pre-trained-models) for a full list of pre-trained models availab
## Training a New Model
### Data Pre-processing
-The fairseq source distribution contains an example pre-processing script for
+The fairseq-py source distribution contains an example pre-processing script for
the IWSLT 2014 German-English corpus.
Pre-process and binarize the data as follows:
```
@@ -118,11 +116,10 @@ This will write binarized data that can be used for model training to `data-bin/
Use `python train.py` to train a new model.
Here a few example settings that work well for the IWSLT 2014 dataset:
```
-$ mkdir -p trainings/fconv
+$ mkdir -p checkpoints/fconv
$ CUDA_VISIBLE_DEVICES=0 python train.py data-bin/iwslt14.tokenized.de-en \
--lr 0.25 --clip-norm 0.1 --dropout 0.2 --max-tokens 4000 \
- --encoder-layers "[(256, 3)] * 4" --decoder-layers "[(256, 3)] * 3" \
- --encoder-embed-dim 256 --decoder-embed-dim 256 --save-dir trainings/fconv
+ --arch fconv_iwslt_de_en --save-dir checkpoints/fconv
```
By default, `python train.py` will use all available GPUs on your machine.
@@ -135,7 +132,7 @@ You may need to use a smaller value depending on the available GPU memory on you
Once your model is trained, you can generate translations using `python generate.py` **(for binarized data)** or `python generate.py -i` **(for raw text)**:
```
$ python generate.py data-bin/iwslt14.tokenized.de-en \
- --path trainings/fconv/checkpoint_best.pt \
+ --path checkpoints/fconv/checkpoint_best.pt \
--batch-size 128 --beam 5
| [de] dictionary: 35475 types
| [en] dictionary: 24739 types
@@ -172,9 +169,12 @@ $ python generate.py data-bin/wmt14.en-fr.newstest2014 \
...
| Translated 3003 sentences (95451 tokens) in 136.3s (700.49 tokens/s)
| Timings: setup 0.1s (0.1%), encoder 1.9s (1.4%), decoder 108.9s (79.9%), search_results 0.0s (0.0%), search_prune 12.5s (9.2%)
+TODO: update scores (should be same as score.py)
| BLEU4 = 43.43, 68.2/49.2/37.4/28.8 (BP=0.996, ratio=1.004, sys_len=92087, ref_len=92448)
-# Word-level BLEU scoring:
+# Scoring with score.py:
+$ grep ^H /tmp/gen.out | cut -f3- | sed 's/@@ //g' > /tmp/gen.out.sys
+$ grep ^T /tmp/gen.out | cut -f2- | sed 's/@@ //g' > /tmp/gen.out.ref
$ python score.py --sys /tmp/gen.out.sys --ref /tmp/gen.out.ref
TODO: update scores
BLEU4 = 40.55, 67.6/46.5/34.0/25.3 (BP=1.000, ratio=0.998, sys_len=81369, ref_len=81194)
@@ -186,6 +186,6 @@ BLEU4 = 40.55, 67.6/46.5/34.0/25.3 (BP=1.000, ratio=0.998, sys_len=81369, ref_le
* Google group: https://groups.google.com/forum/#!forum/fairseq-users
# License
-fairseq is BSD-licensed.
+fairseq-py is BSD-licensed.
The license applies to the pre-trained models as well.
We also provide an additional patent grant.

diff:
# can be found in the PATENTS file in the same directory.
#
-from .fconv import *
+from . import fconv
-__all__ = [
- 'fconv', 'fconv_iwslt_de_en', 'fconv_wmt_en_ro', 'fconv_wmt_en_de',
- 'fconv_wmt_en_fr',
-]
+
+__all__ = ['fconv']
+
+arch_model_map = {}
+for model in __all__:
+ archs = locals()[model].get_archs()
+ for arch in archs:
+ assert arch not in arch_model_map, 'Duplicate model architecture detected: {}'.format(arch)
+ arch_model_map[arch] = model

diff:
@@ -430,56 +430,90 @@ class GradMultiply(torch.autograd.Function):
return grad * ctx.scale, None
-def fconv_iwslt_de_en(dataset, dropout, **kwargs):
- encoder_convs = [(256, 3)] * 4
- decoder_convs = [(256, 3)] * 3
- return fconv(dataset, dropout, 256, encoder_convs, 256, decoder_convs, **kwargs)
-
-
-def fconv_wmt_en_ro(dataset, dropout, **kwargs):
- convs = [(512, 3)] * 20
- return fconv(dataset, dropout, 512, convs, 512, convs, **kwargs)
-
-
-def fconv_wmt_en_de(dataset, dropout, **kwargs):
- convs = [(512, 3)] * 9 # first 10 layers have 512 units
- convs += [(1024, 3)] * 4 # next 3 layers have 768 units
- convs += [(2048, 1)] * 2 # final 2 layers are 1x1
- return fconv(dataset, dropout, 768, convs, 768, convs,
- decoder_out_embed_dim=512,
- **kwargs)
-
+def get_archs():
+ return [
+ 'fconv', 'fconv_iwslt_de_en', 'fconv_wmt_en_ro', 'fconv_wmt_en_de', 'fconv_wmt_en_fr',
+ ]
+
+
+def _check_arch(args):
+ """Check that the specified architecture is valid and not ambiguous."""
+ if args.arch not in get_archs():
+ raise ValueError('Unknown fconv model architecture: {}'.format(args.arch))
+ if args.arch != 'fconv':
+ # check that architecture is not ambiguous
+ for a in ['encoder_embed_dim', 'encoder_layers', 'decoder_embed_dim', 'decoder_layers',
+ 'decoder_out_embed_dim']:
+ if hasattr(args, a):
+ raise ValueError('--{} cannot be combined with --arch={}'.format(a, args.arch))
+
+
+def parse_arch(args):
+ _check_arch(args)
+
+ if args.arch == 'fconv_iwslt_de_en':
+ args.encoder_embed_dim = 256
+ args.encoder_layers = '[(256, 3)] * 4'
+ args.decoder_embed_dim = 256
+ args.decoder_layers = '[(256, 3)] * 3'
+ args.decoder_out_embed_dim = 256
+ elif args.arch == 'fconv_wmt_en_ro':
+ args.encoder_embed_dim = 512
+ args.encoder_layers = '[(512, 3)] * 20'
+ args.decoder_embed_dim = 512
+ args.decoder_layers = '[(512, 3)] * 20'
+ args.decoder_out_embed_dim = 512
+ elif args.arch == 'fconv_wmt_en_de':
+ convs = '[(512, 3)] * 9' # first 9 layers have 512 units
+ convs += ' + [(1024, 3)] * 4' # next 4 layers have 1024 units
+ convs += ' + [(2048, 1)] * 2' # final 2 layers use 1x1 convolutions
+ args.encoder_embed_dim = 768
+ args.encoder_layers = convs
+ args.decoder_embed_dim = 768
+ args.decoder_layers = convs
+ args.decoder_out_embed_dim = 512
+ elif args.arch == 'fconv_wmt_en_fr':
+ convs = '[(512, 3)] * 6' # first 6 layers have 512 units
+ convs += ' + [(768, 3)] * 4' # next 4 layers have 768 units
+ convs += ' + [(1024, 3)] * 3' # next 3 layers have 1024 units
+ convs += ' + [(2048, 1)] * 1' # next 1 layer uses 1x1 convolutions
+ convs += ' + [(4096, 1)] * 1' # final 1 layer uses 1x1 convolutions
+ args.encoder_embed_dim = 768
+ args.encoder_layers = convs
+ args.decoder_embed_dim = 768
+ args.decoder_layers = convs
+ args.decoder_out_embed_dim = 512
+ else:
+ assert args.arch == 'fconv'
-def fconv_wmt_en_fr(dataset, dropout, **kwargs):
- convs = [(512, 3)] * 6 # first 5 layers have 512 units
- convs += [(768, 3)] * 4 # next 4 layers have 768 units
- convs += [(1024, 3)] * 3 # next 4 layers have 1024 units
- convs += [(2048, 1)] * 1 # next 1 layer is 1x1
- convs += [(4096, 1)] * 1 # final 1 layer is 1x1
- return fconv(dataset, dropout, 768, convs, 768, convs,
- decoder_out_embed_dim=512,
- **kwargs)
+ # default architecture
+ args.encoder_embed_dim = getattr(args, 'encoder_embed_dim', 512)
+ args.encoder_layers = getattr(args, 'encoder_layers', '[(512, 3)] * 20')
+ args.decoder_embed_dim = getattr(args, 'decoder_embed_dim', 512)
+ args.decoder_layers = getattr(args, 'decoder_layers', '[(512, 3)] * 20')
+ args.decoder_out_embed_dim = getattr(args, 'decoder_out_embed_dim', 256)
+ args.decoder_attention = getattr(args, 'decoder_attention', 'True')
+ return args
-def fconv(dataset, dropout, encoder_embed_dim, encoder_convolutions,
- decoder_embed_dim, decoder_convolutions, attention=True,
- decoder_out_embed_dim=256, max_positions=1024):
+def build_model(args, dataset):
padding_idx = dataset.dst_dict.pad()
-
encoder = Encoder(
len(dataset.src_dict),
- embed_dim=encoder_embed_dim,
- convolutions=encoder_convolutions,
- dropout=dropout,
+ embed_dim=args.encoder_embed_dim,
+ convolutions=eval(args.encoder_layers),
+ dropout=args.dropout,
padding_idx=padding_idx,
- max_positions=max_positions)
+ max_positions=args.max_positions,
+ )
decoder = Decoder(
len(dataset.dst_dict),
- embed_dim=decoder_embed_dim,
- convolutions=decoder_convolutions,
- out_embed_dim=decoder_out_embed_dim,
- attention=attention,
- dropout=dropout,
+ embed_dim=args.decoder_embed_dim,
+ convolutions=eval(args.decoder_layers),
+ out_embed_dim=args.decoder_out_embed_dim,
+ attention=eval(args.decoder_attention),
+ dropout=args.dropout,
padding_idx=padding_idx,
- max_positions=max_positions)
+ max_positions=args.max_positions,
+ )
return FConvModel(encoder, decoder, padding_idx)

diff:
@@ -109,22 +109,35 @@ def add_generation_args(parser):
def add_model_args(parser):
- group = parser.add_argument_group('Model configuration')
- group.add_argument('--arch', '-a', default='fconv', metavar='ARCH',
- choices=models.__all__,
- help='model architecture ({})'.format(', '.join(models.__all__)))
- group.add_argument('--encoder-embed-dim', default=512, type=int, metavar='N',
+ group = parser.add_argument_group(
+ 'Model configuration',
+ # Only include attributes which are explicitly given as command-line
+ # arguments or which have model-independent default values.
+ argument_default=argparse.SUPPRESS,
+ )
+
+ # The model architecture can be specified in several ways.
+ # In increasing order of priority:
+ # 1) model defaults (lowest priority)
+ # 2) --arch argument
+ # 3) --encoder/decoder-* arguments (highest priority)
+ # Note: --arch cannot be combined with --encoder/decoder-* arguments.
+ group.add_argument('--arch', '-a', default='fconv', metavar='ARCH', choices=models.arch_model_map.keys(),
+ help='model architecture ({})'.format(', '.join(models.arch_model_map.keys())))
+ group.add_argument('--encoder-embed-dim', type=int, metavar='N',
help='encoder embedding dimension')
- group.add_argument('--encoder-layers', default='[(512, 3)] * 20', type=str, metavar='EXPR',
+ group.add_argument('--encoder-layers', type=str, metavar='EXPR',
help='encoder layers [(dim, kernel_size), ...]')
- group.add_argument('--decoder-embed-dim', default=512, type=int, metavar='N',
+ group.add_argument('--decoder-embed-dim', type=int, metavar='N',
help='decoder embedding dimension')
- group.add_argument('--decoder-layers', default='[(512, 3)] * 20', type=str, metavar='EXPR',
+ group.add_argument('--decoder-layers', type=str, metavar='EXPR',
help='decoder layers [(dim, kernel_size), ...]')
- group.add_argument('--decoder-attention', default='True', type=str, metavar='EXPR',
- help='decoder attention [True, ...]')
- group.add_argument('--decoder-out-embed-dim', default=256, type=int, metavar='N',
+ group.add_argument('--decoder-out-embed-dim', type=int, metavar='N',
help='decoder output embedding dimension')
+ group.add_argument('--decoder-attention', type=str, metavar='EXPR',
+ help='decoder attention [True, ...]')
+
+ # These arguments have default values independent of the model:
group.add_argument('--dropout', default=0.1, type=float, metavar='D',
help='dropout probability')
group.add_argument('--label-smoothing', default=0, type=float, metavar='D',

diff:
@@ -14,20 +14,16 @@ from torch.serialization import default_restore_location
from fairseq import criterions, data, models
+def parse_args_and_arch(parser):
+ args = parser.parse_args()
+ args.model = models.arch_model_map[args.arch]
+ args = getattr(models, args.model).parse_arch(args)
+ return args
+
+
def build_model(args, dataset):
- if args.arch == 'fconv':
- encoder_layers = eval(args.encoder_layers)
- decoder_layers = eval(args.decoder_layers)
- decoder_attention = eval(args.decoder_attention)
- model = models.fconv(
- dataset, args.dropout, args.encoder_embed_dim, encoder_layers,
- args.decoder_embed_dim, decoder_layers, decoder_attention,
- decoder_out_embed_dim=args.decoder_out_embed_dim,
- max_positions=args.max_positions)
- else:
- model = models.__dict__[args.arch](dataset, args.dropout,
- max_positions=args.max_positions)
- return model
+ assert hasattr(models, args.model), 'Missing model type'
+ return getattr(models, args.model).build_model(args, dataset)
def build_criterion(args, dataset):
@@ -95,14 +91,14 @@ def load_checkpoint(filename, model, optimizer, lr_scheduler, cuda_device=None):
return epoch, batch_offset
-def load_ensemble_for_inference(models, data_path):
+def load_ensemble_for_inference(filenames, data_path):
# load model architectures and weights
states = []
- for model in models:
- if not os.path.exists(model):
- raise IOError('Model file not found: ' + model)
+ for filename in filenames:
+ if not os.path.exists(filename):
+ raise IOError('Model file not found: {}'.format(filename))
states.append(
- torch.load(model, map_location=lambda s, l: default_restore_location(s, 'cpu'))
+ torch.load(filename, map_location=lambda s, l: default_restore_location(s, 'cpu'))
)
# load dataset
@@ -110,13 +106,13 @@ def load_ensemble_for_inference(models, data_path):
dataset = data.load(data_path, args.source_lang, args.target_lang)
# build models
- models = []
+ ensemble = []
for state in states:
model = build_model(args, dataset)
model.load_state_dict(state['model'])
- models.append(model)
+ ensemble.append(model)
- return models, dataset
+ return ensemble, dataset
def prepare_sample(sample, volatile=False, cuda_device=None):

diff:
@@ -36,7 +36,7 @@ def main():
options.add_checkpoint_args(parser)
options.add_model_args(parser)
- args = parser.parse_args()
+ args = utils.parse_args_and_arch(parser)
print(args)
if args.no_progress_bar:


message: Issue Checking size attribute of dst when dst is None

diff:
@@ -216,7 +216,7 @@ def batches_by_size(src, batch_size=None, max_tokens=None, dst=None, max_positio
# - 2 here stems from make_positions() where we offset positions
# by padding_value + 1
if src.sizes[idx] < 2 or \
- (dst is not None and dst.sizes[idx] < 2) or \
+ (False if dst is None else dst.sizes[idx] < 2) or \
sizes[idx] > max_positions - 2:
raise Exception("Unable to handle input id {} of "
"size {} / {}.".format(idx, src.sizes[idx], dst.sizes[idx]))

original_message: Issue #2, Checking size attribute of dst when dst is None


message: BPE transformation for IWSLT

diff:
@@ -108,7 +108,7 @@ $ cd ..
$ TEXT=data/iwslt14.tokenized.de-en
$ python preprocess.py --source-lang de --target-lang en \
--trainpref $TEXT/train --validpref $TEXT/valid --testpref $TEXT/test \
- --thresholdtgt 3 --thresholdsrc 3 --destdir data-bin/iwslt14.tokenized.de-en
+ --destdir data-bin/iwslt14.tokenized.de-en
```
This will write binarized data that can be used for model training to `data-bin/iwslt14.tokenized.de-en`.

diff:
echo 'Cloning Moses github repository (for tokenization scripts)...'
git clone https://github.com/moses-smt/mosesdecoder.git
+echo 'Cloning Subword NMT repository (for BPE pre-processing)...'
+git clone https://github.com/rsennrich/subword-nmt.git
+
SCRIPTS=mosesdecoder/scripts
TOKENIZER=$SCRIPTS/tokenizer/tokenizer.perl
LC=$SCRIPTS/tokenizer/lowercase.perl
CLEAN=$SCRIPTS/training/clean-corpus-n.perl
+BPEROOT=subword-nmt
+BPE_TOKENS=10000
URL="https://wit3.fbk.eu/archive/2014-01/texts/de/en/de-en.tgz"
GZ=de-en.tgz
@@ -81,13 +86,30 @@ done
echo "creating train, valid, test..."
for l in $src $tgt; do
- awk '{if (NR%23 == 0) print $0; }' $tmp/train.tags.de-en.$l > $prep/valid.$l
- awk '{if (NR%23 != 0) print $0; }' $tmp/train.tags.de-en.$l > $prep/train.$l
+ awk '{if (NR%23 == 0) print $0; }' $tmp/train.tags.de-en.$l > $tmp/valid.$l
+ awk '{if (NR%23 != 0) print $0; }' $tmp/train.tags.de-en.$l > $tmp/train.$l
cat $tmp/IWSLT14.TED.dev2010.de-en.$l \
$tmp/IWSLT14.TEDX.dev2012.de-en.$l \
$tmp/IWSLT14.TED.tst2010.de-en.$l \
$tmp/IWSLT14.TED.tst2011.de-en.$l \
$tmp/IWSLT14.TED.tst2012.de-en.$l \
- > $prep/test.$l
+ > $tmp/test.$l
+done
+
+TRAIN=$tmp/train.en-de
+BPE_CODE=$prep/code
+rm -f $TRAIN
+for l in $src $tgt; do
+ cat $tmp/train.$l >> $TRAIN
+done
+
+echo "learn_bpe.py on ${TRAIN}..."
+python $BPEROOT/learn_bpe.py -s $BPE_TOKENS < $TRAIN > $BPE_CODE
+
+for L in $src $tgt; do
+ for f in train.$L valid.$L test.$L; do
+ echo "apply_bpe.py to ${f}..."
+ python $BPEROOT/apply_bpe.py -c $BPE_CODE < $tmp/$f > $prep/$f
+ done
done

original_message: BPE transformation for IWSLT


message: Don't generate during training, add --quiet to generate.py

diff:
@@ -17,7 +17,7 @@ from fairseq.dictionary import Dictionary
from fairseq.indexed_dataset import IndexedDataset, IndexedInMemoryDataset
-def load_with_check(path, src=None, dst=None):
+def load_with_check(path, load_splits, src=None, dst=None):
"""Loads the train, valid, and test sets from the specified folder
and check that training files exist."""
@@ -43,12 +43,12 @@ def load_with_check(path, src=None, dst=None):
else:
raise ValueError('training file not found for {}-{}'.format(src, dst))
- dataset = load(path, src, dst)
+ dataset = load(path, load_splits, src, dst)
return dataset
-def load(path, src, dst):
- """Loads the train, valid, and test sets from the specified folder."""
+def load(path, load_splits, src, dst):
+ """Loads specified data splits (e.g. test, train or valid) from the path."""
langcode = '{}-{}'.format(src, dst)
@@ -59,7 +59,7 @@ def load(path, src, dst):
dst_dict = Dictionary.load(fmt_path('dict.{}.txt', dst))
dataset = LanguageDatasets(src, dst, src_dict, dst_dict)
- for split in ['train', 'valid', 'test']:
+ for split in load_splits:
for k in itertools.count():
prefix = "{}{}".format(split, k if k > 0 else '')
src_path = fmt_path('{}.{}.{}', prefix, langcode, src)

diff:
@@ -104,6 +104,8 @@ def add_generation_args(parser):
help='length penalty: <1.0 favors shorter, >1.0 favors longer sentences')
group.add_argument('--unk-replace-dict', default='', type=str,
help='performs unk word replacement')
+ group.add_argument('--quiet', action='store_true',
+ help='Only print final scores')
return group

diff:
@@ -94,7 +94,7 @@ def load_checkpoint(filename, model, optimizer, lr_scheduler, cuda_device=None):
return epoch, batch_offset
-def load_ensemble_for_inference(filenames, data_path):
+def load_ensemble_for_inference(filenames, data_path, split):
# load model architectures and weights
states = []
for filename in filenames:
@@ -106,7 +106,7 @@ def load_ensemble_for_inference(filenames, data_path):
# load dataset
args = states[0]['args']
- dataset = data.load(data_path, args.source_lang, args.target_lang)
+ dataset = data.load(data_path, [split], args.source_lang, args.target_lang)
# build models
ensemble = []

diff:
@@ -38,7 +38,7 @@ def main():
# Load model and dataset
print('| loading model(s) from {}'.format(', '.join(args.path)))
- models, dataset = utils.load_ensemble_for_inference(args.path, args.data)
+ models, dataset = utils.load_ensemble_for_inference(args.path, args.data, args.gen_subset)
print('| [{}] dictionary: {} types'.format(dataset.src, len(dataset.src_dict)))
print('| [{}] dictionary: {} types'.format(dataset.dst, len(dataset.dst_dict)))
@@ -81,6 +81,8 @@ def main():
bpe_symbol = '@@ ' if args.remove_bpe else None
def display_hypotheses(id, src, orig, ref, hypos):
+ if args.quiet:
+ return
id_str = '' if id is None else '-{}'.format(id)
src_str = to_sentence(dataset.src_dict, src, bpe_symbol)
print('S{}\t{}'.format(id_str, src_str))

diff:
@@ -29,9 +29,6 @@ def main():
dataset_args.add_argument('--valid-subset', default='valid', metavar='SPLIT',
help='comma separated list ofdata subsets '
' to use for validation (train, valid, valid1,test, test1)')
- dataset_args.add_argument('--test-subset', default='test', metavar='SPLIT',
- help='comma separated list ofdata subset '
- 'to use for testing (train, valid, test)')
options.add_optimization_args(parser)
options.add_checkpoint_args(parser)
options.add_model_args(parser)
@@ -48,7 +45,7 @@ def main():
torch.manual_seed(args.seed)
# Load dataset
- dataset = data.load_with_check(args.data, args.source_lang, args.target_lang)
+ dataset = data.load_with_check(args.data, ['train', 'valid'], args.source_lang, args.target_lang)
if args.source_lang is None or args.target_lang is None:
# record inferred languages in args, so that it's saved in checkpoints
args.source_lang, args.target_lang = dataset.src, dataset.dst
@@ -100,13 +97,6 @@ def main():
train_meter.stop()
print('| done training in {:.1f} seconds'.format(train_meter.sum))
- # Generate on test set and compute BLEU score
- for beam in [1, 5, 10, 20]:
- for subset in args.test_subset.split(','):
- scorer = score_test(args, trainer.get_model(), dataset, subset, beam,
- cuda_device=(0 if num_gpus > 0 else None))
- print('| Test on {} with beam={}: {}'.format(subset, beam, scorer.result_string()))
-
# Stop multiprocessing
trainer.stop()
@@ -192,19 +182,5 @@ def validate(args, epoch, trainer, criterion, dataset, subset, ngpus):
return val_loss
-def score_test(args, model, dataset, subset, beam, cuda_device):
- """Evaluate the model on the test set and return the BLEU scorer."""
-
- translator = SequenceGenerator([model], dataset.dst_dict, beam_size=beam)
- if torch.cuda.is_available():
- translator.cuda()
-
- scorer = bleu.Scorer(dataset.dst_dict.pad(), dataset.dst_dict.eos(), dataset.dst_dict.unk())
- itr = dataset.dataloader(subset, batch_size=4, max_positions=args.max_positions)
- for _, _, ref, hypos in translator.generate_batched_itr(itr, cuda_device=cuda_device):
- scorer.add(ref.int().cpu(), hypos[0]['tokens'].int().cpu())
- return scorer
-
-
if __name__ == '__main__':
main()


message: Ignore invalid sentences in test and valid

diff:
@@ -91,7 +91,8 @@ class LanguageDatasets(object):
def dataloader(self, split, batch_size=1, num_workers=0,
max_tokens=None, seed=None, epoch=1,
- sample_without_replacement=0, max_positions=1024):
+ sample_without_replacement=0, max_positions=1024,
+ skip_invalid_size_inputs_valid_test=False):
dataset = self.splits[split]
if split.startswith('train'):
with numpy_seed(seed):
@@ -102,9 +103,11 @@ class LanguageDatasets(object):
max_positions=max_positions)
elif split.startswith('valid'):
batch_sampler = list(batches_by_size(dataset.src, batch_size, max_tokens, dst=dataset.dst,
- max_positions=max_positions))
+ max_positions=max_positions,
+ ignore_invalid_inputs=skip_invalid_size_inputs_valid_test))
else:
- batch_sampler = list(batches_by_size(dataset.src, batch_size, max_tokens, max_positions=max_positions))
+ batch_sampler = list(batches_by_size(dataset.src, batch_size, max_tokens, max_positions=max_positions,
+ ignore_invalid_inputs=skip_invalid_size_inputs_valid_test))
return torch.utils.data.DataLoader(
dataset,
@@ -207,7 +210,8 @@ class LanguagePairDataset(object):
return res
-def batches_by_size(src, batch_size=None, max_tokens=None, dst=None, max_positions=1024):
+def batches_by_size(src, batch_size=None, max_tokens=None, dst=None,
+ max_positions=1024, ignore_invalid_inputs=False):
"""Returns batches of indices sorted by size. Sequences of different lengths
are not allowed in the same batch."""
assert isinstance(src, IndexedDataset)
@@ -233,14 +237,20 @@ def batches_by_size(src, batch_size=None, max_tokens=None, dst=None, max_positio
return False
cur_max_size = 0
+ ignored = []
for idx in indices:
# - 2 here stems from make_positions() where we offset positions
# by padding_value + 1
if src.sizes[idx] < 2 or \
(False if dst is None else dst.sizes[idx] < 2) or \
sizes[idx] > max_positions - 2:
+ if ignore_invalid_inputs:
+ ignored.append(idx)
+ continue
+
raise Exception("Unable to handle input id {} of "
- "size {} / {}.".format(idx, src.sizes[idx], dst.sizes[idx]))
+ "size {} / {}.".format(idx, src.sizes[idx],
+ "none" if dst is None else dst.sizes[idx]))
if yield_batch(idx, cur_max_size * (len(batch) + 1)):
yield batch
@@ -249,6 +259,10 @@ def batches_by_size(src, batch_size=None, max_tokens=None, dst=None, max_positio
batch.append(idx)
cur_max_size = max(cur_max_size, sizes[idx])
+ if len(ignored) > 0:
+ print("Warning! {} samples are either too short or too long "
+ "and will be ignored, sample ids={}".format(len(ignored), ignored))
+
if len(batch) > 0:
yield batch

diff:
@@ -243,6 +243,10 @@ class Decoder(nn.Module):
context += conv.kernel_size[0] - 1
return context
+ def max_positions(self):
+ """Returns maximum size of positions embeddings supported by this decoder"""
+ return self.embed_positions.num_embeddings
+
def incremental_inference(self, beam_size=None):
"""Context manager for incremental inference.

diff:
@@ -34,6 +34,8 @@ def add_dataset_args(parser):
help='number of data loading workers (default: 1)')
group.add_argument('--max-positions', default=1024, type=int, metavar='N',
help='max number of tokens in the sequence')
+ group.add_argument('--skip-invalid-size-inputs-valid-test', action='store_true',
+ help='Ignore too long or too short lines in valid and test set')
return group

diff:
@@ -35,8 +35,8 @@ class SequenceGenerator(object):
self.vocab_size = len(dst_dict)
self.beam_size = beam_size
self.minlen = minlen
- self.maxlen = maxlen
- self.positions = torch.LongTensor(range(self.pad + 1, self.pad + maxlen + 2))
+ self.maxlen = min(maxlen, *(m.decoder.max_positions() - self.pad - 2 for m in self.models))
+ self.positions = torch.LongTensor(range(self.pad + 1, self.pad + self.maxlen + 2))
self.decoder_context = models[0].decoder.context_size()
self.stop_early = stop_early
self.normalize_scores = normalize_scores

diff:
@@ -45,6 +45,10 @@ def main():
if not args.interactive:
print('| {} {} {} examples'.format(args.data, args.gen_subset, len(dataset.splits[args.gen_subset])))
+ # Max positions is the model property but it is needed in data reader to be able to
+ # ignore too long sentences
+ args.max_positions = min(args.max_positions, *(m.decoder.max_positions() for m in models))
+
# Optimize model for generation
for model in models:
model.make_generation_fast_(not args.no_beamable_mm)
@@ -122,7 +126,9 @@ def main():
# Generate and compute BLEU score
scorer = bleu.Scorer(dataset.dst_dict.pad(), dataset.dst_dict.eos(), dataset.dst_dict.unk())
- itr = dataset.dataloader(args.gen_subset, batch_size=args.batch_size, max_positions=args.max_positions)
+ itr = dataset.dataloader(args.gen_subset, batch_size=args.batch_size,
+ max_positions=args.max_positions,
+ skip_invalid_size_inputs_valid_test=args.skip_invalid_size_inputs_valid_test)
num_sentences = 0
with progress_bar(itr, smoothing=0, leave=False) as t:
wps_meter = TimeMeter()

diff:
@@ -107,7 +107,8 @@ def train(args, epoch, batch_offset, trainer, criterion, dataset, num_gpus):
itr = dataset.dataloader(args.train_subset, num_workers=args.workers,
max_tokens=args.max_tokens, seed=args.seed, epoch=epoch,
max_positions=args.max_positions,
- sample_without_replacement=args.sample_without_replacement)
+ sample_without_replacement=args.sample_without_replacement,
+ skip_invalid_size_inputs_valid_test=args.skip_invalid_size_inputs_valid_test)
loss_meter = AverageMeter()
bsz_meter = AverageMeter() # sentences per batch
wpb_meter = AverageMeter() # words per batch
@@ -163,7 +164,8 @@ def validate(args, epoch, trainer, criterion, dataset, subset, ngpus):
itr = dataset.dataloader(subset, batch_size=None,
max_tokens=args.max_tokens,
- max_positions=args.max_positions)
+ max_positions=args.max_positions,
+ skip_invalid_size_inputs_valid_test=args.skip_invalid_size_inputs_valid_test)
loss_meter = AverageMeter()
desc = '| epoch {:03d} | valid on \'{}\' subset'.format(epoch, subset)


message: Update En2Fr model

diff:
@@ -149,14 +149,14 @@ $ python generate.py data-bin/wmt14.en-fr.newstest2014 \
--path data-bin/wmt14.en-fr.fconv-py/model.pt \
--beam 5 --batch-size 128 --remove-bpe | tee /tmp/gen.out
...
-| Translated 3003 sentences (95451 tokens) in 81.3s (1174.33 tokens/s)
-| Generate test with beam=5: BLEU4 = 40.23, 67.5/46.4/33.8/25.0 (BP=0.997, ratio=1.003, syslen=80963, reflen=81194)
+| Translated 3003 sentences (95451 tokens) in 94.5s (1009.60 tokens/s)
+| Generate test with beam=5: BLEU4 = 40.70, 67.7/46.8/34.2/25.4 (BP=1.000, ratio=1.000, syslen=81190, reflen=81194)
# Scoring with score.py:
$ grep ^H /tmp/gen.out | cut -f3- > /tmp/gen.out.sys
$ grep ^T /tmp/gen.out | cut -f2- > /tmp/gen.out.ref
$ python score.py --sys /tmp/gen.out.sys --ref /tmp/gen.out.ref
-BLEU4 = 40.23, 67.5/46.4/33.8/25.0 (BP=0.997, ratio=1.003, syslen=80963, reflen=81194)
+BLEU4 = 40.70, 67.7/46.8/34.2/25.4 (BP=1.000, ratio=1.000, syslen=81190, reflen=81194)
```
# Join the fairseq community

original_message: Update En2Fr model


message: Add attention matrix to output of SequenceGenerator

diff:
@@ -108,8 +108,8 @@ class SequenceGenerator(object):
tokens = src_tokens.data.new(bsz * beam_size, maxlen + 2).fill_(self.pad)
tokens_buf = tokens.clone()
tokens[:, 0] = self.eos
- align = src_tokens.data.new(bsz * beam_size, maxlen + 2).fill_(-1)
- align_buf = align.clone()
+ attn = scores.new(bsz * beam_size, src_tokens.size(1), maxlen + 2)
+ attn_buf = attn.clone()
# list of completed sentences
finalized = [[] for i in range(bsz)]
@@ -177,10 +177,12 @@ class SequenceGenerator(object):
def get_hypo():
hypo = tokens[idx, 1:step+2].clone() # skip the first index, which is EOS
hypo[step] = self.eos
- alignment = align[idx, 1:step+2].clone()
+ attention = attn[idx, :, 1:step+2].clone()
+ _, alignment = attention.max(dim=0)
return {
'tokens': hypo,
'score': score,
+ 'attention': attention,
'alignment': alignment,
}
@@ -224,9 +226,8 @@ class SequenceGenerator(object):
probs.add_(scores.view(-1, 1))
probs[:, self.pad] = -math.inf # never select pad
- # record alignment to source tokens, based on attention
- _ignore_scores = buffer('_ignore_scores', type_of=scores)
- avg_attn_scores.topk(1, out=(_ignore_scores, align[:, step+1].unsqueeze(1)))
+ # Record attention scores
+ attn[:, :, step+1].copy_(avg_attn_scores)
# take the best 2 x beam_size predictions. We'll choose the first
# beam_size of these which don't predict eos to continue with.
@@ -290,17 +291,17 @@ class SequenceGenerator(object):
cand_indices.gather(1, active_hypos,
out=tokens_buf.view(bsz, beam_size, -1)[:, :, step+1])
- # copy attention/alignment for active hypotheses
- torch.index_select(align[:, :step+2], dim=0, index=active_bbsz_idx,
- out=align_buf[:, :step+2])
+ # copy attention for active hypotheses
+ torch.index_select(attn[:, :, :step+2], dim=0, index=active_bbsz_idx,
+ out=attn_buf[:, :, :step+2])
# swap buffers
old_tokens = tokens
tokens = tokens_buf
tokens_buf = old_tokens
- old_align = align
- align = align_buf
- align_buf = old_align
+ old_attn = attn
+ attn = attn_buf
+ attn_buf = old_attn
# reorder incremental state in decoder
reorder_state = active_bbsz_idx


message: Refactor code in Tokenizer

diff:
# can be found in the PATENTS file in the same directory.
#
+from collections import Counter
import re
+
import torch
from fairseq import dictionary
@@ -32,46 +34,41 @@ class Tokenizer:
@staticmethod
def add_file_to_dictionary(filename, dict, tokenize):
with open(filename, 'r') as f:
- for line in f.readlines():
+ for line in f:
for word in tokenize(line):
dict.add_symbol(word)
dict.add_symbol(dict.eos_word)
@staticmethod
def binarize(filename, dict, consumer, tokenize=tokenize_line):
- nseq, ntok, nunk = 0, 0, 0
- replaced = {}
- with open(filename, 'r') as f:
- for line in f.readlines():
- words = tokenize(line)
- nwords = len(words)
- ids = torch.IntTensor(nwords + 1)
- nseq = nseq + 1
- for i in range(0, len(words)):
- word = words[i]
- idx = dict.index(word)
+ nseq, ntok = 0, 0
+ replaced = Counter()
+
+ def replaced_consumer(word, idx):
if idx == dict.unk_index and word != dict.unk_word:
- nunk = nunk + 1
- if word in replaced:
- replaced[word] = replaced[word] + 1
- else:
- replaced[word] = 1
- ids[i] = idx
+ replaced.update([word])
+
+ with open(filename, 'r') as f:
+ for line in f:
+ ids = Tokenizer.tokenize(line, dict, tokenize, add_if_not_exist=False, consumer=replaced_consumer)
+ nseq += 1
- ids[nwords] = dict.eos_index
consumer(ids)
- ntok = ntok + len(ids)
- return {'nseq': nseq, 'nunk': nunk, 'ntok': ntok, 'replaced': len(replaced)}
+ ntok += len(ids)
+ return {'nseq': nseq, 'nunk': sum(replaced.values()), 'ntok': ntok, 'replaced': len(replaced)}
@staticmethod
- def tokenize(line, dict, tokenize=tokenize_line, add_if_not_exist=True):
+ def tokenize(line, dict, tokenize=tokenize_line, add_if_not_exist=True, consumer=None):
words = tokenize(line)
nwords = len(words)
ids = torch.IntTensor(nwords + 1)
- for i in range(0, len(words)):
+ for i, word in enumerate(words):
if add_if_not_exist:
- ids[i] = dict.add_symbol(words[i])
+ idx = dict.add_symbol(word)
else:
- ids[i] = dict.index(words[i])
+ idx = dict.index(word)
+ if consumer is not None:
+ consumer(word, idx)
+ ids[i] = idx
ids[nwords] = dict.eos_index
return ids

original_message: Refactor code in Tokenizer


message: Prevent math overflow when loss is too high

diff:
@@ -110,6 +110,13 @@ def main():
trainer.stop()
+def get_perplexity(loss):
+ try:
+ return math.pow(2, loss)
+ except OverflowError:
+ return float('inf')
+
+
def train(args, epoch, batch_offset, trainer, dataset, num_gpus):
"""Train the model for one epoch."""
@@ -162,7 +169,7 @@ def train(args, epoch, batch_offset, trainer, dataset, num_gpus):
save_checkpoint(trainer, args, epoch, i + 1)
fmt = desc + ' | train loss {:2.2f} | train ppl {:3.2f}'.format(
- loss_meter.avg, math.pow(2, loss_meter.avg))
+ loss_meter.avg, get_perplexity(loss_meter.avg))
fmt += ' | s/checkpoint {:7d} | words/s {:6d} | words/batch {:6d}'.format(
round(wps_meter.elapsed_time), round(wps_meter.avg), round(wpb_meter.avg))
fmt += ' | bsz {:5d} | lr {:0.6f} | clip {:3.0f}%'.format(
@@ -227,7 +234,7 @@ def validate(args, epoch, trainer, dataset, subset, ngpus):
val_loss = loss_meter.avg
fmt = desc + ' | valid loss {:2.2f} | valid ppl {:3.2f}'.format(
- val_loss, math.pow(2, val_loss))
+ val_loss, get_perplexity(val_loss))
fmt += ''.join(
' | {} {:.4f}'.format(k, meter.avg)
for k, meter in extra_meters.items()


message: Force encoding for dictionary files ( )

diff:
@@ -109,8 +109,12 @@ class Dictionary(object):
"""
if isinstance(f, str):
- with open(f, 'r') as fd:
+ try:
+ with open(f, 'r', encoding='utf-8') as fd:
return Dictionary.load(fd)
+ except:
+ raise Exception("Incorrect encoding detected in {}, please "
+ "rebuild the dataset".format(f))
d = Dictionary()
for line in f.readlines():
@@ -125,7 +129,7 @@ class Dictionary(object):
def save(self, f, threshold=3, nwords=-1):
"""Stores dictionary into a text file"""
if isinstance(f, str):
- with open(f, 'w') as fd:
+ with open(f, 'w', encoding='utf-8') as fd:
return self.save(fd, threshold, nwords)
cnt = 0
for i, t in enumerate(zip(self.symbols, self.count)):

original_message: Force UTF-8 encoding for dictionary files ( #41 )


message: Fix call to non-existing to_string method

diff:
@@ -45,7 +45,7 @@ class Dictionary(object):
Can optionally remove BPE symbols or escape <unk> words.
"""
if torch.is_tensor(tensor) and tensor.dim() == 2:
- return '\n'.join(self.to_string(t) for t in tensor)
+ return '\n'.join(self.string(t) for t in tensor)
def token_string(i):
if i == self.unk():


message: Update README with interactive.py and fix it

diff:
@@ -97,8 +97,8 @@ def main():
remove_bpe=args.remove_bpe)
if not args.quiet:
- print('A-{}\t{}'.format(sample_id, ' '.join(map(str, alignment))))
print('H-{}\t{}\t{}'.format(sample_id, hypo['score'], hypo_str))
+ print('A-{}\t{}'.format(sample_id, ' '.join(map(str, alignment))))
# Score only the top hypothesis
if i == 0:

diff:
@@ -51,7 +51,7 @@ def main():
# (None if no unknown word replacement, empty if no path to align dictionary)
align_dict = utils.load_align_dict(args.replace_unk)
- print('Type the input sentence and press return:')
+ print('| Type the input sentence and press return:')
for src_str in sys.stdin:
src_str = src_str.strip()
src_tokens = tokenizer.Tokenizer.tokenize(src_str, src_dict, add_if_not_exist=False).long()
@@ -70,8 +70,8 @@ def main():
align_dict=align_dict,
dst_dict=dst_dict,
remove_bpe=args.remove_bpe)
- print('A\t{}'.format(' '.join(map(str, alignment))))
print('H\t{}\t{}'.format(hypo['score'], hypo_str))
+ print('A\t{}'.format(' '.join(map(str, alignment))))
if __name__ == '__main__':


message: Prepare scripts for WMT14

diff:
@@ -13,12 +13,11 @@ import torch
from fairseq import dictionary
+SPACE_NORMALIZER = re.compile("\s+")
def tokenize_line(line):
- line = re.sub(r"\t", "", line)
- line = re.sub(r"^\s+", "", line)
- line = re.sub(r"\s+$", "", line)
- line = re.sub(r"\s+", " ", line)
+ line = SPACE_NORMALIZER.sub(" ", line)
+ line = line.strip()
return line.split()

original_message: Prepare scripts for WMT14


message: Share input/output embed

diff:
@@ -135,7 +135,7 @@ class FConvDecoder(FairseqIncrementalDecoder):
"""Convolutional decoder"""
def __init__(self, dictionary, embed_dim=512, out_embed_dim=256,
max_positions=1024, convolutions=((512, 3),) * 20,
- attention=True, dropout=0.1):
+ attention=True, dropout=0.1, share_embed=False):
super().__init__(dictionary)
self.register_buffer('version', torch.Tensor([2]))
self.dropout = dropout
@@ -169,6 +169,13 @@ class FConvDecoder(FairseqIncrementalDecoder):
if attention[i] else None)
in_channels = out_channels
self.fc2 = Linear(in_channels, out_embed_dim)
+ if share_embed:
+ assert out_embed_dim == embed_dim, \
+ "Shared embed weights implies same dimensions " \
+ " out_embed_dim={} vs embed_dim={}".format(out_embed_dim, embed_dim)
+ self.fc3 = nn.Linear(out_embed_dim, num_embeddings)
+ self.fc3.weight = self.embed_tokens.weight
+ else:
self.fc3 = Linear(out_embed_dim, num_embeddings, dropout=dropout)
def forward(self, input_tokens, encoder_out):
@@ -372,6 +379,7 @@ def parse_arch(args):
args.decoder_layers = getattr(args, 'decoder_layers', '[(512, 3)] * 20')
args.decoder_out_embed_dim = getattr(args, 'decoder_out_embed_dim', 256)
args.decoder_attention = getattr(args, 'decoder_attention', 'True')
+ args.share_input_output_embed = getattr(args, 'share_input_output_embed', False)
return args
@@ -391,5 +399,6 @@ def build_model(args, src_dict, dst_dict):
attention=eval(args.decoder_attention),
dropout=args.dropout,
max_positions=args.max_target_positions,
+ share_embed=args.share_input_output_embed
)
return FConvModel(encoder, decoder)

diff:
@@ -173,4 +173,8 @@ def add_model_args(parser):
help='dropout probability')
group.add_argument('--label-smoothing', default=0, type=float, metavar='D',
help='epsilon for label smoothing, 0 means no label smoothing')
+
+ group.add_argument('--share-input-output-embed', action='store_true',
+ help="Share input and output embeddings, "
+ "requires --decoder-out-embed-dim and --decoder-embed-dim be equal ")
return group

diff:
@@ -173,6 +173,8 @@ def _upgrade_args(args):
if not hasattr(args, 'max_source_positions'):
args.max_source_positions = args.max_positions
args.max_target_positions = args.max_positions
+ if not hasattr(args, 'share_input_output_embed'):
+ args.share_input_output_embed = False
return args

original_message: Share input/output embed


message: Report log likelihood for label smoothing

diff:
@@ -65,9 +65,11 @@ class LabelSmoothedCrossEntropyCriterion(FairseqCriterion):
lprobs = model.get_normalized_probs(net_output, log_probs=True)
target = sample['target'].view(-1)
loss = LabelSmoothedNLLLoss.apply(lprobs, target, self.eps, self.padding_idx, self.weights, reduce)
+ nll_loss = F.nll_loss(lprobs, target, size_average=False, ignore_index=self.padding_idx, reduce=reduce)
sample_size = sample['target'].size(0) if self.args.sentence_avg else sample['ntokens']
logging_output = {
'loss': loss.data[0] if reduce else loss.data,
+ 'nll_loss': nll_loss.data[0] if reduce else loss.data,
'sample_size': sample_size,
}
return loss, sample_size, logging_output
@@ -78,4 +80,5 @@ class LabelSmoothedCrossEntropyCriterion(FairseqCriterion):
sample_size = sum(log.get('sample_size', 0) for log in logging_outputs)
return {
'loss': sum(log.get('loss', 0) for log in logging_outputs) / sample_size / math.log(2),
+ 'nll_loss': sum(log.get('nll_loss', 0) for log in logging_outputs) / sample_size / math.log(2),
}

diff:
@@ -150,6 +150,7 @@ def train(args, epoch, batch_offset, trainer, dataset, max_positions):
sample_without_replacement=args.sample_without_replacement,
sort_by_source_size=(epoch <= args.curriculum))
loss_meter = AverageMeter()
+ nll_loss_meter = AverageMeter()
bsz_meter = AverageMeter() # sentences per batch
wpb_meter = AverageMeter() # words per batch
wps_meter = TimeMeter() # words per second
@@ -164,6 +165,11 @@ def train(args, epoch, batch_offset, trainer, dataset, max_positions):
del loss_dict['loss'] # don't include in extra_meters or extra_postfix
ntokens = sum(s['ntokens'] for s in sample)
+
+ if 'nll_loss' in loss_dict:
+ nll_loss = loss_dict['nll_loss']
+ nll_loss_meter.update(nll_loss, ntokens)
+
nsentences = sum(s['net_input']['src_tokens'].size(0) for s in sample)
loss_meter.update(loss, nsentences if args.sentence_avg else ntokens)
bsz_meter.update(nsentences)
@@ -193,7 +199,9 @@ def train(args, epoch, batch_offset, trainer, dataset, max_positions):
t.print(collections.OrderedDict([
('train loss', round(loss_meter.avg, 2)),
- ('train ppl', get_perplexity(loss_meter.avg)),
+ ('train ppl', get_perplexity(nll_loss_meter.avg
+ if nll_loss_meter.count > 0
+ else loss_meter.avg)),
('s/checkpoint', round(wps_meter.elapsed_time)),
('words/s', round(wps_meter.avg)),
('words/batch', round(wpb_meter.avg)),
@@ -242,16 +250,21 @@ def validate(args, epoch, trainer, dataset, max_positions, subset):
descending=True, # largest batch first to warm the caching allocator
)
loss_meter = AverageMeter()
+ nll_loss_meter = AverageMeter()
extra_meters = collections.defaultdict(lambda: AverageMeter())
prefix = 'valid on \'{}\' subset'.format(subset)
with utils.build_progress_bar(args, itr, epoch, prefix) as t:
for _, sample in data.skip_group_enumerator(t, args.num_gpus):
loss_dict = trainer.valid_step(sample)
+ ntokens = sum(s['ntokens'] for s in sample)
loss = loss_dict['loss']
del loss_dict['loss'] # don't include in extra_meters or extra_postfix
- ntokens = sum(s['ntokens'] for s in sample)
+ if 'nll_loss' in loss_dict:
+ nll_loss = loss_dict['nll_loss']
+ nll_loss_meter.update(nll_loss, ntokens)
+
loss_meter.update(loss, ntokens)
extra_postfix = []
@@ -265,7 +278,9 @@ def validate(args, epoch, trainer, dataset, max_positions, subset):
t.print(collections.OrderedDict([
('valid loss', round(loss_meter.avg, 2)),
- ('valid ppl', get_perplexity(loss_meter.avg)),
+ ('valid ppl', get_perplexity(nll_loss_meter.avg
+ if nll_loss_meter.count > 0
+ else loss_meter.avg)),
] + [
(k, meter.avg)
for k, meter in extra_meters.items()


message: Switch to news-commentary-v12

diff:
@@ -18,17 +18,17 @@ BPE_TOKENS=40000
URLS=(
"http://statmt.org/wmt13/training-parallel-europarl-v7.tgz"
"http://statmt.org/wmt13/training-parallel-commoncrawl.tgz"
- "http://statmt.org/wmt14/training-parallel-nc-v9.tgz"
+ "http://data.statmt.org/wmt17/translation-task/training-parallel-nc-v12.tgz"
)
FILES=(
"training-parallel-europarl-v7.tgz"
"training-parallel-commoncrawl.tgz"
- "training-parallel-nc-v9.tgz"
+ "training-parallel-nc-v12.tgz"
)
CORPORA=(
"training/europarl-v7.de-en"
"commoncrawl.de-en"
- "training/news-commentary-v9.de-en"
+ "training/news-commentary-v12.de-en"
)

original_message: Switch to news-commentary-v12


message: Adding README and more parameters to En2De script

diff:
+Sample data processing scripts for FAIR Sequence-to-Sequence Toolkit
+
+These scripts provide an example of pre-processing data for NMT task.
+
+# prepare-iwslt14.sh
+
+Provides an example of pre-processing for IWSLT'14 German to English translation task: ["Report on the 11th IWSLT evaluation campaign" by Cettolo et al.](http://workshop2014.iwslt.org/downloads/proceeding.pdf)
+
+Example usage:
+```
+$ cd data/
+$ bash prepare-iwslt14.sh
+$ cd ..
+
+# Binarize the dataset:
+$ TEXT=data/iwslt14.tokenized.de-en
+$ python preprocess.py --source-lang de --target-lang en \
+ --trainpref $TEXT/train --validpref $TEXT/valid --testpref $TEXT/test \
+ --destdir data-bin/iwslt14.tokenized.de-en
+
+# Train the model:
+$ mkdir -p checkpoints/fconv
+$ CUDA_VISIBLE_DEVICES=0 python train.py data-bin/iwslt14.tokenized.de-en \
+ --lr 0.25 --clip-norm 0.1 --dropout 0.2 --max-tokens 4000 \
+ --label-smoothing 0.1 --force-anneal 200 \
+ --arch fconv_iwslt_de_en --save-dir checkpoints/fconv
+
+# Generate:
+$ python generate.py data-bin/iwslt14.tokenized.de-en \
+ --path checkpoints/fconv/checkpoint_best.pt \
+ --batch-size 128 --beam 5 --remove-bpe
+
+```
+
+
+# prepare-wmt14en2de.sh
+
+Provides an example of pre-processing for WMT'14 English to German translation task. By default it will produce a dataset that was modeled after ["Attention Is All You Need" by Vaswani et al.](https://arxiv.org/abs/1706.03762) that includes news-commentary-v12 data.
+
+To use only data awailable in WMT'14 or to replicate results obtained in the original paper ["Convolutional Sequence to Sequence Learning" by Gehring et al.](https://arxiv.org/abs/1705.03122) run it with --icml17 instead:
+
+```
+$ bash prepare-wmt14en2de.sh --icml17
+```
+
+Example usage:
+
+```
+$ cd data/
+$ bash prepare-wmt14en2de.sh
+$ cd ..
+
+# Binarize the dataset:
+$ TEXT=data/wmt14_en_de
+$ python preprocess.py --source-lang en --target-lang de \
+ --trainpref $TEXT/train --validpref $TEXT/valid --testpref $TEXT/test \
+ --destdir data-bin/wmt14_en_de --thresholdtgt 0 --thresholdsrc 0
+
+# Train the model:
+# If it runs out of memory, try to set --max-tokens 1500 instead
+$ mkdir -p checkpoints/fconv_wmt_en_de
+$ python train.py data-bin/wmt14_en_de \
+ --lr 0.5 --clip-norm 0.1 --dropout 0.2 --max-tokens 4000 \
+ --label-smoothing 0.1 --force-anneal 50 \
+ --arch fconv_wmt_en_de --save-dir checkpoints/fconv_wmt_en_de
+
+# Generate:
+$ python generate.py data-bin/wmt14_en_de \
+ --path checkpoints/fconv_wmt_en_de/checkpoint_best.pt --beam 5 --remove-bpe
+
+```
+
+# prepare-wmt14en2fr.sh
+
+Provides an example of pre-processing for WMT'14 English to French translation task.
+
+Example usage:
+
+```
+$ cd data/
+$ bash prepare-wmt14en2fr.sh
+$ cd ..
+
+# Binarize the dataset:
+$ TEXT=data/wmt14_en_fr
+$ python preprocess.py --source-lang en --target-lang fr \
+ --trainpref $TEXT/train --validpref $TEXT/valid --testpref $TEXT/test \
+ --destdir data-bin/wmt14_en_fr --thresholdtgt 0 --thresholdsrc 0
+
+# Train the model:
+# If it runs out of memory, try to set --max-tokens 1000 instead
+$ mkdir -p checkpoints/fconv_wmt_en_fr
+$ python train.py data-bin/wmt14_en_fr \
+ --lr 0.5 --clip-norm 0.1 --dropout 0.1 --max-tokens 3000 \
+ --label-smoothing 0.1 --force-anneal 50 \
+ --arch fconv_wmt_en_fr --save-dir checkpoints/fconv_wmt_en_fr
+
+# Generate:
+$ python generate.py data-bin/fconv_wmt_en_fr \
+ --path checkpoints/fconv_wmt_en_fr/checkpoint_best.pt --beam 5 --remove-bpe
+
+```
+

diff:
@@ -19,11 +19,13 @@ URLS=(
"http://statmt.org/wmt13/training-parallel-europarl-v7.tgz"
"http://statmt.org/wmt13/training-parallel-commoncrawl.tgz"
"http://data.statmt.org/wmt17/translation-task/training-parallel-nc-v12.tgz"
+ "http://statmt.org/wmt14/test-full.tgz"
)
FILES=(
"training-parallel-europarl-v7.tgz"
"training-parallel-commoncrawl.tgz"
"training-parallel-nc-v12.tgz"
+ "test-full.tgz"
)
CORPORA=(
"training/europarl-v7.de-en"
@@ -31,6 +33,13 @@ CORPORA=(
"training/news-commentary-v12.de-en"
)
+# This will make the dataset compatible to the one used in "Convolutional Sequence to Sequence Learning"
+# https://arxiv.org/abs/1705.03122
+if [ "$1" == "--icml17" ]; then
+ URLS[2]="http://statmt.org/wmt14/training-parallel-nc-v9.tgz"
+ FILES[2]="training-parallel-nc-v9.tgz"
+ CORPORA[2]="training/news-commentary-v9.de-en"
+fi
if [ ! -d "$SCRIPTS" ]; then
echo "Please set SCRIPTS variable correctly to point to Moses scripts."


message: Update README with new models

diff:
@@ -49,7 +49,7 @@ The following command-line tools are available:
## Evaluating Pre-trained Models
First, download a pre-trained model along with its vocabularies:
```
-$ curl https://s3.amazonaws.com/fairseq-py/models/wmt14.en-fr.fconv-py.tar.bz2 | tar xvjf -
+$ curl https://s3.amazonaws.com/fairseq-py/models/wmt14.v2.en-fr.fconv-py.tar.bz2 | tar xvjf -
```
This model uses a [Byte Pair Encoding (BPE) vocabulary](https://arxiv.org/abs/1508.07909), so we'll have to apply the encoding to the source text before it can be translated.
@@ -135,30 +135,30 @@ BPE continuation markers can be removed with the `--remove-bpe` flag.
We provide the following pre-trained fully convolutional sequence-to-sequence models:
-* [wmt14.en-fr.fconv-py.tar.bz2](https://s3.amazonaws.com/fairseq-py/models/wmt14.en-fr.fconv-py.tar.bz2): Pre-trained model for [WMT14 English-French](http://statmt.org/wmt14/translation-task.html#Download) including vocabularies
-* [wmt14.en-de.fconv-py.tar.bz2](https://s3.amazonaws.com/fairseq-py/models/wmt14.en-de.fconv-py.tar.bz2): Pre-trained model for [WMT14 English-German](https://nlp.stanford.edu/projects/nmt) including vocabularies
+* [wmt14.en-fr.fconv-py.tar.bz2](https://s3.amazonaws.com/fairseq-py/models/wmt14.v2.en-fr.fconv-py.tar.bz2): Pre-trained model for [WMT14 English-French](http://statmt.org/wmt14/translation-task.html#Download) including vocabularies
+* [wmt14.en-de.fconv-py.tar.bz2](https://s3.amazonaws.com/fairseq-py/models/wmt14.v2.en-de.fconv-py.tar.bz2): Pre-trained model for [WMT14 English-German](https://nlp.stanford.edu/projects/nmt) including vocabularies
In addition, we provide pre-processed and binarized test sets for the models above:
-* [wmt14.en-fr.newstest2014.tar.bz2](https://s3.amazonaws.com/fairseq-py/data/wmt14.en-fr.newstest2014.tar.bz2): newstest2014 test set for WMT14 English-French
-* [wmt14.en-fr.ntst1213.tar.bz2](https://s3.amazonaws.com/fairseq-py/data/wmt14.en-fr.ntst1213.tar.bz2): newstest2012 and newstest2013 test sets for WMT14 English-French
-* [wmt14.en-de.newstest2014.tar.bz2](https://s3.amazonaws.com/fairseq-py/data/wmt14.en-de.newstest2014.tar.bz2): newstest2014 test set for WMT14 English-German
+* [wmt14.en-fr.newstest2014.tar.bz2](https://s3.amazonaws.com/fairseq-py/data/wmt14.v2.en-fr.newstest2014.tar.bz2): newstest2014 test set for WMT14 English-French
+* [wmt14.en-fr.ntst1213.tar.bz2](https://s3.amazonaws.com/fairseq-py/data/wmt14.v2.en-fr.ntst1213.tar.bz2): newstest2012 and newstest2013 test sets for WMT14 English-French
+* [wmt14.en-de.newstest2014.tar.bz2](https://s3.amazonaws.com/fairseq-py/data/wmt14.v2.en-de.newstest2014.tar.bz2): newstest2014 test set for WMT14 English-German
Generation with the binarized test sets can be run in batch mode as follows, e.g. for English-French on a GTX-1080ti:
```
-$ curl https://s3.amazonaws.com/fairseq-py/models/wmt14.en-fr.fconv-py.tar.bz2 | tar xvjf - -C data-bin
-$ curl https://s3.amazonaws.com/fairseq-py/data/wmt14.en-fr.newstest2014.tar.bz2 | tar xvjf - -C data-bin
+$ curl https://s3.amazonaws.com/fairseq-py/models/wmt14.v2.en-fr.fconv-py.tar.bz2 | tar xvjf - -C data-bin
+$ curl https://s3.amazonaws.com/fairseq-py/data/wmt14.v2.en-fr.newstest2014.tar.bz2 | tar xvjf - -C data-bin
$ python generate.py data-bin/wmt14.en-fr.newstest2014 \
--path data-bin/wmt14.en-fr.fconv-py/model.pt \
--beam 5 --batch-size 128 --remove-bpe | tee /tmp/gen.out
...
-| Translated 3003 sentences (95451 tokens) in 93.8s (1018.09 tokens/s)
-| Generate test with beam=5: BLEU4 = 40.67, 67.7/46.7/34.2/25.3 (BP=1.000, ratio=0.998, syslen=81377, reflen=81194)
+| Translated 3003 sentences (96311 tokens) in 166.0s (580.04 tokens/s)
+| Generate test with beam=5: BLEU4 = 40.83, 67.5/46.9/34.4/25.5 (BP=1.000, ratio=1.006, syslen=83262, reflen=82787)
# Scoring with score.py:
$ grep ^H /tmp/gen.out | cut -f3- > /tmp/gen.out.sys
$ grep ^T /tmp/gen.out | cut -f2- > /tmp/gen.out.ref
$ python score.py --sys /tmp/gen.out.sys --ref /tmp/gen.out.ref
-BLEU4 = 40.67, 67.7/46.7/34.2/25.3 (BP=1.000, ratio=0.998, syslen=81377, reflen=81194)
+BLEU4 = 40.83, 67.5/46.9/34.4/25.5 (BP=1.000, ratio=1.006, syslen=83262, reflen=82787)
```
# Join the fairseq community

original_message: Update README with new models


message: Adjust weight decay by the current learning rate to make it work correctly during annealing

diff:
@@ -96,7 +96,7 @@ class Adam(Optimizer):
step_size = group['lr'] * math.sqrt(bias_correction2) / bias_correction1
if group['weight_decay'] != 0:
- p.data.add_(-group['weight_decay'], p.data)
+ p.data.add_(-group['weight_decay'] * group['lr'], p.data)
p.data.addcdiv_(-step_size, exp_avg, denom)

diff:
@@ -44,7 +44,7 @@ class NAG(Optimizer):
buf = param_state['momentum_buffer']
if weight_decay != 0:
- p.data.mul_(1 - weight_decay)
+ p.data.mul_(1 - lr * weight_decay)
p.data.add_(momentum * momentum * lr_correct, buf)
p.data.add_(-(1 + momentum) * lr, d_p)


message: Simulated big batches

diff:
@@ -176,6 +176,8 @@ def add_optimization_args(parser):
' dataset')
group.add_argument('--curriculum', default=0, type=int, metavar='N',
help='sort batches by source length for first N epochs')
+ group.add_argument('--update-freq', default=1, type=int, metavar='N',
+ help='update parameters every N batches')
return group

diff:
Train a network on multiple GPUs.
"""
-from collections import OrderedDict
+from collections import defaultdict, OrderedDict
+from itertools import chain
import math
import torch
@@ -55,6 +56,7 @@ class Trainer(object):
self.meters['clip'] = AverageMeter() # % of updates clipped
self.meters['oom'] = AverageMeter() # out of memory
+ self._buffered_stats = defaultdict(lambda: [])
self._max_bsz_seen = 0
self._num_updates = 0
self._optim_history = None
@@ -86,22 +88,46 @@ class Trainer(object):
return extra_state
- def train_step(self, sample):
+ def train_step(self, sample, update_params=True):
"""Do forward, backward and parameter update."""
sample = self._prepare_sample(sample, volatile=False)
- # forward pass
- loss, sample_sizes, logging_outputs, ooms_fwd = self._forward(sample)
+ # forward and backward pass
+ loss, sample_size, logging_output, oom_fwd = self._forward(sample)
+ oom_bwd = self._backward(loss)
+
+ # buffer stats and logging outputs
+ self._buffered_stats['sample_sizes'].append(sample_size)
+ self._buffered_stats['logging_outputs'].append(logging_output)
+ self._buffered_stats['ooms_fwd'].append(oom_fwd)
+ self._buffered_stats['ooms_bwd'].append(oom_bwd)
+
+ # update parameters
+ if update_params:
+ # gather logging outputs from all GPUs
+ sample_sizes = self._buffered_stats['sample_sizes']
+ logging_outputs = self._buffered_stats['logging_outputs']
+ ooms_fwd = self._buffered_stats['ooms_fwd']
+ ooms_bwd = self._buffered_stats['ooms_bwd']
+ if self.args.distributed_world_size > 1:
+ sample_sizes, logging_outputs, ooms_fwd, ooms_bwd = map(
+ lambda l: list(chain.from_iterable(l)),
+ zip(*distributed_utils.all_gather_list(
+ (sample_sizes, logging_outputs, ooms_fwd, ooms_bwd)
+ ))
+ )
+ ooms_fwd = sum(ooms_fwd)
+ ooms_bwd = sum(ooms_bwd)
# aggregate stats and logging outputs
ntokens = sum(log.get('ntokens', 0) for log in logging_outputs)
nsentences = sum(log.get('nsentences', 0) for log in logging_outputs)
- grad_denom = self.criterion.__class__.grad_denom(sample_sizes)
agg_logging_output = self.criterion.__class__.aggregate_logging_outputs(logging_outputs)
- # backward pass, all-reduce gradients and take an optimization step
- grad_norm, ooms_bwd = self._backward_and_opt(loss, grad_denom)
+ # all-reduce gradients and take an optimization step
+ grad_denom = self.criterion.__class__.grad_denom(sample_sizes)
+ grad_norm = self._opt(grad_denom)
# update meters
self.meters['wps'].update(ntokens)
@@ -119,7 +145,11 @@ class Trainer(object):
if 'nll_loss' in agg_logging_output:
self.meters['train_nll_loss'].update(agg_logging_output['nll_loss'], ntokens)
+ self._buffered_stats.clear()
+
return agg_logging_output
+ else:
+ return None # buffering updates
def _forward(self, sample, eval=False):
# prepare model and optimizer
@@ -127,7 +157,6 @@ class Trainer(object):
self.model.eval()
else:
self.model.train()
- self.optimizer.zero_grad()
loss = None
sample_size = 0
@@ -152,19 +181,9 @@ class Trainer(object):
else:
raise e
- # synchronize logging outputs for multi-GPU training
- if self.args.distributed_world_size > 1:
- sample_sizes, logging_outputs, ooms = zip(*list(
- distributed_utils.all_gather_list((sample_size, logging_output, oom))))
- ooms = sum(ooms)
- else:
- sample_sizes = [sample_size]
- logging_outputs = [logging_output]
- ooms = oom
-
- return loss, sample_sizes, logging_outputs, ooms
+ return loss, sample_size, logging_output, oom
- def _backward_and_opt(self, loss, grad_denom):
+ def _backward(self, loss):
oom = 0
if loss is not None:
try:
@@ -179,7 +198,9 @@ class Trainer(object):
self.optimizer.zero_grad()
else:
raise e
+ return oom
+ def _opt(self, grad_denom):
# all-reduce grads and rescale by grad_denom
if self.args.distributed_world_size > 1:
grads = [p.grad.data for p in self.model.parameters() if p.requires_grad]
@@ -197,12 +218,13 @@ class Trainer(object):
# take an optimization step
self.optimizer.step()
+ self.optimizer.zero_grad()
self._num_updates += 1
# update learning rate
self.lr_scheduler.step_update(self._num_updates)
- return grad_norm, oom
+ return grad_norm
def valid_step(self, sample):
"""Do forward pass in evaluation mode."""
@@ -210,8 +232,17 @@ class Trainer(object):
sample = self._prepare_sample(sample, volatile=True)
# forward pass
- loss, sample_sizes, logging_outputs, ooms_fwd = self._forward(sample, eval=True)
- assert not ooms_fwd, 'Ran out of memory during validation'
+ _loss, sample_size, logging_output, oom_fwd = self._forward(sample, eval=True)
+ assert not oom_fwd, 'Ran out of memory during validation'
+
+ # gather logging outputs from all GPUs
+ if self.args.distributed_world_size > 1:
+ sample_sizes, logging_outputs = zip(*distributed_utils.all_gather_list(
+ (sample_size, logging_output)
+ ))
+ else:
+ sample_sizes = [sample_size]
+ logging_outputs = [logging_output]
# aggregate stats and logging outputs
ntokens = sum(log.get('ntokens', 0) for log in logging_outputs)

diff:
@@ -132,6 +132,7 @@ def train(args, trainer, dataset, epoch, batch_offset):
num_shards=args.distributed_world_size,
)
progress = progress_bar.build_progress_bar(args, itr, epoch, no_progress_bar='simple')
+ epoch_size = len(itr)
itr = itertools.islice(progress, batch_offset, None)
# reset training meters
@@ -143,7 +144,12 @@ def train(args, trainer, dataset, epoch, batch_offset):
extra_meters = collections.defaultdict(lambda: AverageMeter())
max_update = args.max_update or math.inf
for i, sample in enumerate(itr, start=batch_offset):
- log_output = trainer.train_step(sample)
+ if i < epoch_size - 1 and (i + 1) % args.update_freq > 0:
+ # buffer updates according to --update-freq
+ trainer.train_step(sample, update_params=False)
+ continue
+ else:
+ log_output = trainer.train_step(sample, update_params=True)
# log mid-epoch stats
stats = get_training_stats(trainer)
@@ -157,9 +163,8 @@ def train(args, trainer, dataset, epoch, batch_offset):
stats[k] = extra_meters[k].avg
progress.log(stats)
- # save mid-epoch checkpoints
- if i == batch_offset:
# ignore the first mini-batch in words-per-second calculation
+ if i == batch_offset:
trainer.get_meter('wps').reset()
# save mid-epoch checkpoints

original_message: Simulated big batches


message: better batching

diff:
@@ -322,10 +322,13 @@ def _make_batches(src, dst, indices, max_tokens, max_sentences, max_positions,
sample_len = max(sample_len, src_size, dst_size)
num_tokens = (len(batch) + 1) * sample_len
- if yield_batch(idx, num_tokens):
- yield batch
- batch = []
- sample_len = max(src_size, dst_size)
+ while yield_batch(idx, num_tokens):
+ mod8_len = max(8 * (len(batch) // 8), len(batch) % 8)
+ yield batch[:mod8_len]
+ batch = batch[mod8_len:]
+ sample_len = max([max(src.sizes[id], dst.sizes[id]) for id in batch]) if len(batch) > 0 else 0
+ sample_len = max(sample_len, src_size, dst_size)
+ num_tokens = (len(batch) + 1) * sample_len
batch.append(idx)

original_message: better batching


message: Sampling doesn't work with interactive

diff:
@@ -41,7 +41,7 @@ def main(args):
translator = SequenceGenerator(
models, beam_size=args.beam, stop_early=(not args.no_early_stop),
normalize_scores=(not args.unnormalized), len_penalty=args.lenpen,
- unk_penalty=args.unkpen)
+ unk_penalty=args.unkpen, sampling=args.sampling)
if use_cuda:
translator.cuda()

original_message: Sampling doesn't work with interactive


diff:
@@ -42,8 +42,8 @@ class SinusoidalPositionalEmbedding(nn.Module):
"""
half_dim = embedding_dim // 2
emb = math.log(10000) / (half_dim - 1)
- emb = torch.exp(torch.arange(half_dim) * -emb)
- emb = torch.arange(num_embeddings).unsqueeze(1) * emb.unsqueeze(0)
+ emb = torch.exp(torch.arange(half_dim, dtype=torch.float) * -emb)
+ emb = torch.arange(num_embeddings, dtype=torch.float).unsqueeze(1) * emb.unsqueeze(0)
emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1).view(num_embeddings, -1)
if embedding_dim % 2 == 1:
# zero pad

original_message: torch.arange default return type is changed in the latest pytorch version https://github.com/pytorch/pytorch/pull/7016


message: Pass sampling-temperature trough to the generator in interactive.py

diff:
@@ -90,7 +90,7 @@ def main(args):
models, tgt_dict, beam_size=args.beam, stop_early=(not args.no_early_stop),
normalize_scores=(not args.unnormalized), len_penalty=args.lenpen,
unk_penalty=args.unkpen, sampling=args.sampling, sampling_topk=args.sampling_topk,
- minlen=args.min_len,
+ minlen=args.min_len, sampling_temperature=args.sampling_temperature
)
if use_cuda: