Put validation tests behind #if due to confirmed MSI inconsistencies
When the validation tests are run repeatedly, they will expose an issue in the Windows Installer where ICE validations will occasionally fail to send all error/warning messages. The inconsistency obviously wreaks havoc on the tests. The tests are still interesting (and complex) enough to keep around for manual testing. So they were placed behind an #if for easy manual inclusion. Note that these are "negative tests" that expose the Windows Installer inconsistency when expected failures are missing. Other validation tests are "positive tests" that look for success and thus will always succeed. Therefore, the positive tests stay active.
Rob Mensching committed
Feb 10, 2022 at 19:19 UTC
40492973be6075efbfbd231aed7168194d36cc73
3 files changed
+109
-78
src/wix/test/WixToolsetTest.CoreIntegration/ValidationFixture.cs
+10
@@ -12,6 +12,15 @@ namespace WixToolsetTest.CoreIntegration
12
using WixToolset.Data.WindowsInstaller;
13
using Xunit;
14
15
+ // When these tests are run repeatedly, they will expose an issue
16
+ // in the Windows Installer where ICE validations will occasionally
17
+ // fail to send all error/warning messages. The inconsistency
18
+ // obviously wreaks havoc on the tests.
19
+ //
20
+ // These tests are still interesting (and complex) enough to keep
21
+ // around for manual testing. Uncomment or define the following
22
+ // line to do so.
23
+#if DISABLE_VALIDATION_TESTS_DUE_TO_WINDOWS_INSTALLER_INCONSISTENCIES
24
public class ValidationFixture
25
{
26
[Fact]
@@ -303,4 +312,5 @@ namespace WixToolsetTest.CoreIntegration
312
}
313
}
314
}
315
+#endif
316
}
src/wix/test/WixToolsetTest.Sdk/MsbuildFixture.cs
-78
@@ -6,8 +6,6 @@ namespace WixToolsetTest.Sdk
6
using System.Collections.Generic;
7
using System.IO;
8
using System.Linq;
9
- using System.Runtime.CompilerServices;
10
- using System.Threading;
9
using WixBuildTools.TestSupport;
10
using Xunit;
11
@@ -322,82 +320,6 @@ namespace WixToolsetTest.Sdk
320
}
321
}
322
325
- [Theory]
326
- [InlineData(BuildSystem.DotNetCoreSdk)]
327
- [InlineData(BuildSystem.MSBuild)]
328
- [InlineData(BuildSystem.MSBuild64)]
329
- public void CannotBuildMsiPackageWithIceIssues(BuildSystem buildSystem)
330
- {
331
- var sourceFolder = TestData.Get(@"TestData\MsiPackageWithIceError\MsiPackage");
332
-
333
- var testLogsFolder = TestData.GetUnitTestLogsFolder();
334
- File.Delete(Path.Combine(testLogsFolder, buildSystem + ".binlog"));
335
- File.Delete(Path.Combine(testLogsFolder, buildSystem + ".msi"));
336
-
337
- using (var fs = new TestDataFolderFileSystem())
338
- {
339
- fs.Initialize(sourceFolder);
340
- var baseFolder = fs.BaseFolder;
341
- var projectPath = Path.Combine(baseFolder, "MsiPackage.wixproj");
342
-
343
- var result = MsbuildUtilities.BuildProject(buildSystem, projectPath, suppressValidation: false);
344
-
345
- File.Copy(Path.ChangeExtension(projectPath, ".binlog"), Path.Combine(testLogsFolder, buildSystem + ".binlog"));
346
- File.Copy(Path.Combine(baseFolder, "obj", "x86", "Release", "en-US", "MsiPackage.msi"), Path.Combine(testLogsFolder, buildSystem + ".msi"));
347
-
348
- var iceIssues = result.Output.Where(line => line.Contains(": error") || line.Contains(": warning"))
349
- .Select(line => line.Replace(baseFolder, "<baseFolder>")
350
- .Replace("1>", String.Empty) // remove any multi-proc markers
351
- .Replace("WIX204", "WIX0204") // normalize error number reporting because MSBuild is not consistent on zero padding
352
- .Trim())
353
- .OrderBy(s => s, StringComparer.OrdinalIgnoreCase)
354
- .ToArray();
355
- WixAssert.CompareLineByLine(new[]
356
- {
357
- @"<baseFolder>\Package.wxs(17): error WIX0204: ICE12: CustomAction: CausesICE12Error is of type: 35. Therefore it must come after CostFinalize @ 1000 in Seq Table: InstallExecuteSequence. CA Seq#: 49 [<baseFolder>\MsiPackage.wixproj]",
358
- @"<baseFolder>\Package.wxs(17): error WIX0204: ICE12: CustomAction: CausesICE12Error is of type: 35. Therefore it must come after CostFinalize @ 1000 in Seq Table: InstallExecuteSequence. CA Seq#: 49 [<baseFolder>\MsiPackage.wixproj]",
359
- @"<baseFolder>\Package.wxs(8): warning WIX1076: ICE46: Property 'Myproperty' referenced in column 'LaunchCondition'.'Condition' of row 'Myproperty' differs from a defined property by case only. [<baseFolder>\MsiPackage.wixproj]",
360
- @"<baseFolder>\Package.wxs(8): warning WIX1076: ICE46: Property 'Myproperty' referenced in column 'LaunchCondition'.'Condition' of row 'Myproperty' differs from a defined property by case only. [<baseFolder>\MsiPackage.wixproj]",
361
- }, iceIssues);
362
- }
363
- }
364
-
365
- [Theory(Skip = "Flaky")]
366
- [InlineData(BuildSystem.DotNetCoreSdk)]
367
- [InlineData(BuildSystem.MSBuild)]
368
- [InlineData(BuildSystem.MSBuild64)]
369
- public void CannotBuildMsiPackageWithIceWarningsAsErrors(BuildSystem buildSystem)
370
- {
371
- var sourceFolder = TestData.Get(@"TestData\MsiPackageWithIceError\MsiPackage");
372
-
373
- using (var fs = new TestDataFolderFileSystem())
374
- {
375
- fs.Initialize(sourceFolder);
376
- var baseFolder = fs.BaseFolder;
377
- var projectPath = Path.Combine(baseFolder, "MsiPackage.wixproj");
378
-
379
- var result = MsbuildUtilities.BuildProject(buildSystem, projectPath, new[]
380
- {
381
- $"-p:TreatWarningsAsErrors=true",
382
- MsbuildUtilities.GetQuotedPropertySwitch(buildSystem, "SuppressIces", "ICE12"),
383
- }, suppressValidation: false);
384
- Assert.Equal(1, result.ExitCode);
385
-
386
- var iceIssues = result.Output.Where(line => (line.Contains(": error") || line.Contains(": warning")))
387
- .Select(line => line.Replace(baseFolder, "<baseFolder>")
388
- .Replace("1>", String.Empty) // remove any multi-proc markers
389
- .Replace("WIX204", "WIX0204") // normalize error number reporting because MSBuild is not consistent on zero padding
390
- .Trim())
391
- .OrderBy(s => s, StringComparer.OrdinalIgnoreCase)
392
- .ToArray();
393
- WixAssert.CompareLineByLine(new[]
394
- {
395
- @"<baseFolder>\Package.wxs(8): error WIX1076: ICE46: Property 'Myproperty' referenced in column 'LaunchCondition'.'Condition' of row 'Myproperty' differs from a defined property by case only. [<baseFolder>\MsiPackage.wixproj]",
396
- @"<baseFolder>\Package.wxs(8): error WIX1076: ICE46: Property 'Myproperty' referenced in column 'LaunchCondition'.'Condition' of row 'Myproperty' differs from a defined property by case only. [<baseFolder>\MsiPackage.wixproj]",
397
- }, iceIssues);
398
- }
399
- }
400
-
323
[Theory]
324
[InlineData(BuildSystem.DotNetCoreSdk)]
325
[InlineData(BuildSystem.MSBuild)]
src/wix/test/WixToolsetTest.Sdk/MsbuildValidationFixture.cs
new
+99
@@ -0,0 +1,99 @@
1
+// Copyright (c) .NET Foundation and contributors. All rights reserved. Licensed under the Microsoft Reciprocal License. See LICENSE.TXT file in the project root for full license information.
2
+
3
+namespace WixToolsetTest.Sdk
4
+{
5
+ using System;
6
+ using System.IO;
7
+ using System.Linq;
8
+ using WixBuildTools.TestSupport;
9
+ using Xunit;
10
+
11
+ // When these tests are run repeatedly, they will expose an issue
12
+ // in the Windows Installer where ICE validations will occasionally
13
+ // fail to send all error/warning messages. The inconsistency
14
+ // obviously wreaks havoc on the tests.
15
+ //
16
+ // These tests are still interesting (and complex) enough to keep
17
+ // around for manual testing. Uncomment or define the following
18
+ // line to do so.
19
+#if DISABLE_VALIDATION_TESTS_DUE_TO_WINDOWS_INSTALLER_INCONSISTENCIES
20
+ public class MsbuildValidationFixture
21
+ {
22
+ [Theory]
23
+ [InlineData(BuildSystem.DotNetCoreSdk)]
24
+ [InlineData(BuildSystem.MSBuild)]
25
+ [InlineData(BuildSystem.MSBuild64)]
26
+ public void CannotBuildMsiPackageWithIceIssues(BuildSystem buildSystem)
27
+ {
28
+ var sourceFolder = TestData.Get(@"TestData\MsiPackageWithIceError\MsiPackage");
29
+
30
+ var testLogsFolder = TestData.GetUnitTestLogsFolder();
31
+ File.Delete(Path.Combine(testLogsFolder, buildSystem + ".binlog"));
32
+ File.Delete(Path.Combine(testLogsFolder, buildSystem + ".msi"));
33
+
34
+ using (var fs = new TestDataFolderFileSystem())
35
+ {
36
+ fs.Initialize(sourceFolder);
37
+ var baseFolder = fs.BaseFolder;
38
+ var projectPath = Path.Combine(baseFolder, "MsiPackage.wixproj");
39
+
40
+ var result = MsbuildUtilities.BuildProject(buildSystem, projectPath, suppressValidation: false);
41
+
42
+ File.Copy(Path.ChangeExtension(projectPath, ".binlog"), Path.Combine(testLogsFolder, buildSystem + ".binlog"));
43
+ File.Copy(Path.Combine(baseFolder, "obj", "x86", "Release", "en-US", "MsiPackage.msi"), Path.Combine(testLogsFolder, buildSystem + ".msi"));
44
+
45
+ var iceIssues = result.Output.Where(line => line.Contains(": error") || line.Contains(": warning"))
46
+ .Select(line => line.Replace(baseFolder, "<baseFolder>")
47
+ .Replace("1>", String.Empty) // remove any multi-proc markers
48
+ .Replace("WIX204", "WIX0204") // normalize error number reporting because MSBuild is not consistent on zero padding
49
+ .Trim())
50
+ .OrderBy(s => s, StringComparer.OrdinalIgnoreCase)
51
+ .ToArray();
52
+ WixAssert.CompareLineByLine(new[]
53
+ {
54
+ @"<baseFolder>\Package.wxs(17): error WIX0204: ICE12: CustomAction: CausesICE12Error is of type: 35. Therefore it must come after CostFinalize @ 1000 in Seq Table: InstallExecuteSequence. CA Seq#: 49 [<baseFolder>\MsiPackage.wixproj]",
55
+ @"<baseFolder>\Package.wxs(17): error WIX0204: ICE12: CustomAction: CausesICE12Error is of type: 35. Therefore it must come after CostFinalize @ 1000 in Seq Table: InstallExecuteSequence. CA Seq#: 49 [<baseFolder>\MsiPackage.wixproj]",
56
+ @"<baseFolder>\Package.wxs(8): warning WIX1076: ICE46: Property 'Myproperty' referenced in column 'LaunchCondition'.'Condition' of row 'Myproperty' differs from a defined property by case only. [<baseFolder>\MsiPackage.wixproj]",
57
+ @"<baseFolder>\Package.wxs(8): warning WIX1076: ICE46: Property 'Myproperty' referenced in column 'LaunchCondition'.'Condition' of row 'Myproperty' differs from a defined property by case only. [<baseFolder>\MsiPackage.wixproj]",
58
+ }, iceIssues);
59
+ }
60
+ }
61
+
62
+ [Theory]
63
+ [InlineData(BuildSystem.DotNetCoreSdk)]
64
+ [InlineData(BuildSystem.MSBuild)]
65
+ [InlineData(BuildSystem.MSBuild64)]
66
+ public void CannotBuildMsiPackageWithIceWarningsAsErrors(BuildSystem buildSystem)
67
+ {
68
+ var sourceFolder = TestData.Get(@"TestData\MsiPackageWithIceError\MsiPackage");
69
+
70
+ using (var fs = new TestDataFolderFileSystem())
71
+ {
72
+ fs.Initialize(sourceFolder);
73
+ var baseFolder = fs.BaseFolder;
74
+ var projectPath = Path.Combine(baseFolder, "MsiPackage.wixproj");
75
+
76
+ var result = MsbuildUtilities.BuildProject(buildSystem, projectPath, new[]
77
+ {
78
+ $"-p:TreatWarningsAsErrors=true",
79
+ MsbuildUtilities.GetQuotedPropertySwitch(buildSystem, "SuppressIces", "ICE12"),
80
+ }, suppressValidation: false);
81
+ Assert.Equal(1, result.ExitCode);
82
+
83
+ var iceIssues = result.Output.Where(line => (line.Contains(": error") || line.Contains(": warning")))
84
+ .Select(line => line.Replace(baseFolder, "<baseFolder>")
85
+ .Replace("1>", String.Empty) // remove any multi-proc markers
86
+ .Replace("WIX204", "WIX0204") // normalize error number reporting because MSBuild is not consistent on zero padding
87
+ .Trim())
88
+ .OrderBy(s => s, StringComparer.OrdinalIgnoreCase)
89
+ .ToArray();
90
+ WixAssert.CompareLineByLine(new[]
91
+ {
92
+ @"<baseFolder>\Package.wxs(8): error WIX1076: ICE46: Property 'Myproperty' referenced in column 'LaunchCondition'.'Condition' of row 'Myproperty' differs from a defined property by case only. [<baseFolder>\MsiPackage.wixproj]",
93
+ @"<baseFolder>\Package.wxs(8): error WIX1076: ICE46: Property 'Myproperty' referenced in column 'LaunchCondition'.'Condition' of row 'Myproperty' differs from a defined property by case only. [<baseFolder>\MsiPackage.wixproj]",
94
+ }, iceIssues);
95
+ }
96
+ }
97
+ }
98
+#endif
99
+}