Skip to content

Commit 8343ce0

Browse files
michaelnebelmichaelnebel
committed
C#: Re-factor the make_stubs_nuget script to more easily allow multiple nuget references.
1 parent c8b4a21 commit 8343ce0

File tree

1 file changed

+190
-175
lines changed

1 file changed

+190
-175
lines changed

‎csharp/scripts/stubs/make_stubs_nuget.py‎

Lines changed: 190 additions & 175 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@ def write_csproj_prefix(ioWrapper):
1515
' <AppendTargetFrameworkToOutputPath>false</AppendTargetFrameworkToOutputPath>\n')
1616
ioWrapper.write(' </PropertyGroup>\n\n')
1717

18-
1918
print('Script to generate stub file from a nuget package')
2019
print(' Usage: python3 ' + sys.argv[0] +
2120
' TEMPLATE NUGET_PACKAGE_NAME [VERSION=latest] [WORK_DIR=tempDir]')
@@ -34,191 +33,207 @@ def write_csproj_prefix(ioWrapper):
3433
thisDir = os.path.abspath(os.path.dirname(thisScript))
3534
template = sys.argv[1]
3635
nuget = sys.argv[2]
37-
38-
# /input contains a dotnet project that's being extracted
39-
workDir = os.path.abspath(helpers.get_argv(4, "tempDir"))
40-
projectNameIn = "input"
41-
projectDirIn = os.path.join(workDir, projectNameIn)
42-
43-
def run_cmd(cmd, msg="Failed to run command"):
44-
helpers.run_cmd_cwd(cmd, workDir, msg)
45-
46-
# /output contains the output of the stub generation
47-
outputDirName = "output"
48-
outputDir = os.path.join(workDir, outputDirName)
49-
50-
# /output/raw contains the bqrs result from the query, the json equivalent
51-
rawOutputDirName = "raw"
52-
rawOutputDir = os.path.join(outputDir, rawOutputDirName)
53-
os.makedirs(rawOutputDir)
54-
55-
# /output/output contains a dotnet project with the generated stubs
56-
projectNameOut = "output"
57-
projectDirOut = os.path.join(outputDir, projectNameOut)
58-
59-
# /db contains the extracted QL DB
60-
dbName = 'db'
61-
dbDir = os.path.join(workDir, dbName)
62-
outputName = "stub"
63-
outputFile = os.path.join(projectDirOut, outputName + '.cs')
64-
bqrsFile = os.path.join(rawOutputDir, outputName + '.bqrs')
65-
jsonFile = os.path.join(rawOutputDir, outputName + '.json')
6636
version = helpers.get_argv(3, "latest")
37+
relativeWorkDir = helpers.get_argv(4, "tempDir")
38+
39+
class Generator:
40+
def __init__(self, thisScript, relativeWorkDir, template):
41+
# /input contains a dotnet project that's being extracted
42+
self.workDir = os.path.abspath(relativeWorkDir)
43+
os.makedirs(self.workDir)
44+
self.thisDir = os.path.abspath(os.path.dirname(thisScript))
45+
self.projectNameIn = "input"
46+
self.projectDirIn = os.path.join(self.workDir, self.projectNameIn)
47+
self.template = template
48+
print("\n* Creating new input project")
49+
self.run_cmd(['dotnet', 'new', self.template, "-f", "net8.0", "--language", "C#", '--name',
50+
self.projectNameIn, '--output', self.projectDirIn])
51+
helpers.remove_files(self.projectDirIn, '.cs')
52+
53+
def run_cmd(self, cmd, msg="Failed to run command"):
54+
helpers.run_cmd_cwd(cmd, self.workDir, msg)
55+
56+
def add_nuget(self, nuget, version="latest"):
57+
print("\n* Adding reference to package: " + nuget)
58+
cmd = ['dotnet', 'add', self.projectDirIn, 'package', nuget]
59+
if (version != "latest"):
60+
cmd.append('--version')
61+
cmd.append(version)
62+
self.run_cmd(cmd)
63+
64+
def make_stubs(self):
65+
# /output contains the output of the stub generation
66+
outputDirName = "output"
67+
outputDir = os.path.join(self.workDir, outputDirName)
68+
69+
# /output/raw contains the bqrs result from the query, the json equivalent
70+
rawOutputDirName = "raw"
71+
rawOutputDir = os.path.join(outputDir, rawOutputDirName)
72+
os.makedirs(rawOutputDir)
73+
74+
# /output/output contains a dotnet project with the generated stubs
75+
projectNameOut = "output"
76+
projectDirOut = os.path.join(outputDir, projectNameOut)
77+
78+
# /db contains the extracted QL DB
79+
dbName = 'db'
80+
dbDir = os.path.join(self.workDir, dbName)
81+
outputName = "stub"
82+
outputFile = os.path.join(projectDirOut, outputName + '.cs')
83+
bqrsFile = os.path.join(rawOutputDir, outputName + '.bqrs')
84+
jsonFile = os.path.join(rawOutputDir, outputName + '.json')
85+
86+
sdk_version = '8.0.100'
87+
print("\n* Creating new global.json file and setting SDK to " + sdk_version)
88+
self.run_cmd(['dotnet', 'new', 'globaljson', '--force', '--sdk-version', sdk_version, '--output', self.workDir])
89+
90+
print("\n* Running stub generator")
91+
helpers.run_cmd_cwd(['dotnet', 'run', '--project', self.thisDir + '/../../extractor/Semmle.Extraction.CSharp.DependencyStubGenerator/Semmle.Extraction.CSharp.DependencyStubGenerator.csproj'], self.projectDirIn)
92+
93+
print("\n* Creating new raw output project")
94+
rawSrcOutputDirName = 'src'
95+
rawSrcOutputDir = os.path.join(rawOutputDir, rawSrcOutputDirName)
96+
self.run_cmd(['dotnet', 'new', self.template, "--language", "C#",
97+
'--name', rawSrcOutputDirName, '--output', rawSrcOutputDir])
98+
helpers.remove_files(rawSrcOutputDir, '.cs')
99+
100+
# copy each file from projectDirIn to rawSrcOutputDir
101+
pathInfos = {}
102+
codeqlStubsDir = os.path.join(self.projectDirIn, 'codeql_csharp_stubs')
103+
for root, dirs, files in os.walk(codeqlStubsDir):
104+
for file in files:
105+
if file.endswith('.cs'):
106+
path = os.path.join(root, file)
107+
relPath, _ = os.path.splitext(os.path.relpath(path, codeqlStubsDir))
108+
origDllPath = "/" + relPath + ".dll"
109+
pathInfos[origDllPath] = os.path.join(rawSrcOutputDir, file)
110+
shutil.copy2(path, rawSrcOutputDir)
111+
112+
print("\n --> Generated stub files: " + rawSrcOutputDir)
113+
114+
print("\n* Formatting files")
115+
self.run_cmd(['dotnet', 'format', 'whitespace', rawSrcOutputDir])
116+
117+
print("\n --> Generated (formatted) stub files: " + rawSrcOutputDir)
118+
119+
print("\n* Processing project.assets.json to generate folder structure")
120+
stubsDirName = 'stubs'
121+
stubsDir = os.path.join(outputDir, stubsDirName)
122+
os.makedirs(stubsDir)
123+
124+
frameworksDirName = '_frameworks'
125+
frameworksDir = os.path.join(stubsDir, frameworksDirName)
126+
127+
frameworks = set()
128+
copiedFiles = set()
129+
130+
assetsJsonFile = os.path.join(self.projectDirIn, 'obj', 'project.assets.json')
131+
with open(assetsJsonFile) as json_data:
132+
data = json.load(json_data)
133+
if len(data['targets']) > 1:
134+
print("ERROR: More than one target found in " + assetsJsonFile)
135+
exit(1)
136+
target = list(data['targets'].keys())[0]
137+
print("Found target: " + target)
138+
for package in data['targets'][target].keys():
139+
parts = package.split('/')
140+
name = parts[0]
141+
version = parts[1]
142+
packageDir = os.path.join(stubsDir, name, version)
143+
if not os.path.exists(packageDir):
144+
os.makedirs(packageDir)
145+
print(' * Processing package: ' + name + '/' + version)
146+
with open(os.path.join(packageDir, name + '.csproj'), 'a') as pf:
147+
148+
write_csproj_prefix(pf)
149+
pf.write(' <ItemGroup>\n')
150+
151+
dlls = set()
152+
if 'compile' in data['targets'][target][package]:
153+
for dll in data['targets'][target][package]['compile']:
154+
dlls.add(
155+
(name + '/' + version + '/' + dll).lower())
156+
if 'runtime' in data['targets'][target][package]:
157+
for dll in data['targets'][target][package]['runtime']:
158+
dlls.add((name + '/' + version + '/' + dll).lower())
159+
160+
for pathInfo in pathInfos:
161+
for dll in dlls:
162+
if pathInfo.lower().endswith(dll):
163+
copiedFiles.add(pathInfo)
164+
shutil.copy2(pathInfos[pathInfo], packageDir)
165+
166+
if 'dependencies' in data['targets'][target][package]:
167+
for dependency in data['targets'][target][package]['dependencies'].keys():
168+
depVersion = data['targets'][target][package]['dependencies'][dependency]
169+
pf.write(' <ProjectReference Include="../../' +
170+
dependency + '/' + depVersion + '/' + dependency + '.csproj" />\n')
171+
172+
if 'frameworkReferences' in data['targets'][target][package]:
173+
if not os.path.exists(frameworksDir):
174+
os.makedirs(frameworksDir)
175+
for framework in data['targets'][target][package]['frameworkReferences']:
176+
frameworks.add(framework)
177+
frameworkDir = os.path.join(
178+
frameworksDir, framework)
179+
if not os.path.exists(frameworkDir):
180+
os.makedirs(frameworkDir)
181+
pf.write(' <ProjectReference Include="../../' +
182+
frameworksDirName + '/' + framework + '/' + framework + '.csproj" />\n')
67183

68-
print("\n* Creating new input project")
69-
run_cmd(['dotnet', 'new', template, "-f", "net8.0", "--language", "C#", '--name',
70-
projectNameIn, '--output', projectDirIn])
71-
helpers.remove_files(projectDirIn, '.cs')
72-
73-
print("\n* Adding reference to package: " + nuget)
74-
cmd = ['dotnet', 'add', projectDirIn, 'package', nuget]
75-
if (version != "latest"):
76-
cmd.append('--version')
77-
cmd.append(version)
78-
run_cmd(cmd)
79-
80-
sdk_version = '8.0.100'
81-
print("\n* Creating new global.json file and setting SDK to " + sdk_version)
82-
run_cmd(['dotnet', 'new', 'globaljson', '--force', '--sdk-version', sdk_version, '--output', workDir])
83-
84-
print("\n* Running stub generator")
85-
helpers.run_cmd_cwd(['dotnet', 'run', '--project', thisDir + '/../../extractor/Semmle.Extraction.CSharp.DependencyStubGenerator/Semmle.Extraction.CSharp.DependencyStubGenerator.csproj'], projectDirIn)
86-
87-
print("\n* Creating new raw output project")
88-
rawSrcOutputDirName = 'src'
89-
rawSrcOutputDir = os.path.join(rawOutputDir, rawSrcOutputDirName)
90-
run_cmd(['dotnet', 'new', template, "--language", "C#",
91-
'--name', rawSrcOutputDirName, '--output', rawSrcOutputDir])
92-
helpers.remove_files(rawSrcOutputDir, '.cs')
93-
94-
# copy each file from projectDirIn to rawSrcOutputDir
95-
pathInfos = {}
96-
codeqlStubsDir = os.path.join(projectDirIn, 'codeql_csharp_stubs')
97-
for root, dirs, files in os.walk(codeqlStubsDir):
98-
for file in files:
99-
if file.endswith('.cs'):
100-
path = os.path.join(root, file)
101-
relPath, _ = os.path.splitext(os.path.relpath(path, codeqlStubsDir))
102-
origDllPath = "/" + relPath + ".dll"
103-
pathInfos[origDllPath] = os.path.join(rawSrcOutputDir, file)
104-
shutil.copy2(path, rawSrcOutputDir)
105-
106-
print("\n --> Generated stub files: " + rawSrcOutputDir)
107-
108-
print("\n* Formatting files")
109-
run_cmd(['dotnet', 'format', 'whitespace', rawSrcOutputDir])
110-
111-
print("\n --> Generated (formatted) stub files: " + rawSrcOutputDir)
112-
113-
print("\n* Processing project.assets.json to generate folder structure")
114-
stubsDirName = 'stubs'
115-
stubsDir = os.path.join(outputDir, stubsDirName)
116-
os.makedirs(stubsDir)
117-
118-
frameworksDirName = '_frameworks'
119-
frameworksDir = os.path.join(stubsDir, frameworksDirName)
120-
121-
frameworks = set()
122-
copiedFiles = set()
123-
124-
assetsJsonFile = os.path.join(projectDirIn, 'obj', 'project.assets.json')
125-
with open(assetsJsonFile) as json_data:
126-
data = json.load(json_data)
127-
if len(data['targets']) > 1:
128-
print("ERROR: More than one target found in " + assetsJsonFile)
129-
exit(1)
130-
target = list(data['targets'].keys())[0]
131-
print("Found target: " + target)
132-
for package in data['targets'][target].keys():
133-
parts = package.split('/')
134-
name = parts[0]
135-
version = parts[1]
136-
packageDir = os.path.join(stubsDir, name, version)
137-
if not os.path.exists(packageDir):
138-
os.makedirs(packageDir)
139-
print(' * Processing package: ' + name + '/' + version)
140-
with open(os.path.join(packageDir, name + '.csproj'), 'a') as pf:
184+
pf.write(' <ProjectReference Include="../../' +
185+
frameworksDirName + '/Microsoft.NETCore.App/Microsoft.NETCore.App.csproj" />\n')
141186

142-
write_csproj_prefix(pf)
143-
pf.write(' <ItemGroup>\n')
187+
pf.write(' </ItemGroup>\n')
188+
pf.write('</Project>\n')
144189

145-
dlls = set()
146-
if 'compile' in data['targets'][target][package]:
147-
for dll in data['targets'][target][package]['compile']:
148-
dlls.add(
149-
(name + '/' + version + '/' + dll).lower())
150-
if 'runtime' in data['targets'][target][package]:
151-
for dll in data['targets'][target][package]['runtime']:
152-
dlls.add((name + '/' + version + '/' + dll).lower())
190+
# Processing references frameworks
191+
for framework in frameworks:
192+
with open(os.path.join(frameworksDir, framework, framework + '.csproj'), 'a') as pf:
153193

154-
for pathInfo in pathInfos:
155-
for dll in dlls:
156-
if pathInfo.lower().endswith(dll):
194+
write_csproj_prefix(pf)
195+
pf.write(' <ItemGroup>\n')
196+
pf.write(
197+
' <ProjectReference Include="../Microsoft.NETCore.App/Microsoft.NETCore.App.csproj" />\n')
198+
pf.write(' </ItemGroup>\n')
199+
pf.write('</Project>\n')
200+
201+
for pathInfo in pathInfos:
202+
if framework.lower() + '.ref' in pathInfo.lower():
157203
copiedFiles.add(pathInfo)
158-
shutil.copy2(pathInfos[pathInfo], packageDir)
204+
shutil.copy2(pathInfos[pathInfo], os.path.join(
205+
frameworksDir, framework))
206+
207+
# Processing assemblies in Microsoft.NETCore.App.Ref
208+
frameworkDir = os.path.join(frameworksDir, 'Microsoft.NETCore.App')
209+
if not os.path.exists(frameworkDir):
210+
os.makedirs(frameworkDir)
211+
with open(os.path.join(frameworksDir, 'Microsoft.NETCore.App', 'Microsoft.NETCore.App.csproj'), 'a') as pf:
212+
write_csproj_prefix(pf)
213+
pf.write('</Project>\n')
159214

160-
if 'dependencies' in data['targets'][target][package]:
161-
for dependency in data['targets'][target][package]['dependencies'].keys():
162-
depVersion = data['targets'][target][package]['dependencies'][dependency]
163-
pf.write(' <ProjectReference Include="../../' +
164-
dependency + '/' + depVersion + '/' + dependency + '.csproj" />\n')
165-
166-
if 'frameworkReferences' in data['targets'][target][package]:
167-
if not os.path.exists(frameworksDir):
168-
os.makedirs(frameworksDir)
169-
for framework in data['targets'][target][package]['frameworkReferences']:
170-
frameworks.add(framework)
171-
frameworkDir = os.path.join(
172-
frameworksDir, framework)
173-
if not os.path.exists(frameworkDir):
174-
os.makedirs(frameworkDir)
175-
pf.write(' <ProjectReference Include="../../' +
176-
frameworksDirName + '/' + framework + '/' + framework + '.csproj" />\n')
215+
for pathInfo in pathInfos:
216+
if 'microsoft.netcore.app.ref/' in pathInfo.lower():
217+
copiedFiles.add(pathInfo)
218+
shutil.copy2(pathInfos[pathInfo], frameworkDir)
177219

178-
pf.write(' <ProjectReference Include="../../' +
179-
frameworksDirName + '/Microsoft.NETCore.App/Microsoft.NETCore.App.csproj" />\n')
220+
for pathInfo in pathInfos:
221+
if pathInfo not in copiedFiles:
222+
print('Not copied to nuget or framework folder: ' + pathInfo)
223+
othersDir = os.path.join(stubsDir, 'others')
224+
if not os.path.exists(othersDir):
225+
os.makedirs(othersDir)
226+
shutil.copy2(pathInfos[pathInfo], othersDir)
180227

181-
pf.write(' </ItemGroup>\n')
182-
pf.write('</Project>\n')
228+
print("\n --> Generated structured stub files: " + stubsDir)
183229

184-
# Processing references frameworks
185-
for framework in frameworks:
186-
with open(os.path.join(frameworksDir, framework, framework + '.csproj'), 'a') as pf:
187230

188-
write_csproj_prefix(pf)
189-
pf.write(' <ItemGroup>\n')
190-
pf.write(
191-
' <ProjectReference Include="../Microsoft.NETCore.App/Microsoft.NETCore.App.csproj" />\n')
192-
pf.write(' </ItemGroup>\n')
193-
pf.write('</Project>\n')
194231

195-
for pathInfo in pathInfos:
196-
if framework.lower() + '.ref' in pathInfo.lower():
197-
copiedFiles.add(pathInfo)
198-
shutil.copy2(pathInfos[pathInfo], os.path.join(
199-
frameworksDir, framework))
200-
201-
# Processing assemblies in Microsoft.NETCore.App.Ref
202-
frameworkDir = os.path.join(frameworksDir, 'Microsoft.NETCore.App')
203-
if not os.path.exists(frameworkDir):
204-
os.makedirs(frameworkDir)
205-
with open(os.path.join(frameworksDir, 'Microsoft.NETCore.App', 'Microsoft.NETCore.App.csproj'), 'a') as pf:
206-
write_csproj_prefix(pf)
207-
pf.write('</Project>\n')
208-
209-
for pathInfo in pathInfos:
210-
if 'microsoft.netcore.app.ref/' in pathInfo.lower():
211-
copiedFiles.add(pathInfo)
212-
shutil.copy2(pathInfos[pathInfo], frameworkDir)
213-
214-
for pathInfo in pathInfos:
215-
if pathInfo not in copiedFiles:
216-
print('Not copied to nuget or framework folder: ' + pathInfo)
217-
othersDir = os.path.join(stubsDir, 'others')
218-
if not os.path.exists(othersDir):
219-
os.makedirs(othersDir)
220-
shutil.copy2(pathInfos[pathInfo], othersDir)
221-
222-
print("\n --> Generated structured stub files: " + stubsDir)
232+
233+
234+
generator = Generator(thisScript, relativeWorkDir, template)
235+
generator.add_nuget(nuget, version)
236+
generator.make_stubs()
237+
223238

224239
exit(0)

0 commit comments

Comments
 (0)