Skip to content
This repository was archived by the owner on May 14, 2025. It is now read-only.

Commit eccc429

Browse files
cppwfsmarkpollack
cppwfs
authored and
markpollack
committed
Redacts sensitive/password information when listing streams
resolves #1542 * Uses regex to identify values that need to be redacted. * Added a few more tests Removed Parser changes * Added docs to regex * Added empty string check to sanitizer
1 parent 2c86b38 commit eccc429

File tree

4 files changed

+181
-10
lines changed

4 files changed

+181
-10
lines changed

‎spring-cloud-dataflow-core/src/test/java/org/springframework/cloud/dataflow/core/dsl/StreamParserTests.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -493,8 +493,6 @@ public void needAdjacentTokensForParameters() {
493493
checkForParseError("foo --name= value", DSLMessage.NO_WHITESPACE_BEFORE_ARG_VALUE, 12);
494494
}
495495

496-
// ---
497-
498496
@Test
499497
public void testComposedOptionNameErros() {
500498
checkForParseError("foo --name.=value", DSLMessage.NOT_EXPECTED_TOKEN, 11);
@@ -513,6 +511,9 @@ public void testXD2416() {
513511
equalTo("payload" + ".replace(\"abc\", '')"));
514512
}
515513

514+
515+
// ---
516+
516517
StreamNode parse(String streamDefinition) {
517518
return new StreamParser(streamDefinition).parse();
518519
}

‎spring-cloud-dataflow-server-core/src/main/java/org/springframework/cloud/dataflow/server/controller/StreamDefinitionController.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@
4242
import org.springframework.cloud.dataflow.rest.resource.DeploymentStateResource;
4343
import org.springframework.cloud.dataflow.rest.resource.StreamDefinitionResource;
4444
import org.springframework.cloud.dataflow.server.DataFlowServerUtil;
45+
import org.springframework.cloud.dataflow.server.controller.support.ArgumentSanitizer;
4546
import org.springframework.cloud.dataflow.server.controller.support.ControllerUtils;
4647
import org.springframework.cloud.dataflow.server.controller.support.InvalidStreamDefinitionException;
4748
import org.springframework.cloud.dataflow.server.repository.DeploymentIdRepository;
@@ -81,6 +82,7 @@
8182
* @author Ilayaperumal Gopinathan
8283
* @author Gunnar Hillert
8384
* @author Oleg Zhurakousky
85+
* @author Glenn Renfro
8486
*/
8587
@RestController
8688
@RequestMapping("/streams/definitions")
@@ -189,9 +191,9 @@ static DeploymentState aggregateState(Set<DeploymentState> states) {
189191
/**
190192
* Return a page-able list of {@link StreamDefinitionResource} defined streams.
191193
*
192-
* @param pageable page-able collection of {@code StreamDefinitionResource}s.
194+
* @param pageable page-able collection of {@code StreamDefinitionResource}s.
193195
* @param assembler assembler for {@link StreamDefinition}
194-
* @param search optional search parameter
196+
* @param search optional search parameter
195197
* @return list of stream definitions
196198
*/
197199
@RequestMapping(value = "", method = RequestMethod.GET)
@@ -407,7 +409,7 @@ public StreamDefinitionResource toResource(StreamDefinition stream) {
407409
@Override
408410
public StreamDefinitionResource instantiateResource(StreamDefinition stream) {
409411
final StreamDefinitionResource resource = new StreamDefinitionResource(stream.getName(),
410-
stream.getDslText());
412+
ArgumentSanitizer.sanitizeStream(stream.getDslText()));
411413
final DeploymentStateResource deploymentStateResource = ControllerUtils
412414
.mapState(streamDeploymentStates.get(stream));
413415
resource.setStatus(deploymentStateResource.getKey());

‎spring-cloud-dataflow-server-core/src/main/java/org/springframework/cloud/dataflow/server/controller/support/ArgumentSanitizer.java‎

Lines changed: 58 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,17 +16,34 @@
1616

1717
package org.springframework.cloud.dataflow.server.controller.support;
1818

19+
import java.util.regex.Matcher;
1920
import java.util.regex.Pattern;
2021

22+
import org.springframework.util.Assert;
23+
import org.springframework.util.StringUtils;
24+
2125
/**
2226
* Sanitizes potentially sensitive keys for a specific command line arg.
27+
*
2328
* @author Glenn Renfro
2429
*/
2530
public class ArgumentSanitizer {
26-
private static final String[] REGEX_PARTS = { "*", "$", "^", "+" };
31+
private static final String[] REGEX_PARTS = {"*", "$", "^", "+"};
32+
33+
private static final String REDACTION_STRING = "******";
34+
35+
private static final String[] KEYS_TO_SANITIZE = {"password", "secret", "key", "token", ".*credentials.*",
36+
"vcap_services"};
37+
//used to find the passwords embedded in a stream definition
38+
private static Pattern passwordParameterPatternForStreams = Pattern.compile(
39+
//Search for the -- characters then look for unicode letters
40+
"(?i)(--[\\p{Z}]*[\\p{L}]*("
41+
//that match the following strings from the KEYS_TO_SANITIZE array
42+
+ StringUtils.arrayToDelimitedString(KEYS_TO_SANITIZE, "|")
43+
//Following the equal sign (group1) accept any number of unicode characters(letters, open punctuation, close punctuation etc) for the value to be sanitized for group 3.
44+
+ ")[\\p{L}]*[\\p{Z}]*=[\\p{Z}]*)((\"[\\p{L}|\\p{Pd}|\\p{Ps}|\\p{Pe}|\\p{Pc}|\\p{S}|\\p{N}|\\p{Z}]*\")|([\\p{N}|\\p{L}|\\p{Po}|\\p{Pc}|\\p{S}]*))",
45+
Pattern.UNICODE_CASE);
2746

28-
private static final String[] KEYS_TO_SANITIZE = { "password", "secret", "key", "token", ".*credentials.*",
29-
"vcap_services" };
3047

3148
private Pattern[] keysToSanitize;
3249

@@ -55,6 +72,7 @@ private boolean isRegex(String value) {
5572

5673
/**
5774
* Replaces a potential secure value with "******".
75+
*
5876
* @param argument the argument to cleanse.
5977
* @return the argument with a potentially sanitized value
6078
*/
@@ -67,10 +85,46 @@ public String sanitize(String argument) {
6785
String value = argument.substring(indexOfFirstEqual + 1);
6886
for (Pattern pattern : this.keysToSanitize) {
6987
if (pattern.matcher(key).matches()) {
70-
value = "******";
88+
value = REDACTION_STRING;
7189
break;
7290
}
7391
}
7492
return String.format("%s=%s", key, value);
7593
}
94+
95+
/**
96+
* Redacts sensitive values in a stream.
97+
*
98+
* @param definition the definition to sanitize
99+
* @return Stream definition that has sensitive data redacted.
100+
*/
101+
public static String sanitizeStream(String definition) {
102+
Assert.hasText(definition, "definition must not be null nor empty");
103+
final StringBuffer output = new StringBuffer();
104+
final Matcher matcher = passwordParameterPatternForStreams.matcher(definition);
105+
while (matcher.find()) {
106+
String passwordValue = matcher.group(3);
107+
108+
String maskedPasswordValue;
109+
boolean isPipeAppended = false;
110+
boolean isNameChannelAppended = false;
111+
if (passwordValue.endsWith("|")) {
112+
isPipeAppended = true;
113+
}
114+
if (passwordValue.endsWith(">")) {
115+
isNameChannelAppended = true;
116+
}
117+
maskedPasswordValue = REDACTION_STRING;
118+
if (isPipeAppended) {
119+
maskedPasswordValue = maskedPasswordValue + " |";
120+
}
121+
if (isNameChannelAppended) {
122+
maskedPasswordValue = maskedPasswordValue + " >";
123+
}
124+
matcher.appendReplacement(output, matcher.group(1) + maskedPasswordValue);
125+
}
126+
matcher.appendTail(output);
127+
return output.toString();
128+
}
129+
76130
}

‎spring-cloud-dataflow-server-core/src/test/java/org/springframework/cloud/dataflow/server/controller/StreamControllerTests.java‎

Lines changed: 115 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,7 @@
9292
* @author Ilayaperumal Gopinathan
9393
* @author Janne Valkealahti
9494
* @author Gunnar Hillert
95+
* @author Glenn Renfro
9596
*/
9697
@RunWith(SpringRunner.class)
9798
@SpringBootTest(classes = TestDependencies.class)
@@ -214,6 +215,7 @@ public void testFindRelatedAndNestedStreams() throws Exception {
214215
assertEquals(0, repository.count());
215216
mockMvc.perform(post("/streams/definitions/").param("name", "myStream1").param("definition", "time | log")
216217
.accept(MediaType.APPLICATION_JSON)).andDo(print()).andExpect(status().isCreated());
218+
217219
mockMvc.perform(post("/streams/definitions/").param("name", "myAnotherStream1")
218220
.param("definition", "time | log").accept(MediaType.APPLICATION_JSON)).andDo(print())
219221
.andExpect(status().isCreated());
@@ -234,20 +236,37 @@ public void testFindRelatedAndNestedStreams() throws Exception {
234236
mockMvc.perform(post("/streams/definitions/").param("name", "myStream4")
235237
.param("definition", ":myAnotherStream1 > log").accept(MediaType.APPLICATION_JSON)).andDo(print())
236238
.andExpect(status().isCreated());
237-
assertEquals(8, repository.count());
239+
240+
mockMvc.perform(post("/streams/definitions/").param("name", "myStream5").param("definition", "time | log --secret=foo")
241+
.accept(MediaType.APPLICATION_JSON)).andDo(print()).andExpect(status().isCreated());
242+
243+
mockMvc.perform(post("/streams/definitions/").param("name", "myStream6")
244+
.param("definition", ":myStream5.time > log --password=bar").accept(MediaType.APPLICATION_JSON)).andDo(print())
245+
.andExpect(status().isCreated());
246+
247+
assertEquals(10, repository.count());
238248
String response = mockMvc
239249
.perform(get("/streams/definitions/myStream1/related?nested=true").accept(MediaType.APPLICATION_JSON))
240250
.andReturn().getResponse().getContentAsString();
241251
assertTrue(response.contains(":myStream1 > log"));
242252
assertTrue(response.contains(":myStream1.time > log"));
243253
assertTrue(response.contains("time | log"));
244254
assertTrue(response.contains("\"totalElements\":6"));
255+
256+
response = mockMvc
257+
.perform(get("/streams/definitions/myStream5/related?nested=true").accept(MediaType.APPLICATION_JSON))
258+
.andReturn().getResponse().getContentAsString();
259+
assertTrue(response.contains(":myStream5.time > log --password=******"));
260+
assertTrue(response.contains("time | log --secret=******"));
261+
assertTrue(response.contains("\"totalElements\":2"));
262+
245263
String response2 = mockMvc.perform(
246264
get("/streams/definitions/myAnotherStream1/related?nested=true").accept(MediaType.APPLICATION_JSON))
247265
.andReturn().getResponse().getContentAsString();
248266
assertTrue(response2.contains(":myAnotherStream1 > log"));
249267
assertTrue(response2.contains("time | log"));
250268
assertTrue(response2.contains("\"totalElements\":2"));
269+
251270
String response3 = mockMvc
252271
.perform(get("/streams/definitions/myStream2/related?nested=true").accept(MediaType.APPLICATION_JSON))
253272
.andReturn().getResponse().getContentAsString();
@@ -256,6 +275,83 @@ public void testFindRelatedAndNestedStreams() throws Exception {
256275
assertTrue(response3.contains("\"totalElements\":2"));
257276
}
258277

278+
@Test
279+
public void testFindAll() throws Exception {
280+
assertEquals(0, repository.count());
281+
mockMvc.perform(post("/streams/definitions/").param("name", "myStream1").param("definition", "time --password=foo| log")
282+
.accept(MediaType.APPLICATION_JSON)).andDo(print()).andExpect(status().isCreated());
283+
mockMvc.perform(post("/streams/definitions/").param("name", "myStream1A").param("definition", "time --foo=bar| log")
284+
.accept(MediaType.APPLICATION_JSON)).andDo(print()).andExpect(status().isCreated());
285+
mockMvc.perform(post("/streams/definitions/").param("name", "myAnotherStream1")
286+
.param("definition", "time | log").accept(MediaType.APPLICATION_JSON)).andDo(print())
287+
.andExpect(status().isCreated());
288+
mockMvc.perform(post("/streams/definitions/").param("name", "myStream2").param("definition", ":myStream1 > log")
289+
.accept(MediaType.APPLICATION_JSON)).andDo(print()).andExpect(status().isCreated());
290+
mockMvc.perform(post("/streams/definitions/").param("name", "TapOnmyStream2")
291+
.param("definition", ":myStream2 > log").accept(MediaType.APPLICATION_JSON)).andDo(print())
292+
.andExpect(status().isCreated());
293+
mockMvc.perform(post("/streams/definitions/").param("name", "myStream3")
294+
.param("definition", ":myStream1.time > log").accept(MediaType.APPLICATION_JSON)).andDo(print())
295+
.andExpect(status().isCreated());
296+
mockMvc.perform(post("/streams/definitions/").param("name", "TapOnMyStream3")
297+
.param("definition", ":myStream3 > log").accept(MediaType.APPLICATION_JSON)).andDo(print())
298+
.andExpect(status().isCreated());
299+
mockMvc.perform(post("/streams/definitions/").param("name", "MultipleNestedTaps")
300+
.param("definition", ":TapOnMyStream3 > log").accept(MediaType.APPLICATION_JSON)).andDo(print())
301+
.andExpect(status().isCreated());
302+
mockMvc.perform(post("/streams/definitions/").param("name", "myStream4")
303+
.param("definition", ":myAnotherStream1 > log").accept(MediaType.APPLICATION_JSON)).andDo(print())
304+
.andExpect(status().isCreated());
305+
mockMvc.perform(post("/streams/definitions")
306+
.param("name", "timelogSingleTick")
307+
.param("definition", "time --format='YYYY MM DD' | log")
308+
.param("deploy", "false"))
309+
.andExpect(status().isCreated());
310+
mockMvc.perform(post("/streams/definitions").param("name", "timelogDoubleTick")
311+
.param("definition", "time --format=\"YYYY MM DD\" | log")
312+
.param("deploy", "false")).andExpect(status().isCreated());
313+
mockMvc.perform(post("/streams/definitions/").param("name", "twoPassword")
314+
.param("definition", "time --password='foo'| log --password=bar")
315+
.accept(MediaType.APPLICATION_JSON)).andDo(print()).andExpect(status().isCreated());
316+
mockMvc.perform(post("/streams/definitions/").param("name", "nameChannelPassword")
317+
.param("definition", "time --password='foo'> :foobar")
318+
.accept(MediaType.APPLICATION_JSON)).andDo(print()).andExpect(status().isCreated());
319+
mockMvc.perform(post("/streams/definitions/").param("name", "twoParam").param("definition", "time --password=foo --arg=foo | log")
320+
.accept(MediaType.APPLICATION_JSON)).andDo(print()).andExpect(status().isCreated());
321+
mockMvc.perform(post("/streams/definitions/").param("name", "twoPipeInQuotes").param("definition", "time --password='fo|o' --arg=bar | log")
322+
.accept(MediaType.APPLICATION_JSON)).andDo(print()).andExpect(status().isCreated());
323+
324+
325+
assertEquals(15, repository.count());
326+
String response = mockMvc
327+
.perform(get("/streams/definitions/").accept(MediaType.APPLICATION_JSON))
328+
.andReturn().getResponse().getContentAsString();
329+
330+
assertTrue(response.contains("time --password=****** | log"));
331+
assertTrue(response.contains("time --foo=bar| log"));
332+
333+
assertTrue(response.contains(":myStream1.time > log"));
334+
assertTrue(response.contains("time | log"));
335+
assertTrue(response.contains(":myStream1 > log"));
336+
assertTrue(response.contains(":myStream1.time > log"));
337+
assertTrue(response.contains("time | log"));
338+
assertTrue(response.contains(":myAnotherStream1 > log"));
339+
assertTrue(response.contains("time | log"));
340+
assertTrue(response.contains(":myStream1 > log"));
341+
assertTrue(response.contains(":myStream2 > log"));
342+
assertTrue(response.contains(":myStream3 > log"));
343+
assertTrue(response.contains("time --format='YYYY MM DD' | log"));
344+
assertTrue(response.contains("time --format=\\\"YYYY MM DD\\\" | log"));
345+
assertTrue(response.contains("time --password=****** | log --password=******"));
346+
System.out.println(response);
347+
assertTrue(response.contains("time --password=****** > :foobar"));
348+
assertTrue(response.contains("time --password=****** --arg=foo | log"));
349+
assertTrue(response.contains("time --password=****** --arg=bar | log"));
350+
351+
assertTrue(response.contains("\"totalElements\":15"));
352+
353+
}
354+
259355
@Test
260356
public void testSaveInvalidAppDefintions() throws Exception {
261357
mockMvc.perform(post("/streams/definitions/").param("name", "myStream").param("definition", "foo | bar")
@@ -503,6 +599,24 @@ public void testDisplaySingleStream() throws Exception {
503599
.andExpect(content().json("{dslText: \"time | log\"}"));
504600
}
505601

602+
@Test
603+
public void testDisplaySingleStreamWithRedaction() throws Exception {
604+
StreamDefinition streamDefinition1 = new StreamDefinition("myStream", "time --secret=foo | log");
605+
for (StreamAppDefinition appDefinition : streamDefinition1.getAppDefinitions()) {
606+
deploymentIdRepository.save(DeploymentKey.forStreamAppDefinition(appDefinition),
607+
streamDefinition1.getName() + "." + appDefinition.getName());
608+
}
609+
repository.save(streamDefinition1);
610+
assertEquals(1, repository.count());
611+
AppStatus status = mock(AppStatus.class);
612+
when(status.getState()).thenReturn(DeploymentState.unknown);
613+
when(appDeployer.status("myStream.time")).thenReturn(status);
614+
when(appDeployer.status("myStream.log")).thenReturn(status);
615+
mockMvc.perform(get("/streams/definitions/myStream").accept(MediaType.APPLICATION_JSON))
616+
.andExpect(status().isOk()).andExpect(content().json("{name: \"myStream\"}"))
617+
.andExpect(content().json("{dslText: \"time --secret=****** | log\"}"));
618+
}
619+
506620
@Test
507621
public void testDestroyStreamNotFound() throws Exception {
508622
mockMvc.perform(delete("/streams/definitions/myStream").accept(MediaType.APPLICATION_JSON)).andDo(print())

0 commit comments

Comments
 (0)