Skip to content

Commit 2d12a86

Browse files
Linu-EliasLinu-Elias
authored
[Apache_Tomcat] Ingest pipeline pattern enhancement (#13896)
* Add new field in ingest pipeline * change.log * changelog * lint fix * build fix * build fix * added units * build fix * doc fix * resolved comments
1 parent 2c6b396 commit 2d12a86

File tree

9 files changed

+386
-14
lines changed

9 files changed

+386
-14
lines changed

‎packages/apache_tomcat/_dev/build/docs/README.md‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -89,14 +89,14 @@ Here are the steps to configure Log format in Apache Tomcat instance:
8989
```
9090
<Valve className="org.apache.catalina.valves.AccessLogValve" directory="logs"
9191
prefix="localhost_access_log" suffix=".txt"
92-
pattern='%h %l %u %t "%r" %s %b %A %X %T "%{Referer}i" "%{User-Agent}i" X-Forwarded-For="%{X-Forwarded-For}i"' />
92+
pattern='%h %l %u %t "%r" %s %b %A %X %F "%{Referer}i" "%{User-Agent}i" X-Forwarded-For="%{X-Forwarded-For}i"' />
9393
```
9494

9595
3. The supported log formats are:
9696
```
97-
Common Log Format :- '%h %l %u %t "%r" %s %b'
98-
Combined Log Format :- '%h %l %u %t "%r" %s %b "%{Referrer}i" "%{User-Agent}i"'
99-
Combined Log Format + X-Forwarded-For header :- '%h %l %u %t "%r" %s %b %A %X %T "%{Referer}i" "%{User-Agent}i" X-Forwarded-For="%{X-Forwarded-For}i"'
97+
Common Log Format :- '%h %l %u %t "%r" %s %b ms:%D'
98+
Combined Log Format :- '%h %l %u %t "%r" %s %b ms:%D "%{Referrer}i" "%{User-Agent}i"'
99+
Combined Log Format + X-Forwarded-For header :- '%h %l %u %t "%r" %s %b ms:%D %A %X %F "%{Referer}i" "%{User-Agent}i" X-Forwarded-For="%{X-Forwarded-For}i"'
100100
```
101101

102102
4. Run the following commands to restart Apache Tomcat instance: -

‎packages/apache_tomcat/changelog.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,9 @@
11
# newer versions go on top
2+
- version: "1.10.0"
3+
changes:
4+
- description: Added support for %D attribute in access logs
5+
type: enhancement
6+
link: https://github.com/elastic/integrations/pull/13896
27
- version: "1.9.1"
38
changes:
49
- description: Added description to ssl nodes including links to documentation.

‎packages/apache_tomcat/data_stream/access/_dev/test/pipeline/test-access.log‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,4 +8,9 @@
88
81.2.69.144 - admin [02/Mar/2023:19:01:17 +0530] "GET /manager/status HTTP/1.1" 200 4654 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36
99
81.2.69.144 - admin [02/Mar/2023:19:02:25 +0530] "GET / HTTP/1.1" 200 11235
1010
81.2.69.144 - - [24/Oct/2024:14:18:49 +1100] "-" 400 - 81.2.69.145 + 0.000 "-" "-" X-Forwarded-For="-"
11-
10.10.10.10 - - [28/May/2024:17:20:05 +0200] "GET / " 200 17
11+
10.10.10.10 - - [28/May/2024:17:20:05 +0200] "GET / " 200 17
12+
81.2.69.144 - admin [02/Mar/2023:18:58:17 +0530] "POST /host-manager/images/asf-logo.svg HTTP/1.1" 200 20486 ms:71198 "http://localhost:8080/host-manager/html" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36" X-Forwarded-For=""
13+
81.2.69.144 - admin [02/Mar/2023:18:58:17 +0530] "POST /host-manager/images/asf-logo.svg HTTP/1.1" 200 20486 ms:54321 X 400 "http://localhost:8080/host-manager/html" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36" X-Forwarded-For="127.0.0.1"
14+
81.2.69.144 - admin [02/Mar/2023:18:58:17 +0530] "POST /host-manager/images/asf-logo.svg HTTP/1.1" 200 20486 ms:3214 81.2.69.145 40 "http://localhost:8080/host-manager/html" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36" X-Forwarded-For="127.0.0.1, 127.0.0.3"
15+
81.2.69.144 - admin [02/Mar/2023:18:58:17 +0530] "POST /host-manager/images/asf-logo.svg HTTP/1.1" 200 20486 ms:98765 50 "http://localhost:8080/host-manager/html" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36" X-Forwarded-For=""
16+
10.10.10.10 - - [28/May/2024:17:20:05 +0200] "GET / " 200 17 ms:1234

‎packages/apache_tomcat/data_stream/access/_dev/test/pipeline/test-access.log-expected.json‎

Lines changed: 344 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -746,6 +746,350 @@
746746
"tags": [
747747
"preserve_original_event"
748748
]
749+
},
750+
{
751+
"@timestamp": "2023-03-02T13:28:17.000Z",
752+
"apache_tomcat": {
753+
"access": {
754+
"http": {
755+
"ident": "-",
756+
"useragent": "admin"
757+
},
758+
"request_process_time": 71198.0
759+
}
760+
},
761+
"destination": {
762+
"bytes": 20486
763+
},
764+
"ecs": {
765+
"version": "8.11.0"
766+
},
767+
"event": {
768+
"category": [
769+
"web"
770+
],
771+
"kind": "event",
772+
"module": "apache_tomcat",
773+
"original": "81.2.69.144 - admin [02/Mar/2023:18:58:17 +0530] \"POST /host-manager/images/asf-logo.svg HTTP/1.1\" 200 20486 ms:71198 \"http://localhost:8080/host-manager/html\" \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36\" X-Forwarded-For=\"\"",
774+
"outcome": "success",
775+
"type": [
776+
"access"
777+
]
778+
},
779+
"http": {
780+
"request": {
781+
"method": "POST",
782+
"referrer": "http://localhost:8080/host-manager/html"
783+
},
784+
"response": {
785+
"status_code": 200
786+
},
787+
"version": "1.1"
788+
},
789+
"related": {
790+
"ip": [
791+
"81.2.69.144"
792+
]
793+
},
794+
"source": {
795+
"ip": "81.2.69.144"
796+
},
797+
"tags": [
798+
"preserve_original_event"
799+
],
800+
"url": {
801+
"extension": "svg",
802+
"original": "/host-manager/images/asf-logo.svg",
803+
"path": "/host-manager/images/asf-logo.svg"
804+
},
805+
"user_agent": {
806+
"device": {
807+
"name": "Other"
808+
},
809+
"name": "Chrome",
810+
"original": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36",
811+
"os": {
812+
"full": "Windows 10",
813+
"name": "Windows",
814+
"version": "10"
815+
},
816+
"version": "109.0.0.0"
817+
}
818+
},
819+
{
820+
"@timestamp": "2023-03-02T13:28:17.000Z",
821+
"apache_tomcat": {
822+
"access": {
823+
"connection_status": "X",
824+
"http": {
825+
"ident": "-",
826+
"useragent": "admin"
827+
},
828+
"request_process_time": 54321.0,
829+
"response_time": 400.0
830+
}
831+
},
832+
"client": {
833+
"ip": [
834+
"127.0.0.1"
835+
]
836+
},
837+
"destination": {
838+
"bytes": 20486
839+
},
840+
"ecs": {
841+
"version": "8.11.0"
842+
},
843+
"event": {
844+
"category": [
845+
"web"
846+
],
847+
"kind": "event",
848+
"module": "apache_tomcat",
849+
"original": "81.2.69.144 - admin [02/Mar/2023:18:58:17 +0530] \"POST /host-manager/images/asf-logo.svg HTTP/1.1\" 200 20486 ms:54321 X 400 \"http://localhost:8080/host-manager/html\" \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36\" X-Forwarded-For=\"127.0.0.1\"",
850+
"outcome": "success",
851+
"type": [
852+
"access"
853+
]
854+
},
855+
"http": {
856+
"request": {
857+
"method": "POST",
858+
"referrer": "http://localhost:8080/host-manager/html"
859+
},
860+
"response": {
861+
"status_code": 200
862+
},
863+
"version": "1.1"
864+
},
865+
"related": {
866+
"ip": [
867+
"81.2.69.144",
868+
"127.0.0.1"
869+
]
870+
},
871+
"source": {
872+
"ip": "81.2.69.144"
873+
},
874+
"tags": [
875+
"preserve_original_event"
876+
],
877+
"url": {
878+
"extension": "svg",
879+
"original": "/host-manager/images/asf-logo.svg",
880+
"path": "/host-manager/images/asf-logo.svg"
881+
},
882+
"user_agent": {
883+
"device": {
884+
"name": "Other"
885+
},
886+
"name": "Chrome",
887+
"original": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36",
888+
"os": {
889+
"full": "Windows 10",
890+
"name": "Windows",
891+
"version": "10"
892+
},
893+
"version": "109.0.0.0"
894+
}
895+
},
896+
{
897+
"@timestamp": "2023-03-02T13:28:17.000Z",
898+
"apache_tomcat": {
899+
"access": {
900+
"http": {
901+
"ident": "-",
902+
"useragent": "admin"
903+
},
904+
"ip": {
905+
"local": "81.2.69.145"
906+
},
907+
"request_process_time": 3214.0,
908+
"response_time": 40.0
909+
}
910+
},
911+
"client": {
912+
"ip": [
913+
"127.0.0.1",
914+
"127.0.0.3"
915+
]
916+
},
917+
"destination": {
918+
"bytes": 20486
919+
},
920+
"ecs": {
921+
"version": "8.11.0"
922+
},
923+
"event": {
924+
"category": [
925+
"web"
926+
],
927+
"kind": "event",
928+
"module": "apache_tomcat",
929+
"original": "81.2.69.144 - admin [02/Mar/2023:18:58:17 +0530] \"POST /host-manager/images/asf-logo.svg HTTP/1.1\" 200 20486 ms:3214 81.2.69.145 40 \"http://localhost:8080/host-manager/html\" \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36\" X-Forwarded-For=\"127.0.0.1, 127.0.0.3\"",
930+
"outcome": "success",
931+
"type": [
932+
"access"
933+
]
934+
},
935+
"http": {
936+
"request": {
937+
"method": "POST",
938+
"referrer": "http://localhost:8080/host-manager/html"
939+
},
940+
"response": {
941+
"status_code": 200
942+
},
943+
"version": "1.1"
944+
},
945+
"related": {
946+
"ip": [
947+
"81.2.69.144",
948+
"81.2.69.145",
949+
"127.0.0.1",
950+
"127.0.0.3"
951+
]
952+
},
953+
"source": {
954+
"ip": "81.2.69.144"
955+
},
956+
"tags": [
957+
"preserve_original_event"
958+
],
959+
"url": {
960+
"extension": "svg",
961+
"original": "/host-manager/images/asf-logo.svg",
962+
"path": "/host-manager/images/asf-logo.svg"
963+
},
964+
"user_agent": {
965+
"device": {
966+
"name": "Other"
967+
},
968+
"name": "Chrome",
969+
"original": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36",
970+
"os": {
971+
"full": "Windows 10",
972+
"name": "Windows",
973+
"version": "10"
974+
},
975+
"version": "109.0.0.0"
976+
}
977+
},
978+
{
979+
"@timestamp": "2023-03-02T13:28:17.000Z",
980+
"apache_tomcat": {
981+
"access": {
982+
"http": {
983+
"ident": "-",
984+
"useragent": "admin"
985+
},
986+
"request_process_time": 98765.0,
987+
"response_time": 50.0
988+
}
989+
},
990+
"destination": {
991+
"bytes": 20486
992+
},
993+
"ecs": {
994+
"version": "8.11.0"
995+
},
996+
"event": {
997+
"category": [
998+
"web"
999+
],
1000+
"kind": "event",
1001+
"module": "apache_tomcat",
1002+
"original": "81.2.69.144 - admin [02/Mar/2023:18:58:17 +0530] \"POST /host-manager/images/asf-logo.svg HTTP/1.1\" 200 20486 ms:98765 50 \"http://localhost:8080/host-manager/html\" \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36\" X-Forwarded-For=\"\"",
1003+
"outcome": "success",
1004+
"type": [
1005+
"access"
1006+
]
1007+
},
1008+
"http": {
1009+
"request": {
1010+
"method": "POST",
1011+
"referrer": "http://localhost:8080/host-manager/html"
1012+
},
1013+
"response": {
1014+
"status_code": 200
1015+
},
1016+
"version": "1.1"
1017+
},
1018+
"related": {
1019+
"ip": [
1020+
"81.2.69.144"
1021+
]
1022+
},
1023+
"source": {
1024+
"ip": "81.2.69.144"
1025+
},
1026+
"tags": [
1027+
"preserve_original_event"
1028+
],
1029+
"url": {
1030+
"extension": "svg",
1031+
"original": "/host-manager/images/asf-logo.svg",
1032+
"path": "/host-manager/images/asf-logo.svg"
1033+
},
1034+
"user_agent": {
1035+
"device": {
1036+
"name": "Other"
1037+
},
1038+
"name": "Chrome",
1039+
"original": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36",
1040+
"os": {
1041+
"full": "Windows 10",
1042+
"name": "Windows",
1043+
"version": "10"
1044+
},
1045+
"version": "109.0.0.0"
1046+
}
1047+
},
1048+
{
1049+
"@timestamp": "2024-05-28T15:20:05.000Z",
1050+
"apache_tomcat": {
1051+
"access": {
1052+
"http": {
1053+
"ident": "-",
1054+
"useragent": "-"
1055+
},
1056+
"request_process_time": 1234.0
1057+
}
1058+
},
1059+
"destination": {
1060+
"bytes": 17
1061+
},
1062+
"ecs": {
1063+
"version": "8.11.0"
1064+
},
1065+
"event": {
1066+
"category": [
1067+
"web"
1068+
],
1069+
"kind": "event",
1070+
"module": "apache_tomcat",
1071+
"original": "10.10.10.10 - - [28/May/2024:17:20:05 +0200] \"GET / \" 200 17 ms:1234",
1072+
"outcome": "success",
1073+
"type": [
1074+
"access"
1075+
]
1076+
},
1077+
"http": {
1078+
"response": {
1079+
"status_code": 200
1080+
}
1081+
},
1082+
"related": {
1083+
"ip": [
1084+
"10.10.10.10"
1085+
]
1086+
},
1087+
"source": {
1088+
"ip": "10.10.10.10"
1089+
},
1090+
"tags": [
1091+
"preserve_original_event"
1092+
]
7491093
}
7501094
]
7511095
}

0 commit comments

Comments
 (0)