diff --git a/common/src/components/Scraper.jsx b/common/src/components/Scraper.jsx index 52f89370..dace74a4 100644 --- a/common/src/components/Scraper.jsx +++ b/common/src/components/Scraper.jsx @@ -8,86 +8,93 @@ import Fields from './Fields' const customScraperFields = [ { - "field": "id", - "description": "A deterministic or natural id for the resource", - "scheme": "JSONPathOrString", - "required": true + field: 'id', + description: 'A deterministic or natural id for the resource', + scheme: 'JSONPathOrString', + required: true }, { - "field": "name", - default: "`id`", - "scheme": "JSONPathOrString" + field: 'name', + default: '`id`', + scheme: 'JSONPathOrString' }, { - "field": "items", - "description": "Extract multiple config items from this array", - "scheme": "JSONPath" + field: 'items', + description: 'Extract multiple config items from this array', + scheme: 'JSONPath' }, { - "field": "type", - "description": "e.g. `File::Host`, `File::Tomcat`, `File::Pom`", - "scheme": "JSONPathOrString", - "required": "true" + field: 'type', + description: 'e.g. `File::Host`, `File::Tomcat`, `File::Pom`', + scheme: 'JSONPathOrString', + required: 'true' }, { - "field": "class", - "scheme": "JSONPathOrString" + field: 'class', + scheme: 'JSONPathOrString' }, { - "field": "format", - "description": "Format of config item e.g. `xml`, `properties`", - "default": "`JSON`", - "scheme": "string" + field: 'format', + description: 'Format of config item e.g. `xml`, `properties`', + default: '`JSON`', + scheme: 'string' }, { - "field": "timestampFormat", - "description": "Format to parse timestamps in `createFields` and `deletedFields`", - "default": "RFC3339", - "scheme": "Go time format" + field: 'timestampFormat', + description: + 'Format to parse timestamps in `createFields` and `deletedFields`', + default: 'RFC3339', + scheme: 'Go time format' }, { - "field": "createFields", - "description": "Identify the created time for a resource (if different to scrape time). If multiple fields are specified, the first non-empty value will be used", - "scheme": "[]JSONPathOrString" + field: 'createFields', + description: + 'Identify the created time for a resource (if different to scrape time). If multiple fields are specified, the first non-empty value will be used', + scheme: '[]JSONPathOrString' }, { - "field": "deleteFields", - "description": "Identify when a config item was deleted. If multiple fields are specified, the first non-empty value will be used", - "scheme": "[]JSONPathOrString" - }, + field: 'deleteFields', + description: + 'Identify when a config item was deleted. If multiple fields are specified, the first non-empty value will be used', + scheme: '[]JSONPathOrString' + } ] const commonsRows = [ - { - "field": "ignore", - "description": "Fields to ignore and strip out of the config", - "scheme": "[]JSONPath" + field: 'transform', + description: "Transform configs after they've been scraped", + scheme: '[`Transform`](/config-db/concepts/transform)' }, { - "field": "properties", - "description": "Custom templatable properties for the scraped config items.", - "scheme": "[`[]ConfigProperty`](/reference/config-db/properties)" + field: 'properties', + description: 'Custom templatable properties for the scraped config items.', + scheme: '[`[]ConfigProperty`](/reference/config-db/properties)' }, { - "field": "tags", - "description": "Additional tags ", - "scheme": "map[string]string" + field: 'labels', + description: 'Labels for each config item.', + scheme: '`map[string]string`' }, + { + field: 'tags', + description: 'Tags for each config item. Max allowed: 5', + scheme: '[`[]ConfigTag`](/config-db/concepts/tags)' + } ] export function Scraper({ name, edition, rows, ...props }) { - const { siteConfig } = useDocusaurusContext() - - return - } export function CustomScraper({ name, edition, rows, ...props }) { - const { siteConfig } = useDocusaurusContext() - return - + return ( + + ) } diff --git a/mission-control/docs/config-db/scrapers/kubernetes.mdx b/mission-control/docs/config-db/scrapers/kubernetes.mdx index 32277cb8..8c30551c 100644 --- a/mission-control/docs/config-db/scrapers/kubernetes.mdx +++ b/mission-control/docs/config-db/scrapers/kubernetes.mdx @@ -17,32 +17,102 @@ The `kubernetes` scraper collects all of the resources and events in a Kubernete ``` -| Field | Description | Scheme | Required | -| ------------ | ---------------------------------------------------------------------------- | -------------------------------------------- | -------- | -| `logLevel` | Specify the level of logging. | `string` | | -| `schedule` | Specify the interval to scrape in cron format. Defaults to every 60 minutes. | `string` | | -| `retention` | Settings for retaining changes, analysis and scraped items | [`Retention`](/config-db/concepts/retention) | | -| `kubernetes` | Specifies the list of Kubernetes configurations to scrape. | [`[]Kubernetes`](#kubernetes) | | - -### Kubernetes - -| Field | Description | Scheme | Required | -| ---------------- | ----------------------------------------------------------------------------- | ----------------------------------------------------- | -------- | -| `clusterName` | Specify cluster name | `string` | | -| `event` | Specify configuration to handle Kubernetes events. | [`Event`](#events) | | -| `exclusions` | Resources to be excluded from scraping | `[]string` | | -| `fieldSelector` | Resources to be included e.g `status.Phase=Running` | `string` | | -| **`kubeconfig`** | Kubeconfig to connect to the cluster | []_EnvVar_ | | -| `namespace` | Include resources only from this namespace | `string` | | -| `relationships` | Create relationships between kubernetes objects. | [`[]Relationship`](#relationships) | | -| `scope` | Specify scope for scrape. e.g `cluster` for scraping at Cluster level | `string` | | -| `selector` | Include resources matching this selector only e.g `matchLabels` | `string` | | -| `since` | Set time constraint for scraping resources within the set period | `string` | | -| `properties` | Custom properties to be added for each item | [`[]ConfigProperty`](/reference/config-db/properties) | | -| `transform` | Custom transformations to apply | [`Transform`](/config-db/concepts/transform) | | -| `tags` | Tags to set on each config item. `cluster` and `namespace` are set by default | `map[string]string` | | - -## Events +| Field | Description | Scheme | +| ------------ | ---------------------------------------------------------------------------- | -------------------------------------------- | +| `logLevel` | Specify the level of logging. | `string` | +| `schedule` | Specify the interval to scrape in cron format. Defaults to every 60 minutes. | `string` | +| `retention` | Settings for retaining changes, analysis and scraped items | [`Retention`](/config-db/concepts/retention) | +| `kubernetes` | Specifies the list of Kubernetes configurations to scrape. | [`[]Kubernetes`](#kubernetes) | + +## Kubernetes + + + +### Watch Events & Resources + +Use property `watch.disable` to disable watching events & resources. + +#### Events `Kubernetes::Event` resources are mapped to config changes. Events can be very verbose so they can be excluded or their severity level changed: @@ -51,16 +121,35 @@ The `kubernetes` scraper collects all of the resources and events in a Kubernete | `exclusions` | A list of keywords used to exclude event objects based on the reason | `[]string` | | | `severityKeywords` | Specify keywords used to identify the severity of the Kubernetes Event based on the reason | [`SeverityKeywords`](#severitykeywords) | | -### SeverityKeywords +##### SeverityKeywords | Field | Description | Scheme | Required | | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ---------- | -------- | | `warn` | A list of keywords used to identify a warning severity from the reason. It could also be a match pattern: e.g. `*` to match all or `!badword` to exclude `badword` | `[]string` | | | `error` | Same as `warn` but used to map to error severity. | `[]string` | | -## Relationships +#### Watch Selector + + -You can create relationships between kubernetes objects on the basis of +### Relationships + +You can create relationships between kubernetes objects on the basis of kind, name & namespace. :::info [Relationships](../concepts/relationships) can also be defined under `transform.relationships`, however defining them under `kubernetes.relationships` is simpler with specific support for `kind`, `name` and `namespace` fields. @@ -101,7 +190,7 @@ kubernetes: | `name` | `name` of Kubernetes Object | [`Lookup`](#lookup) | `true` | | `namespace` | `namespace` of Kubernetes Object | [`Lookup`](#lookup) | `true` | -##### Lookup +###### Lookup There are 3 different ways to specify which value to use when finding related configs: @@ -111,6 +200,35 @@ There are 3 different ways to specify which value to use when finding related co | `value` | Specify a static value | `string` | | | `label` | Get the value from a label | `string` | | +### Exclusion + +excludes certain kubernetes objects from being scraped + + + ## Annotations Kubernetes resources can be annotated with annotations that can direct the scraper to certain behaviors. @@ -171,3 +289,78 @@ type: Opaque data: token: ... ``` + +## Scraping remote clusters + +A single config-db instance can scrape multiple clusters when provided with a kubeconfig. Either the kubeconfig itself or the path to the kubeconfig can be provided. + +### Local path to kubeconfig + +```yaml title="remote-cluster.yaml" +apiVersion: configs.flanksource.com/v1 +kind: ScrapeConfig +metadata: + name: azure-scraper +spec: + schedule: '@every 5h' + kubernetes: + - clusterName: 'azure production cluster' + kubeconfig: + value: /home/flanksource/.kube/azure_config +``` + +### Kubeconfig + +```yaml title="remote-cluster.yaml" +apiVersion: configs.flanksource.com/v1 +kind: ScrapeConfig +metadata: + name: aws-scraper +spec: +spec: + schedule: '@every 5h' + kubernetes: + - clusterName: 'aws cluster' + kubeconfig: + value: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: xxxxx + server: https://xxxxx.sk1.eu-west-1.eks.amazonaws.com + name: arn:aws:eks:eu-west-1:765618022540:cluster/aws-cluster + contexts: + - context: + cluster: arn:aws:eks:eu-west-1:765618022540:cluster/aws-cluster + namespace: mission-control + user: arn:aws:eks:eu-west-1:765618022540:cluster/aws-cluster + name: arn:aws:eks:eu-west-1:765618022540:cluster/aws-cluster + current-context: arn:aws:eks:eu-west-1:765618022540:cluster/aws-cluster + kind: Config + preferences: {} + users: + - name: arn:aws:eks:eu-west-1:765618022540:cluster/aws-cluster + user: + exec: + .... + +``` + +or, a kubeconfig inside a secret can be referenced as follows: + +```yaml title="remote-cluster.yaml" +apiVersion: configs.flanksource.com/v1 +kind: ScrapeConfig +metadata: + name: aws-scraper +spec: +spec: + schedule: '@every 5h' + kubernetes: + - clusterName: 'aws cluster' + kubeconfig: + valueFrom: + secretKeyRef: + name: aws-kubeconfig + key: kubeconfig +```